pandas duplicate row code example
Example 1: Return a new DataFrame with duplicate rows removed
# Return a new DataFrame with duplicate rows removed
from pyspark.sql import Row
df = sc.parallelize([
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# | 5| 80|Alice|
# | 10| 80|Alice|
# +---+------+-----+
df.dropDuplicates(['name', 'height']).show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# | 5| 80|Alice|
# +---+------+-----+
Example 2: how to duplicate a row in python
import pandas as pd
In [603]: df = pd.DataFrame({'col1':list("abc"),'col2':range(3)},index = range(3))
In [604]: df
Out[604]:
col1 col2
0 a 0
1 b 1
2 c 2
In [605]: pd.concat([df]*3, ignore_index=True) # Ignores the index
Out[605]:
col1 col2
0 a 0
1 b 1
2 c 2
3 a 0
4 b 1
5 c 2
6 a 0
7 b 1
8 c 2
In [606]: pd.concat([df]*3)
Out[606]:
col1 col2
0 a 0
1 b 1
2 c 2
0 a 0
1 b 1
2 c 2
0 a 0
1 b 1
2 c 2