pandas drop duplicates on column code example
Example 1: drop duplicates pandas first column
import pandas as pd
# making data frame from csv file
data = pd.read_csv("employees.csv")
# sorting by first name
data.sort_values("First Name", inplace = True)
# dropping ALL duplicte values
data.drop_duplicates(subset ="First Name",keep = False, inplace = True)
# displaying data
print(data)
Example 2: remove duplicate row in df
df = df.drop_duplicates()
Example 3: Return a new DataFrame with duplicate rows removed
# Return a new DataFrame with duplicate rows removed
from pyspark.sql import Row
df = sc.parallelize([
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# | 5| 80|Alice|
# | 10| 80|Alice|
# +---+------+-----+
df.dropDuplicates(['name', 'height']).show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# | 5| 80|Alice|
# +---+------+-----+