drop_duplicates in python code example
Example 1: drop duplicates pandas first column
import pandas as pd
data = pd.read_csv("employees.csv")
data.sort_values("First Name", inplace = True)
data.drop_duplicates(subset ="First Name",keep = False, inplace = True)
print(data)
Example 2: remove duplicates function python
def remove_dupiclates(list_):
new_list = []
for a in list_:
if a not in new_list:
new_list.append(a)
return new_list
Example 3: remove duplicates python
mylist = ["a", "b", "a", "c", "c"]
mylist = list(dict.fromkeys(mylist))
Example 4: remove duplicate columns python dataframe
df = df.loc[:,~df.columns.duplicated()]
Example 5: python remove duplicates
word = input().split()
for i in word:
if word.count(i) > 1:
word.remove(i)
Example 6: Return a new DataFrame with duplicate rows removed
from pyspark.sql import Row
df = sc.parallelize([
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=5, height=80),
Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
df.dropDuplicates(['name', 'height']).show()