python drop_duplicates code example

Example 1: remove duplicates function python

def remove_dupiclates(list_):
	new_list = []
	for a in list_:
    	if a not in new_list:
        	new_list.append(a)
	return new_list

Example 2: remove duplicates python

mylist = ["a", "b", "a", "c", "c"]
mylist = list(dict.fromkeys(mylist))

Example 3: python remove duplicates

if mylist:
    mylist.sort()
    last = mylist[-1]
    for i in range(len(mylist)-2, -1, -1):
        if last == mylist[i]:
            del mylist[i]
        else:
            last = mylist[i]
# Quicker if all elements are hashables:
mylist = list(set(mylist))

Example 4: Return a new DataFrame with duplicate rows removed

# Return a new DataFrame with duplicate rows removed

from pyspark.sql import Row
df = sc.parallelize([
  Row(name='Alice', age=5, height=80),
  Row(name='Alice', age=5, height=80),
  Row(name='Alice', age=10, height=80)]).toDF()
df.dropDuplicates().show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# |  5|    80|Alice|
# | 10|    80|Alice|
# +---+------+-----+

df.dropDuplicates(['name', 'height']).show()
# +---+------+-----+
# |age|height| name|
# +---+------+-----+
# |  5|    80|Alice|
# +---+------+-----+