aggregate one variable on a subset of the dataframe and the other on the total dataframe pandas code example

Example 1: Groups the DataFrame using the specified columns

# Groups the DataFrame using the specified columns

df.groupBy().avg().collect()
# [Row(avg(age)=3.5)]
sorted(df.groupBy('name').agg({'age': 'mean'}).collect())
# [Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]
sorted(df.groupBy(df.name).avg().collect())
# [Row(name='Alice', avg(age)=2.0), Row(name='Bob', avg(age)=5.0)]
sorted(df.groupBy(['name', df.age]).count().collect())
# [Row(name='Alice', age=2, count=1), Row(name='Bob', age=5, count=1)]

Example 2: Aggregate on the entire DataFrame without group

# Aggregate on the entire DataFrame without group

df.agg({"age": "max"}).collect()
# [Row(max(age)=5)]
from pyspark.sql import functions as F
df.agg(F.min(df.age)).collect()
# [Row(min(age)=2)]