pyspark返回每个分组某个值最大的行

这种用reducebykey就可以了

from pyspark.sql import functions as F
from pyspark.sql.window import Window
df = sql_context.createDataFrame([
("A", "ARON", 1),
("A", "BILL", 2),
("A", "CLAIR", 3),
("B", "DANIEL", 1),
("B", "ERIC", 4),
("B", "FRANK", 2),
], ["id", "name", "weight"])

def max_row(row1,row2):

    if row1['weight'] > row2['weight']:
       return row1
    else:
        return row2


df_rdd = df.rdd.map(lambda row: (row['id'], row)).\
reduceByKey(lambda row1,row2: max_row(row1,row2)).map(lambda row: row[1])
sql_context.createDataFrame(df_rdd).show()

或者用分组排序后取第一个

windows_spec = Window.partitionBy("id").orderBy(F.col("weight").desc())
max_df = df.withColumn("rank", F.rank().over(windows_spec)).filter("rank =1")
max_df.show()

 可能第一个会快一些吧，一般也都是用第一个