yuankui
1/3/2019 - 3:02 AM

#spark #options

#spark #options

{
    "spark.yarn.maxAppAttempts": "1",
    "spark.serializer": "org.apache.spark.serializer.KryoSerializer",
    "spark.kryoserializer.buffer.mb": "32",
    "spark.default.parallelism": "500",
    "spark.sql.shuffle.partitions": "1000",
    "spark.executor.cores": "4"
}