rui-liu
2/24/2020 - 2:40 AM

PySpark snippets

PySpark snippets

from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType, FloatType
from interaction import hive_common

sc = SparkContext(appName="recommend 2c for team ...")
spark = SparkSession.builder \
    .appName(APP_NAME) \
    .enableHiveSupport() \
    .getOrCreate()

rdd_left = hive_common.get_data(spark, tbl_left).rdd
/opt/spark2-hadoop2.6/bin/spark-submit --master yarn --queue data --deploy-mode client --num-executors 20 --driver-memory 3G --executor-memory 1G --conf spark.yarn.executor.memoryOverhead=4096 --py-files dist/spark_knn-0.1-py3.7.egg cosine_knn.py -l dm_ai.r_df_cps_title_embdding_input_left -r dm_ai.r_df_cps_title_embdding_input_right -o tmp.test_knn_spark -s 6 -rk -k 50> run_log.txt 2>&1