obar1
11/2/2017 - 1:38 AM

Initializing spark job using spark-shell

Initializing spark job using spark-shell

spark-shell --master yarn \
  --conf spark.ui.port=12654
  
// hadoop fs -ls /public/retail_db
// hadoop fs -du -s -h /public/retail_db
// Properties files location /etc/spark/conf/
spark-shell --master yarn \
  --conf spark.ui.port=12654 \
  --num-executors 1 \
  --executor-memory 512M
sc.getConf.getAll.foreach(println)

//Initialize programmatically
import org.apache.spark.{SparkConf, SparkContext}
val conf = new SparkConf().setAppName("Daily Revenue").setMaster("yarn-client")
val sc = new SparkContext(conf)
sc.getConf.getAll.foreach(println)