diff --git a/datasets/add_months.sh b/datasets/add_months.sh index e23b761..8f331c7 100755 --- a/datasets/add_months.sh +++ b/datasets/add_months.sh @@ -108,6 +108,7 @@ parallel --joblog add_months_joblog.txt --results add_months_logs \ # --- Part 2: sort new months into staging (Spark, single fat node) ---------- source "$SPARK_CONF_DIR/spark-env.sh" +export PYSPARK_PYTHON="$PYTHON" start_spark_cluster.sh spark-submit --master "spark://$(hostname):$SPARK_MASTER_PORT" \