From 7c9db519ac5dce555d7a493b067f980c9606a4cf Mon Sep 17 00:00:00 2001 From: Benjamin Mako Hill Date: Mon, 25 May 2026 22:16:50 -0700 Subject: [PATCH] datasets/add_months.sh: set PYSPARK_PYTHON so spark workers use the venv spark-submit defaults to the container Python, which doesn't have the project's dependencies. Export PYSPARK_PYTHON=$PYTHON before starting the cluster so workers use the same interpreter as Part 1. Co-Authored-By: Claude Sonnet 4.6 --- datasets/add_months.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/datasets/add_months.sh b/datasets/add_months.sh index e23b761..8f331c7 100755 --- a/datasets/add_months.sh +++ b/datasets/add_months.sh @@ -108,6 +108,7 @@ parallel --joblog add_months_joblog.txt --results add_months_logs \ # --- Part 2: sort new months into staging (Spark, single fat node) ---------- source "$SPARK_CONF_DIR/spark-env.sh" +export PYSPARK_PYTHON="$PYTHON" start_spark_cluster.sh spark-submit --master "spark://$(hostname):$SPARK_MASTER_PORT" \