29 lines
1.5 KiB
Makefile
29 lines
1.5 KiB
Makefile
|
all: ../../data/reddit_comments_by_subreddit.parquet ../../data/reddit_submissions_by_subreddit.parquet
|
||
|
|
|
||
|
|
../../data/reddit_comments_by_subreddit.parquet:../../data/temp/reddit_comments.parquet
|
||
|
|
../start_spark_and_run.sh 4 comments_2_parquet_part2.py
|
||
|
|
|
||
|
|
../../data/temp/reddit_comments.parquet: comments_task_list.sh run_comments_jobs.sbatch
|
||
|
|
mkdir -p comments_jobs
|
||
|
|
mkdir -p ../../data/temp/
|
||
|
|
sbatch --wait --array=1-$(shell cat comments_task_list.sh | wc -l) run_comments_jobs.sbatch 0
|
||
|
|
|
||
|
|
temp_reddit_comments.parquet: ../../data/temp/reddit_comments.parquet
|
||
|
|
|
||
|
|
comments_task_list.sh: comments_2_parquet_part1.py
|
||
|
|
srun -p compute-bigmem -A comdata --nodes=1 --mem-per-cpu=9g -c 40 --time=120:00:00 bash -c "source ~/.bashrc && python3 comments_2_parquet_part1.py gen_task_list --overwrite=False"
|
||
|
|
|
||
|
|
submissions_task_list.sh: submissions_2_parquet_part1.py
|
||
|
|
srun -p compute-bigmem -A comdata --nodes=1 --mem-per-cpu=9g -c 40 --time=120:00:00 python3 submissions_2_parquet_part1.py gen_task_list
|
||
|
|
|
||
|
|
../../data/reddit_submissions_by_subreddit.parquet:../../data/temp/reddit_submissions.parquet
|
||
|
|
../start_spark_and_run.sh 4 submissions_2_parquet_part2.py
|
||
|
|
|
||
|
|
../../data/temp/reddit_submissions.parquet: submissions_task_list.sh run_submissions_jobs.sbatch
|
||
|
|
mkdir -p submissions_jobs
|
||
|
|
rm -rf ../../data/temp/reddit_submissions.parquet
|
||
|
|
mkdir -p ../../data/temp/
|
||
|
|
sbatch --wait --array=1-$(shell cat submissions_task_list.sh | wc -l) run_submissions_jobs.sbatch 0
|
||
|
|
|
||
|
|
temp_reddit_submissions.parquet: ../../data/temp/reddit_submissions.parquet
|