groceryheist/cdsc_reddit
1
0
Fork 0
Code Activity

changes for archiving.

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2023-05-23 17:18:19 -07:00
parent 811a0d87c4
commit 07b0dff9bc
47 changed files with 396 additions and 1549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
datasets/Makefile Normal file
View File

@@ -0,0 +1,28 @@
all: ../../data/reddit_comments_by_subreddit.parquet ../../data/reddit_submissions_by_subreddit.parquet
../../data/reddit_comments_by_subreddit.parquet:../../data/temp/reddit_comments.parquet
../start_spark_and_run.sh 4 comments_2_parquet_part2.py
../../data/temp/reddit_comments.parquet: comments_task_list.sh run_comments_jobs.sbatch
mkdir -p comments_jobs
mkdir -p ../../data/temp/
sbatch --wait --array=1-$(shell cat comments_task_list.sh | wc -l) run_comments_jobs.sbatch 0
temp_reddit_comments.parquet: ../../data/temp/reddit_comments.parquet
comments_task_list.sh: comments_2_parquet_part1.py
srun -p compute-bigmem -A comdata --nodes=1 --mem-per-cpu=9g -c 40 --time=120:00:00 bash -c "source ~/.bashrc && python3 comments_2_parquet_part1.py gen_task_list --overwrite=False"
submissions_task_list.sh: submissions_2_parquet_part1.py
srun -p compute-bigmem -A comdata --nodes=1 --mem-per-cpu=9g -c 40 --time=120:00:00 python3 submissions_2_parquet_part1.py gen_task_list
../../data/reddit_submissions_by_subreddit.parquet:../../data/temp/reddit_submissions.parquet
../start_spark_and_run.sh 4 submissions_2_parquet_part2.py
../../data/temp/reddit_submissions.parquet: submissions_task_list.sh run_submissions_jobs.sbatch
mkdir -p submissions_jobs
rm -rf ../../data/temp/reddit_submissions.parquet
mkdir -p ../../data/temp/
sbatch --wait --array=1-$(shell cat submissions_task_list.sh | wc -l) run_submissions_jobs.sbatch 0
temp_reddit_submissions.parquet: ../../data/temp/reddit_submissions.parquet