29 lines
1.5 KiB
Makefile
29 lines
1.5 KiB
Makefile
srun=srun -p compute-bigmem -A comdata --mem-per-cpu=9g --time=200:00:00 -c 40
|
|
srun_huge=srun -p compute-hugemem -A comdata --mem=724g --time=200:00:00 -c 40
|
|
|
|
similarity_data=../../data/reddit_similarity
|
|
tfidf_data=${similarity_data}/tfidf
|
|
lsi_components=[10,50,100,200,300,400,500,600,700,850]
|
|
|
|
lsi_similarities: ${similarity_data}/subreddit_comment_authors-tf_10k_LSI
|
|
|
|
all: ${similarity_data}/subreddit_comment_authors-tf_10k.feather
|
|
|
|
${similarity_data}/subreddit_comment_authors-tf_10k_LSI: ${tfidf_data}/comment_authors_100k.parquet similarities_helper.py ${similarity_data}/subreddits_by_num_comments_nonsfw.csv
|
|
${srun_huge} /bin/bash -c "source ~/.bashrc; python3 lsi_similarities.py author-tf --outfile=${similarity_data}/subreddit_comment_authors-tf_10k_LSI --topN=10000 --n_components=${lsi_components} --min_df=10 --inpath=$<"
|
|
|
|
${similarity_data}/subreddits_by_num_comments_nonsfw.csv: ../../data/reddit_submissions_by_subreddit.parquet ../../data/reddit_comments_by_subreddit.parquet
|
|
../start_spark_and_run.sh 3 top_subreddits_by_comments.py
|
|
|
|
${tfidf_data}/comment_authors_100k.parquet: ../../data/reddit_ngrams/comment_authors_sorted.parquet ${similarity_data}/subreddits_by_num_comments_nonsfw.csv
|
|
../start_spark_and_run.sh 3 tfidf.py authors --topN=100000 --inpath=$< --outpath=${tfidf_data}/comment_authors_100k.parquet
|
|
|
|
../../data/reddit_ngrams/comment_authors_sorted.parquet:
|
|
$(MAKE) -C ../ngrams
|
|
|
|
../../data/reddit_submissions_by_subreddit.parquet:
|
|
$(MAKE) -C ../datasets
|
|
|
|
../../data/reddit_comments_by_subreddit.parquet:
|
|
$(MAKE) -C ../datasets
|