collective/cdsc_reddit
18
0
Fork 0
Code Activity

Updating to support wang-style user overlaps.

Browse Source
This commit is contained in:
Nate E TeBlunthuis
2020-12-24 22:38:04 -08:00
parent 56269deee3
commit 4e20dce188
11 changed files with 193 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
similarities/job_script.sh
View File

@@ -1,4 +1,4 @@
#!/usr/bin/bash
start_spark_cluster.sh
spark-submit --master spark://$(hostname):18899 weekly_cosine_similarities.py term --outfile=/gscratch/comdata/output/reddit_similarity/subreddit_comment_terms_10000_weely.parquet
spark-submit --master spark://$(hostname):18899 wang_similarity.py --infile=/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet --max_df=10 --outfile=/gscratch/comdata/output/reddit_similarity/wang_similarity_1000_max10.feather
stop-all.sh