collective/cdsc_reddit
18
0
Fork 0
Code Activity

Changes from hyak.

Browse Source
This commit is contained in:
Nate E TeBlunthuis
2021-02-22 16:03:48 -08:00
parent 140d1bdd17
commit 4dc949de5f
14 changed files with 128 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
similarities/job_script.sh
View File

@@ -1,4 +1,4 @@
#!/usr/bin/bash
start_spark_cluster.sh
spark-submit --master spark://$(hostname):18899 wang_similarity.py --infile=/gscratch/comdata/output/reddit_similarity/tfidf/comment_authors.parquet --max_df=10 --outfile=/gscratch/comdata/output/reddit_similarity/wang_similarity_1000_max10.feather
spark-submit --master spark://$(hostname):18899 cosine_similarities.py term --outfile=/gscratch/comdata/output/reddit_similarity/comment_terms_10000.feather
stop-all.sh