collective/wikiq
20
0
Fork 0
Code Issues Pull Requests 3 Activity

add more variables and support for persistence

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2018-08-31 15:57:48 -07:00
parent bc1f5428f0
commit 915c864ee5
2 changed files with 74 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
wikiq_users/run_wikiq_users_cluster.sh
View File

@@ -1,2 +1,2 @@
#!/usr/bin/env bash
spark-submit --master spark://n0649:18899 wikiq_users_spark.py --output-format parquet -i "/com/output/wikiq-enwiki-20180301/enwiki-20180301-pages-meta-history*.tsv" -o "/com/output/wikiq-users-enwiki-20180301-parquet/" --num-partitions 500
spark-submit --master spark://n0649:18899 wikiq_users_spark.py --output-format parquet -i "/com/output/wikiq-enwiki-persist-sequence-20180301/enwiki/enwiki-20180301-pages-meta-history*.tsv" -o "/com/output/wikiq-users-enwiki-20180301-parquet/" --num-partitions 500 --schema-opt persistence+collapse