collective/cdsc_reddit
20
0
Fork 0
Code Activity

Bugfixes in scripts.

Browse Source
This commit is contained in:
Nate E TeBlunthuis
2020-07-07 23:29:36 -07:00
parent 06fd99e7cd
commit aa84a7df03
3 changed files with 10 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
comments_2_parquet.sh
View File

@@ -1,9 +1,9 @@
#!/usr/bin/env bash
echo "!#/usr/bin/bash" > job_script.sh
echo "#!/usr/bin/bash" > job_script.sh
echo "source $(pwd)/../bin/activate" >> job_script.sh
echo "python3 $(pwd)/comments_2_parquet_part1.py" >> job_script.sh
srun -p comdata -A comdata --nodes=1 --mem=120G --time=48:00:00 job_script.sh
srun -p comdata -A comdata --nodes=1 --mem=120G --time=48:00:00 --pty job_script.sh
start_spark_and_run.sh 1 $(pwd)/comments_2_parquet_part2.py