groceryheist/cdsc_reddit
1
0
Fork 0
Code

fix configuration code

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-01 15:21:51 -08:00
parent 9911f758f9
commit abe217d2d5
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ngrams/term_frequencies.py
View File

@ -246,9 +246,10 @@ def sort_tf(input_parquet="/gscratch/comdata/output/temp_reddit_comments_by_subr
from pyspark.sql import functions as f
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
spark.config('spark.executor.memory','900g')
spark = SparkSession.builder.config(map={'spark.executor.memory':'900g'}).getOrCreate()
getOrCreate()
df = spark.read.parquet(input_parquet)
df = df.repartition(2000,tf_name)
df = df.sort([tf_name,'week','subreddit'])