groceryheist/cdsc_reddit
1
0
Fork 0
Code

bugfix.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-01 15:28:25 -08:00
parent b25c332cea
commit 224fb89317
1 changed files with 0 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
ngrams/term_frequencies.py
View File

@ -248,9 +248,6 @@ def sort_tf(input_parquet="/gscratch/comdata/output/temp_reddit_comments_by_subr
from pyspark.sql import functions as f from pyspark.sql import functions as f
from pyspark.sql import SparkSession from pyspark.sql import SparkSession
spark = SparkSession.builder.config(map={'spark.executor.memory':'900g'}).getOrCreate() spark = SparkSession.builder.config(map={'spark.executor.memory':'900g'}).getOrCreate()
getOrCreate()
df = spark.read.parquet(input_parquet) df = spark.read.parquet(input_parquet)
df = df.repartition(2000,tf_name) df = df.repartition(2000,tf_name)
df = df.sort([tf_name,'week','subreddit']) df = df.sort([tf_name,'week','subreddit'])