collective/cdsc_reddit
13
0
Fork 0
Code

code to sort tf

Browse Source
This commit is contained in:
Nate E TeBlunthuis 2020-08-03 17:56:36 -07:00
parent c666302b4a
commit 40be7bedb6
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
sort_tf_comments.py Normal file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env python3
from pyspark.sql import functions as f
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
df = spark.read.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test.parquet_temp/")
df = df.repartition(2000,'term')
df = df.sort(['term','week','subreddit'])
df = df.sortWithinPartitions(['term','week','subreddit'])
df.write.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test_sorted_tf.parquet_temp",mode='overwrite',compression='snappy')