collective/cdsc_reddit
18
0
Fork 0
Code Activity

Refactor and reorganze.

Browse Source
This commit is contained in:
Nate E TeBlunthuis
2020-12-08 17:32:20 -08:00
parent a60747292e
commit e6294b5b90
47 changed files with 731 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
ngrams/sort_tf_comments.py Normal file
View File

@@ -0,0 +1,13 @@
#!/usr/bin/env python3
from pyspark.sql import functions as f
from pyspark.sql import SparkSession
spark = SparkSession.builder.getOrCreate()
df = spark.read.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test.parquet_temp/")
df = df.repartition(2000,'term')
df = df.sort(['term','week','subreddit'])
df = df.sortWithinPartitions(['term','week','subreddit'])
df.write.parquet("/gscratch/comdata/users/nathante/reddit_tfidf_test_sorted_tf.parquet_temp",mode='overwrite',compression='snappy')