collective/cdsc_reddit
13
0
Fork 0
Code

Cache before sorting so we don't extract twice.

Browse Source
This commit is contained in:
Nate E TeBlunthuis 2020-07-06 22:30:04 -07:00
parent 4ec9c14247
commit fd3b615544
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
comments_2_parquet.py
View File

@ -128,6 +128,10 @@ df = df.withColumn("Month",f.month(f.col("CreatedAt")))
df = df.withColumn("Year",f.year(f.col("CreatedAt"))) df = df.withColumn("Year",f.year(f.col("CreatedAt")))
df = df.withColumn("Day",f.dayofmonth(f.col("CreatedAt"))) df = df.withColumn("Day",f.dayofmonth(f.col("CreatedAt")))
df = df.withColumn("subreddit_hash",f.sha2(f.col("subreddit"), 256)[0:3]) df = df.withColumn("subreddit_hash",f.sha2(f.col("subreddit"), 256)[0:3])
# cache so we don't have to extract everythin twice
df = df.cache()
df2 = df.sort(["subreddit","author","link_id","parent_id","Year","Month","Day"],ascending=True) df2 = df.sort(["subreddit","author","link_id","parent_id","Year","Month","Day"],ascending=True)
df2.write.parquet("/gscratch/comdata/output/reddit_comments_by_subreddit.parquet", partitionBy=["Year",'Month'],mode='overwrite') df2.write.parquet("/gscratch/comdata/output/reddit_comments_by_subreddit.parquet", partitionBy=["Year",'Month'],mode='overwrite')