groceryheist/cdsc_reddit
1
0
Fork 0
Code

correct paths.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-04 09:06:02 -08:00
parent 1cca01fb69
commit 85945eae90
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
ngrams/top_comment_phrases.py
View File

@ -5,8 +5,8 @@ from pyspark.sql import SparkSession
import numpy as np import numpy as np
spark = SparkSession.builder.getOrCreate() spark = SparkSession.builder.getOrCreate()
df = spark.read.text("/gscratch/comdata/users/nathante/reddit_comment_ngrams_10p_sample/") df = spark.read.text("/gscratch/comdata/output/reddit_ngrams/reddit_comment_ngrams_10p_sample/")
df2 = spark.read.text("/gscratch/comdata/users/nathante/reddit_post_ngrams_10p_sample/") df2 = spark.read.text("/gscratch/comdata/output/reddit_ngrams/reddit_post_ngrams_10p_sample/")
df = df.union(df2) df = df.union(df2)
df = df.withColumnRenamed("value","phrase") df = df.withColumnRenamed("value","phrase")