groceryheist/cdsc_reddit
1
0
Fork 0
Code

it's selftext not body

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-03 18:59:27 -08:00
parent 5045d6052e
commit 73dd2a96a6
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ngrams/term_frequencies.py
View File

@ -84,7 +84,7 @@ def tf_comments(subreddit_weeks, mwe_pass, mwe_tokenize, stopWords):
tfs = Counter([]) tfs = Counter([])
authors = Counter([]) authors = Counter([])
for post in posts: for post in posts:
tokens = my_tokenizer(post.body, mwe_pass, mwe_tokenize, stopWords) tokens = my_tokenizer(post.selftext, mwe_pass, mwe_tokenize, stopWords)
tfs.update(tokens) tfs.update(tokens)
authors.update([post.author]) authors.update([post.author])
@ -102,9 +102,9 @@ def tf_posts(subreddit_weeks, mwe_pass, mwe_tokenize, stopWords):
for post in posts: for post in posts:
title_tokens = my_tokenizer(post.title, mwe_pass, mwe_tokenize, stopWords) title_tokens = my_tokenizer(post.title, mwe_pass, mwe_tokenize, stopWords)
tfs.update(title_tokens) tfs.update(title_tokens)
if post.body is not None and post.body != "": if post.selftext is not None and post.selftext != "":
body_tokens = my_tokenizer(post.body, mwe_pass, mwe_tokenize, stopWords) selftext_tokens = my_tokenizer(post.selftext, mwe_pass, mwe_tokenize, stopWords)
tfs.update(body_tokens) tfs.update(selftext_tokens)
authors.update([post.author]) authors.update([post.author])
for term, tf in tfs.items(): for term, tf in tfs.items():
@ -147,7 +147,7 @@ def weekly_tf(partition,
if reddit_dataset == 'comments': if reddit_dataset == 'comments':
batches = dataset.to_batches(columns=['CreatedAt','subreddit','body','author']) batches = dataset.to_batches(columns=['CreatedAt','subreddit','body','author'])
if reddit_dataset == 'posts': if reddit_dataset == 'posts':
batches = dataset.to_batches(columns=['CreatedAt','subreddit','title','body','author']) batches = dataset.to_batches(columns=['CreatedAt','subreddit','title','selftext','author'])
schema = pa.schema([pa.field('subreddit', pa.string(), nullable=nullable_schema), schema = pa.schema([pa.field('subreddit', pa.string(), nullable=nullable_schema),
pa.field('term', pa.string(), nullable=nullable_schema), pa.field('term', pa.string(), nullable=nullable_schema),