groceryheist/cdsc_reddit
1
0
Fork 0
Code

allow posts schemas to be nullable.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-01 09:55:12 -08:00
parent 271cbea7d9
commit 88fca0f82b
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ngrams/term_frequencies.py
View File

@ -126,7 +126,7 @@ def weekly_tf(partition,
nullable_schema = False nullable_schema = False
elif reddit_dataset == 'posts': elif reddit_dataset == 'posts':
tf_func = tf_posts tf_func = tf_posts
nullable_schema = False nullable_schema = True
dataset = ds.dataset(f"{input_parquet}/{partition}", format='parquet') dataset = ds.dataset(f"{input_parquet}/{partition}", format='parquet')
if not os.path.exists(output_10p_sample_path): if not os.path.exists(output_10p_sample_path):