groceryheist/cdsc_reddit
1
0
Fork 0
Code

make the output directory.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-11-27 19:06:24 -08:00
parent 8cb9683bc2
commit 0631256956
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ngrams/term_frequencies.py
View File

@ -187,6 +187,8 @@ def weekly_tf(partition,
outchunksize = 10000 outchunksize = 10000
Path(output_terms_path).mkdir(parents=True, exist_ok=True)
with pq.ParquetWriter(f"{output_terms_path}/{partition}",schema=schema,compression='snappy',flavor='spark') as writer, pq.ParquetWriter(f"{output_authors_path}/{partition}",schema=author_schema,compression='snappy',flavor='spark') as author_writer: with pq.ParquetWriter(f"{output_terms_path}/{partition}",schema=schema,compression='snappy',flavor='spark') as writer, pq.ParquetWriter(f"{output_authors_path}/{partition}",schema=author_schema,compression='snappy',flavor='spark') as author_writer:
while True: while True: