groceryheist/cdsc_reddit
1
0
Fork 0
Code

use Path to make directories not os.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-04 07:47:47 -08:00
parent 39c0fa7a29
commit 1cca01fb69
1 changed files with 2 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
ngrams/term_frequencies.py
View File

@ -133,11 +133,8 @@ def weekly_tf(partition,
nullable_schema = True nullable_schema = True
dataset = ds.dataset(f"{input_parquet}/{partition}", format='parquet') dataset = ds.dataset(f"{input_parquet}/{partition}", format='parquet')
if not os.path.exists(output_10p_sample_path): Path(output_10p_sample_path).mkdir(parents=True, exist_ok=True)
os.mkdir(output_10p_sample_path) Path(temp_output_tfidf_path).mkdir(parents=True, exist_ok=True)
if not os.path.exists(temp_output_tfidf_path):
os.mkdir(temp_output_tfidf_path)
ngram_output = partition.replace("parquet","txt") ngram_output = partition.replace("parquet","txt")