groceryheist/cdsc_reddit
1
0
Fork 0
Code

smaller outchunk size.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2024-12-07 13:23:44 -08:00
parent 25bfc57baf
commit f38ec6c129
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ngrams/term_frequencies.py
View File

@ -193,7 +193,7 @@ def weekly_tf(partition,
outrows = tf_func(subreddit_weeks, mwe_pass, mwe_tokenize, stopWords, Path(output_10p_sample_path) / ngram_output) outrows = tf_func(subreddit_weeks, mwe_pass, mwe_tokenize, stopWords, Path(output_10p_sample_path) / ngram_output)
outchunksize = 100000 outchunksize = 10000
Path(output_terms_path).mkdir(parents=True, exist_ok=True) Path(output_terms_path).mkdir(parents=True, exist_ok=True)
Path(output_authors_path).mkdir(parents=True, exist_ok=True) Path(output_authors_path).mkdir(parents=True, exist_ok=True)