From f38ec6c1291754f924d562bb2c843eb6dd8da5bd Mon Sep 17 00:00:00 2001 From: Nathan TeBlunthuis Date: Sat, 7 Dec 2024 13:23:44 -0800 Subject: [PATCH] smaller outchunk size. --- ngrams/term_frequencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ngrams/term_frequencies.py b/ngrams/term_frequencies.py index c74d47e..0e77682 100755 --- a/ngrams/term_frequencies.py +++ b/ngrams/term_frequencies.py @@ -193,7 +193,7 @@ def weekly_tf(partition, outrows = tf_func(subreddit_weeks, mwe_pass, mwe_tokenize, stopWords, Path(output_10p_sample_path) / ngram_output) - outchunksize = 100000 + outchunksize = 10000 Path(output_terms_path).mkdir(parents=True, exist_ok=True) Path(output_authors_path).mkdir(parents=True, exist_ok=True)