From 7528dc8b8eefa329948554290e34eeca49c783fd Mon Sep 17 00:00:00 2001 From: Nathan TeBlunthuis Date: Fri, 1 Aug 2025 19:52:18 -0700 Subject: [PATCH] try reducing memory more. --- src/wikiq/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index e9dcc33..4910c4d 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -493,9 +493,9 @@ class WikiqParser: num_context_lines=1000000, max_word_level_diff_complexity=-1, moved_paragraph_detection_cutoff=-1, - words_cache_capacity=2000, - diff_cache_capacity=2000, - stats_cache_capacity=2000, + words_cache_capacity=1000, + diff_cache_capacity=1000, + stats_cache_capacity=1000, ) while not on_last_batch: @@ -875,7 +875,7 @@ def main(): parser.add_argument( "--batch-size", dest="batch_size", - default=8000, + default=4000, type=int, help="How many revisions to process in each batch. This ends up being the Parquet row group size", )