collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

reduce memory usage.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2025-08-01 19:45:21 -07:00
parent 32bc05ddfd
commit 615d630ff0
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/wikiq/__init__.py
View File

@ -493,9 +493,9 @@ class WikiqParser:
num_context_lines=1000000, num_context_lines=1000000,
max_word_level_diff_complexity=-1, max_word_level_diff_complexity=-1,
moved_paragraph_detection_cutoff=-1, moved_paragraph_detection_cutoff=-1,
words_cache_capacity=5000, words_cache_capacity=2000,
diff_cache_capacity=5000, diff_cache_capacity=2000,
stats_cache_capacity=5000, stats_cache_capacity=2000,
) )
while not on_last_batch: while not on_last_batch:
@ -875,7 +875,7 @@ def main():
parser.add_argument( parser.add_argument(
"--batch-size", "--batch-size",
dest="batch_size", dest="batch_size",
default=16000, default=8000,
type=int, type=int,
help="How many revisions to process in each batch. This ends up being the Parquet row group size", help="How many revisions to process in each batch. This ends up being the Parquet row group size",
) )