collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

reduce memory a tich more.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2025-08-01 20:10:38 -07:00
parent 9799919470
commit 2f853a879d
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/wikiq/__init__.py
View File

@ -493,9 +493,9 @@ class WikiqParser:
num_context_lines=1000000, num_context_lines=1000000,
max_word_level_diff_complexity=-1, max_word_level_diff_complexity=-1,
moved_paragraph_detection_cutoff=-1, moved_paragraph_detection_cutoff=-1,
words_cache_capacity=1000, words_cache_capacity=500,
diff_cache_capacity=1000, diff_cache_capacity=500,
stats_cache_capacity=1000, stats_cache_capacity=500,
) )
while not on_last_batch: while not on_last_batch:
@ -875,7 +875,7 @@ def main():
parser.add_argument( parser.add_argument(
"--batch-size", "--batch-size",
dest="batch_size", dest="batch_size",
default=2000, default=1500,
type=int, type=int,
help="How many revisions to process in each batch. This ends up being the Parquet row group size", help="How many revisions to process in each batch. This ends up being the Parquet row group size",
) )