collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

try reducing memory more.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2025-08-01 19:52:18 -07:00
parent 615d630ff0
commit 7528dc8b8e
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/wikiq/__init__.py
View File

@ -493,9 +493,9 @@ class WikiqParser:
num_context_lines=1000000,
max_word_level_diff_complexity=-1,
moved_paragraph_detection_cutoff=-1,
words_cache_capacity=2000,
diff_cache_capacity=2000,
stats_cache_capacity=2000,
words_cache_capacity=1000,
diff_cache_capacity=1000,
stats_cache_capacity=1000,
)
while not on_last_batch:
@ -875,7 +875,7 @@ def main():
parser.add_argument(
"--batch-size",
dest="batch_size",
default=8000,
default=4000,
type=int,
help="How many revisions to process in each batch. This ends up being the Parquet row group size",
)