collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

force garbage collection.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2025-07-22 13:13:18 -07:00
parent 6557e25af7
commit 076df15740
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/wikiq/__init__.py
View File

@ -3,6 +3,7 @@
# original wikiq headers are: title articleid revid date_time anon # original wikiq headers are: title articleid revid date_time anon
# editor editor_id minor text_size text_entropy text_md5 reversion # editor editor_id minor text_size text_entropy text_md5 reversion
# additions_size deletions_size # additions_size deletions_size
import gc
import argparse import argparse
import json import json
import os.path import os.path
@ -675,7 +676,7 @@ class WikiqParser:
if self.partition_namespaces is True: if self.partition_namespaces is True:
writer = pq_writers[page.mwpage.namespace] writer = pq_writers[page.mwpage.namespace]
writer.write(pa.record_batch(row_buffer, schema=schema)) writer.write(pa.record_batch(row_buffer, schema=schema))
gc.collect()
page_count += 1 page_count += 1
print( print(