diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index 25bb8d8..5047658 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -3,6 +3,7 @@ # original wikiq headers are: title articleid revid date_time anon # editor editor_id minor text_size text_entropy text_md5 reversion # additions_size deletions_size +import gc import argparse import json import os.path @@ -675,7 +676,7 @@ class WikiqParser: if self.partition_namespaces is True: writer = pq_writers[page.mwpage.namespace] writer.write(pa.record_batch(row_buffer, schema=schema)) - + gc.collect() page_count += 1 print(