force garbage collection.
This commit is contained in:
parent
6557e25af7
commit
076df15740
@ -3,6 +3,7 @@
|
||||
# original wikiq headers are: title articleid revid date_time anon
|
||||
# editor editor_id minor text_size text_entropy text_md5 reversion
|
||||
# additions_size deletions_size
|
||||
import gc
|
||||
import argparse
|
||||
import json
|
||||
import os.path
|
||||
@ -675,7 +676,7 @@ class WikiqParser:
|
||||
if self.partition_namespaces is True:
|
||||
writer = pq_writers[page.mwpage.namespace]
|
||||
writer.write(pa.record_batch(row_buffer, schema=schema))
|
||||
|
||||
gc.collect()
|
||||
page_count += 1
|
||||
|
||||
print(
|
||||
|
Loading…
Reference in New Issue
Block a user