collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

fix a couple possible bugs.

Browse Source
This commit is contained in:
Nathan TeBlunthuis 2025-08-05 23:20:04 -07:00
parent bd8c30d80f
commit 9b3237014d
1 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/wikiq/__init__.py
View File

@ -52,7 +52,7 @@ async def diff_async(differ, last_text, text):
return differ.inline_json_diff(last_text, text) return differ.inline_json_diff(last_text, text)
try: try:
result = await asyncio.wait_for(_diff(), DIFF_TIMEOUT) result = await asyncio.wait_for(_diff(), DIFF_TIMEOUT)
except asyncio.TimeoutError as e: except TimeoutError as e:
raise e raise e
return result return result
@ -676,11 +676,11 @@ class WikiqParser:
if self.diff: if self.diff:
last_text = last_rev_text last_text = last_rev_text
new_diffs = [] new_diffs = []
for text in row_buffer["text"]: for i, text in enumerate(row_buffer["text"]):
try: try:
diff = asyncio.run(diff_async(differ, last_text, text)) diff = asyncio.run(diff_async(differ, last_text, text))
except asyncio.TimeoutError: except TimeoutError:
print(f"WARNING! wikidiff2 timeout for rev: {row_buffer['revid']}. Falling back to default limits.", file=sys.stderr) print(f"WARNING! wikidiff2 timeout for rev: {row_buffer['revid'][i]}. Falling back to default limits.", file=sys.stderr)
diff = fast_differ.inline_json_diff(last_text, text) diff = fast_differ.inline_json_diff(last_text, text)
new_diffs.append(diff) new_diffs.append(diff)
last_text = text last_text = text
@ -711,7 +711,11 @@ class WikiqParser:
"Done: %s revisions and %s pages." % (rev_count, page_count), "Done: %s revisions and %s pages." % (rev_count, page_count),
file=sys.stderr, file=sys.stderr,
) )
writer.close() if self.partition_namespaces is True:
for writer in pq_writers.values():
writer.close()
else:
writer.close()
def match_archive_suffix(input_filename): def match_archive_suffix(input_filename):