collective/mediawiki_dump_tools
18
0
Fork 0
Code Issues Pull Requests Activity

make wikiq memory efficient again via batch processing.

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2025-07-15 15:20:17 -07:00
parent 76d54ae597
commit c40506137b
6 changed files with 1809 additions and 1605 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
test/test_wiki_diff_matcher.py
View File

@@ -59,7 +59,7 @@ def assert_correct_equal_section(ops, expected_equal_lines, expected_equal_token
def test_equality():
rev1 = open("test/test_diff_revisions/1285792388").read()
# whitespace is added because exact identity reverts do not result in diffs.
matcher = WikiDiffMatcher([rev1,rev1 + " "])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
ops, a, b = diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev1 + " ")
@@ -75,7 +75,7 @@ def test_equality():
def test_highlight_range_3():
rev1 = open("test/test_diff_revisions/test_highlight_3_from").read()
rev2 = open("test/test_diff_revisions/test_highlight_3_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -85,7 +85,7 @@ def test_highlight_range_3():
def test_highlight_range_4():
rev1 = open("test/test_diff_revisions/test_highlight_4_from").read()
rev2 = open("test/test_diff_revisions/test_highlight_4_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -95,7 +95,7 @@ def test_highlight_range_4():
def test_complex_diff():
rev1 = open("test/test_diff_revisions/test_complex_from").read()
rev2 = open("test/test_diff_revisions/test_complex_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -107,7 +107,7 @@ def test_complex_diff():
def test_highlight_range_unicode():
rev1 = open("test/test_diff_revisions/test_unicode_highlight_from").read()
rev2 = open("test/test_diff_revisions/test_unicode_highlight_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -118,7 +118,7 @@ def test_highlight_range_unicode():
def test_highlight_range():
rev1 = open("test/test_diff_revisions/1295229484_rangeedit0").read()
rev2 = open("test/test_diff_revisions/1295229484_rangeedit1").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -128,7 +128,7 @@ def test_highlight_range():
def test_unmatched_parmoves():
rev1 = open("test/test_diff_revisions/test_unmatched_parmoves_from").read()
rev2 = open("test/test_diff_revisions/test_unmatched_parmoves_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -138,7 +138,7 @@ def test_unmatched_parmoves():
def test_bug_4():
rev1 = open("test/test_diff_revisions/test_bug_4_from").read()
rev2 = open("test/test_diff_revisions/test_bug_4_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -151,7 +151,7 @@ def test_delete():
rev2 = open("test/test_diff_revisions/1295229484_delete").read()
# whitespace is added because exact identity reverts do not result in diffs.
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev2)
@@ -207,7 +207,7 @@ def test_delete():
def test_addition():
rev1 = open("test/test_diff_revisions/1285792388").read()
rev2 = open("test/test_diff_revisions/1295229484").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -255,7 +255,7 @@ def test_addition():
def test_paragraph_move():
rev1 = open("test/test_diff_revisions/1295229484").read()
rev2 = open("test/test_diff_revisions/1295229484_parmove").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -268,7 +268,7 @@ def test_paragraph_move():
def test_paragraph_move_and_change():
rev1 = open("test/test_diff_revisions/1295229484").read()
rev2 = open("test/test_diff_revisions/1295229484_parmove_and_change").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -281,7 +281,7 @@ def test_paragraph_move_and_change():
def test_infobox():
rev1 = open("test/test_diff_revisions/test_infobox_from").read()
rev2 = open("test/test_diff_revisions/test_infobox_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -294,7 +294,7 @@ def test_infobox():
def test_leading_whitespace():
rev1 = open("test/test_diff_revisions/test_leading_ws_from").read()
rev2 = open("test/test_diff_revisions/test_leading_ws_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -307,7 +307,7 @@ def test_leading_whitespace():
def test_whitespace_bug():
rev1 = open("test/test_diff_revisions/test_whitespace_bug_from").read()
rev2 = open("test/test_diff_revisions/test_whitespace_bug_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -320,7 +320,7 @@ def test_whitespace_bug():
def test_bug_3():
rev1 = open("test/test_diff_revisions/test_bug_3_from").read()
rev2 = open("test/test_diff_revisions/test_bug_3_to").read()
matcher = WikiDiffMatcher([rev1,rev2])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
# note that a and b are constructed from the diffs.
@@ -335,7 +335,7 @@ def test_bug_3():
def test_actually_equal():
rev1 = open("test/test_diff_revisions/1285792388").read()
# whitespace is added because exact identity reverts do not result in diffs.
matcher = WikiDiffMatcher([rev1,rev1])
matcher = WikiDiffMatcher()
diff_processor = matcher.processor()
ops, a, b = diff_processor.process(rev1)
ops, a, b = diff_processor.process(rev1)