collective/mediawiki_dump_tools
16
0
Fork 0
Code Issues Pull Requests Activity

Add ability to disable revert detection

Browse Source 
Also add test to ensure functionality works.

Signed-off-by: Will Beason <willbeason@gmail.com>
This commit is contained in:
Will Beason 2025-05-29 11:59:10 -05:00
parent 9009bb6fa4
commit c0e629a313
3 changed files with 27809 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
test/Wikiq_Unit_Test.py
View File

@ -160,6 +160,23 @@ class WikiqTestCase(unittest.TestCase):
baseline = pd.read_table(tester.baseline_file) baseline = pd.read_table(tester.baseline_file)
assert_frame_equal(test, baseline, check_like=True) assert_frame_equal(test, baseline, check_like=True)
def test_WP_no_revert_radius(self):
tester = WikiqTester(IKWIKI, "no_revert_radius")
try:
tester.call_wikiq("-rr 0")
except subprocess.CalledProcessError as exc:
self.fail(exc.stderr.decode("utf8"))
copyfile(tester.call_output, tester.test_file)
# as a test let's make sure that we get equal data frames
test = pd.read_table(tester.test_file)
num_reverted = sum(i is None for i in test.revert)
self.assertEqual(num_reverted, 0)
baseline = pd.read_table(tester.baseline_file)
assert_frame_equal(test, baseline, check_like=True)
def test_noargs(self): def test_noargs(self):
tester = WikiqTester(SAILORMOON, "noargs", in_compression="7z") tester = WikiqTester(SAILORMOON, "noargs", in_compression="7z")

27780
test/baseline_output/no_revert_radius_ikwiki-20180301-pages-meta-history.tsv Normal file
View File

File diff suppressed because it is too large Load Diff

5
wikiq
View File

@ -523,7 +523,11 @@ class WikiqParser:
if namespace not in self.namespace_filter: if namespace not in self.namespace_filter:
continue continue
# Disable detecting reverts if radius is 0.
if self.revert_radius > 0:
rev_detector = mwreverts.Detector(radius=self.revert_radius) rev_detector = mwreverts.Detector(radius=self.revert_radius)
else:
rev_detector = None
if self.persist != PersistMethod.none: if self.persist != PersistMethod.none:
window = deque(maxlen=PERSISTENCE_RADIUS) window = deque(maxlen=PERSISTENCE_RADIUS)
@ -574,6 +578,7 @@ class WikiqParser:
rev_data.text_chars = len(rev.text) rev_data.text_chars = len(rev.text)
# generate revert data # generate revert data
if rev_detector is not None:
revert = rev_detector.process(text_sha1, rev.id) revert = rev_detector.process(text_sha1, rev.id)
if revert: if revert: