From c61eb10e471e261ca408d5bb0c17c3d62614cb52 Mon Sep 17 00:00:00 2001 From: Benjamin Mako Hill Date: Thu, 6 Aug 2026 14:20:02 -0700 Subject: [PATCH] fix regex matching on revisions with deleted text or comments RegexPair.matchmake crashed with a TypeError when a capture-group pattern was applied to a revision whose text or comment was deleted or suppressed (content is None). Guard both matching paths against None, and make the no-capture-group path emit a None column for such revisions instead of omitting the key entirely. This carries forward the fix Kaylea Champion and Mako Hill made on the mako_changes-20230429 branch (7e6cd5b), which predated the rewrite. Adds a unit test for matchmake(None) and an end-to-end test against the ikwiki dump, which contains revisions with deleted text and comments. Co-Authored-By: Claude Fable 5 (cherry picked from commit dbcae5c64e316f90d60cd879cd0ab62c7864b039) --- src/wikiq/__init__.py | 14 +++++++------- test/Wikiq_Unit_Test.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index 9ff63ee..c3d01cb 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -463,7 +463,8 @@ class RegexPair(object): # if there are named capture groups in the regex if self.has_groups: # if there are matches of some sort in this revision content, fill the lists for each cap_group - if self.pattern.search(content) is not None: + # content can be None when the text or comment was deleted/suppressed + if content is not None and self.pattern.search(content) is not None: m = self.pattern.finditer(content) matchobjects = list(m) @@ -491,12 +492,11 @@ class RegexPair(object): # there are no capture groups, we just search for all the matches of the regex else: # given that there are matches to be made - if type(content) in (str, bytes): - if self.pattern.search(content) is not None: - m = self.pattern.findall(content) - temp_dict[self.label] = ", ".join(m) - else: - temp_dict[self.label] = None + if content is not None and self.pattern.search(content) is not None: + m = self.pattern.findall(content) + temp_dict[self.label] = ", ".join(m) + else: + temp_dict[self.label] = None return temp_dict diff --git a/test/Wikiq_Unit_Test.py b/test/Wikiq_Unit_Test.py index e56545e..4c22eb3 100644 --- a/test/Wikiq_Unit_Test.py +++ b/test/Wikiq_Unit_Test.py @@ -12,7 +12,7 @@ import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal, assert_series_equal -from wikiq import build_table, build_schema +from wikiq import build_table, build_schema, RegexPair from wikiq_test_utils import ( BASELINE_DIR, IKWIKI, @@ -391,6 +391,35 @@ def test_capturegroup_regex(): baseline = pd.read_table(tester.baseline_file) assert_frame_equal(test, baseline, check_like=True) +def test_regex_none_content(): + # deleted or suppressed revisions yield None for text and comments, and + # matchmake must tolerate that in both the capture-group and plain paths + pair = RegexPair(r"(?P\b[a-zA-Z]{3}\b)|(?P\b\d+\b)", "cap") + assert pair.matchmake(None) == {"cap_letter": None, "cap_number": None} + + pair = RegexPair(r"\b\d{3}\b", "digits") + assert pair.matchmake(None) == {"digits": None} + +def test_regex_deleted_revisions(): + # the ikwiki dump contains revisions with deleted text and deleted + # comments; regex matching must handle them rather than crashing + tester = WikiqTester(wiki=IKWIKI, case_name="regex_deleted") + + try: + tester.call_wikiq( + "-RP '(?Pnpov|NPOV)' -RPl npov", + "-CP '(?P[Tt]alk)' -CPl talk", + ) + except subprocess.CalledProcessError as exc: + pytest.fail(exc.stderr.decode("utf8")) + + test = pd.read_table(tester.output) + + deleted = test[test["deleted"]] + assert len(deleted) > 0 + assert deleted["npov_npov"].isna().all() + assert deleted["talk_talk"].isna().all() + def test_external_links_only(): """Test that --external-links extracts external links correctly.""" import mwparserfromhell