diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index 9ff63ee..c3d01cb 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -463,7 +463,8 @@ class RegexPair(object): # if there are named capture groups in the regex if self.has_groups: # if there are matches of some sort in this revision content, fill the lists for each cap_group - if self.pattern.search(content) is not None: + # content can be None when the text or comment was deleted/suppressed + if content is not None and self.pattern.search(content) is not None: m = self.pattern.finditer(content) matchobjects = list(m) @@ -491,12 +492,11 @@ class RegexPair(object): # there are no capture groups, we just search for all the matches of the regex else: # given that there are matches to be made - if type(content) in (str, bytes): - if self.pattern.search(content) is not None: - m = self.pattern.findall(content) - temp_dict[self.label] = ", ".join(m) - else: - temp_dict[self.label] = None + if content is not None and self.pattern.search(content) is not None: + m = self.pattern.findall(content) + temp_dict[self.label] = ", ".join(m) + else: + temp_dict[self.label] = None return temp_dict diff --git a/test/Wikiq_Unit_Test.py b/test/Wikiq_Unit_Test.py index e56545e..4c22eb3 100644 --- a/test/Wikiq_Unit_Test.py +++ b/test/Wikiq_Unit_Test.py @@ -12,7 +12,7 @@ import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal, assert_series_equal -from wikiq import build_table, build_schema +from wikiq import build_table, build_schema, RegexPair from wikiq_test_utils import ( BASELINE_DIR, IKWIKI, @@ -391,6 +391,35 @@ def test_capturegroup_regex(): baseline = pd.read_table(tester.baseline_file) assert_frame_equal(test, baseline, check_like=True) +def test_regex_none_content(): + # deleted or suppressed revisions yield None for text and comments, and + # matchmake must tolerate that in both the capture-group and plain paths + pair = RegexPair(r"(?P\b[a-zA-Z]{3}\b)|(?P\b\d+\b)", "cap") + assert pair.matchmake(None) == {"cap_letter": None, "cap_number": None} + + pair = RegexPair(r"\b\d{3}\b", "digits") + assert pair.matchmake(None) == {"digits": None} + +def test_regex_deleted_revisions(): + # the ikwiki dump contains revisions with deleted text and deleted + # comments; regex matching must handle them rather than crashing + tester = WikiqTester(wiki=IKWIKI, case_name="regex_deleted") + + try: + tester.call_wikiq( + "-RP '(?Pnpov|NPOV)' -RPl npov", + "-CP '(?P[Tt]alk)' -CPl talk", + ) + except subprocess.CalledProcessError as exc: + pytest.fail(exc.stderr.decode("utf8")) + + test = pd.read_table(tester.output) + + deleted = test[test["deleted"]] + assert len(deleted) > 0 + assert deleted["npov_npov"].isna().all() + assert deleted["talk_talk"].isna().all() + def test_external_links_only(): """Test that --external-links extracts external links correctly.""" import mwparserfromhell