document regex match counting in the README

Describes -RPc and -CPc alongside the regex matching options they
modify. Kept with the feature rather than on the release branch, so the
README there documents only what that branch actually provides.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -85,17 +85,11 @@ The most commonly useful options (`wikiq --help` describes them all):
 - `--external-links`, `--citations`, `--wikilinks`, `--templates`, and
   `--headings` parse each revision's wikitext and add a column with the
   extracted elements.
-- `--content-sizes` splits each revision into `content_chars`, the
-  information it carries, and `markup_chars`, the wikitext around it. A
-  contributor who does not know wikitext writes `Wonderful Hotel,
-  +15555550123`; someone polishing it later writes
-  `{{listing|name=Wonderful Hotel|phone=+15555550123}}`. Both carry the
-  same information, and the second adds two dozen characters of markup.
-  The two columns sum to the revision's length.
 - `-RP REGEX -RPl LABEL` searches revision text for a regular expression
   and reports the matches in a column named by the label; repeat the
   pair for multiple patterns. `-CP`/`-CPl` do the same for edit
-  summaries.
+  summaries. Adding `-RPc` or `-CPc` reports the number of matches
+  instead of the matched text.
 - `--resume` continues an interrupted run from the last complete line of
   an existing JSONL output file. Combined with `--time-limit HOURS`,
   this supports processing very large dumps in bounded chunks.

src/wikiq/__init__.py

@@ -174,7 +174,6 @@ def build_table(
     wikilinks: bool = False,
     templates: bool = False,
     headings: bool = False,
-    content_sizes: bool = False,
     redirect_aliases: Union[list[str], None] = None,
 ):
     """Build the RevisionTable with appropriate columns based on flags.
@@ -214,7 +213,7 @@ def build_table(
         table.columns.append(tables.RevisionCollapsed())
 
     wikitext_parser = None
-    if external_links or citations or wikilinks or templates or headings or content_sizes:
+    if external_links or citations or wikilinks or templates or headings:
         wikitext_parser = WikitextParser()
 
         if external_links:
@@ -232,10 +231,6 @@ def build_table(
         if headings:
             table.columns.append(tables.RevisionHeadings(wikitext_parser))
 
-        if content_sizes:
-            table.columns.append(tables.RevisionContentChars(wikitext_parser))
-            table.columns.append(tables.RevisionMarkupChars(wikitext_parser))
-
         table.columns.append(tables.RevisionParserTimeout(wikitext_parser))
 
     return table, reverts_column, wikitext_parser
@@ -279,10 +274,10 @@ def build_schema(
     return schema
 
 
-def make_regex_pairs(patterns, labels) -> list:
+def make_regex_pairs(patterns, labels, count_only=False) -> list:
     """Create RegexPair objects from patterns and labels."""
     if (patterns is not None and labels is not None) and (len(patterns) == len(labels)):
-        return [RegexPair(pattern, label) for pattern, label in zip(patterns, labels)]
+        return [RegexPair(pattern, label, count_only) for pattern, label in zip(patterns, labels)]
     elif patterns is None and labels is None:
         return []
     else:
@@ -507,21 +502,23 @@ If the pattern does not include a capture group, then only one output column wil
 
 
 class RegexPair(object):
-    def __init__(self, pattern, label):
+    def __init__(self, pattern, label, count_only=False):
         self.pattern = re.compile(pattern)
         self.label = label
+        self.count_only = count_only
         self.has_groups = bool(self.pattern.groupindex)
         if self.has_groups:
             self.capture_groups = list(self.pattern.groupindex.keys())
 
     def get_pyarrow_fields(self):
+        value_type = pa.int64() if self.count_only else pa.string()
         if self.has_groups:
             fields = [
-                pa.field(self._make_key(cap_group), pa.string())
+                pa.field(self._make_key(cap_group), value_type)
                 for cap_group in self.capture_groups
             ]
         else:
-            fields = [pa.field(self.label, pa.string())]
+            fields = [pa.field(self.label, value_type)]
 
         return fields
 
@@ -534,39 +531,50 @@ class RegexPair(object):
         if self.has_groups:
             # if there are matches of some sort in this revision content, fill the lists for each cap_group
             # content can be None when the text or comment was deleted/suppressed
-            if content is not None and self.pattern.search(content) is not None:
-                m = self.pattern.finditer(content)
-                matchobjects = list(m)
-
+            if content is not None and (matchobjects := list(self.pattern.finditer(content))):
                 for cap_group in self.capture_groups:
                     key = self._make_key(cap_group)
                     temp_list = []
                     for match in matchobjects:
                         # we only want to add the match for the capture group if the match is not None
-                        if match.group(cap_group) is not None:
-                            temp_list.append(match.group(cap_group))
+                        if (group := match.group(cap_group)) is not None:
+                            temp_list.append(group)
 
-                    # if temp_list of matches is empty just make that column None
+                    # if temp_list of matches is empty just make that column None (0 in count mode)
                     if len(temp_list) == 0:
-                        temp_dict[key] = None
+                        if self.count_only:
+                            temp_dict[key] = 0
+                        else:
+                            temp_dict[key] = None
                     # else we put in the list we made in the for-loop above
                     else:
-                        temp_dict[key] = ", ".join(temp_list)
+                        if self.count_only:
+                            temp_dict[key] = len(temp_list)
+                        else:
+                            temp_dict[key] = ", ".join(temp_list)
 
-            # there are no matches at all in this revision content, we default values to None
+            # there are no matches at all in this revision content, we default values to None (0 in count mode)
             else:
                 for cap_group in self.capture_groups:
                     key = self._make_key(cap_group)
-                    temp_dict[key] = None
+                    if self.count_only:
+                        temp_dict[key] = 0
+                    else:
+                        temp_dict[key] = None
 
         # there are no capture groups, we just search for all the matches of the regex
         else:
             # given that there are matches to be made
-            if content is not None and self.pattern.search(content) is not None:
-                m = self.pattern.findall(content)
-                temp_dict[self.label] = ", ".join(m)
+            if content is not None and (matches := self.pattern.findall(content)):
+                if self.count_only:
+                    temp_dict[self.label] = len(matches)
+                else:
+                    temp_dict[self.label] = ", ".join(matches)
             else:
-                temp_dict[self.label] = None
+                if self.count_only:
+                    temp_dict[self.label] = 0
+                else:
+                    temp_dict[self.label] = None
 
         return temp_dict
 
@@ -580,6 +588,8 @@ class WikiqParser:
         regex_match_comment: list[str],
         regex_revision_label: list[str],
         regex_comment_label: list[str],
+        regex_revision_output_count: bool = False,
+        regex_comment_output_count: bool = False,
         text: bool = False,
         diff: bool = False,
         collapse_user: bool = False,
@@ -595,7 +605,6 @@ class WikiqParser:
         wikilinks: bool = False,
         templates: bool = False,
         headings: bool = False,
-        content_sizes: bool = False,
         redirect_aliases: Union[list[str], None] = None,
         time_limit_seconds: Union[float, None] = None,
         input_filename: Union[str, None] = None,
@@ -622,7 +631,6 @@ class WikiqParser:
         self.wikilinks = wikilinks
         self.templates = templates
         self.headings = headings
-        self.content_sizes = content_sizes
         self.redirect_aliases = redirect_aliases
         self.shutdown_requested = False
         self.time_limit_seconds = time_limit_seconds
@@ -633,10 +641,10 @@ class WikiqParser:
 
         self.regex_schemas = []
         self.regex_revision_pairs: list[RegexPair] = self.make_matchmake_pairs(
-            regex_match_revision, regex_revision_label
+            regex_match_revision, regex_revision_label, regex_revision_output_count
         )
         self.regex_comment_pairs: list[RegexPair] = self.make_matchmake_pairs(
-            regex_match_comment, regex_comment_label
+            regex_match_comment, regex_comment_label, regex_comment_output_count
         )
 
         # Initialize output
@@ -679,13 +687,13 @@ class WikiqParser:
         if timer is not None:
             timer.cancel()
 
-    def make_matchmake_pairs(self, patterns, labels) -> list[RegexPair]:
+    def make_matchmake_pairs(self, patterns, labels, count_only=False) -> list[RegexPair]:
         if (patterns is not None and labels is not None) and (
             len(patterns) == len(labels)
         ):
             result: list[RegexPair] = []
             for pattern, label in zip(patterns, labels):
-                rp = RegexPair(pattern, label)
+                rp = RegexPair(pattern, label, count_only)
                 result.append(rp)
                 self.regex_schemas = self.regex_schemas + rp.get_pyarrow_fields()
             return result
@@ -749,7 +757,6 @@ class WikiqParser:
             wikilinks=self.wikilinks,
             templates=self.templates,
             headings=self.headings,
-            content_sizes=self.content_sizes,
             redirect_aliases=self.redirect_aliases,
         )
 
@@ -1228,6 +1235,14 @@ def main():
         help="The label for the outputted column based on matching the regex in revision text.",
     )
 
+    parser.add_argument(
+        "-RPc",
+        "--revision-pattern-count",
+        dest="regex_revision_output_count",
+        action="store_true",
+        help="If present, this will cause the revision patterns to return counts of the number of matches instead of the text of the matches themselves. It will affect all revision patterns.",
+    )
+
     parser.add_argument(
         "-CP",
         "--comment-pattern",
@@ -1248,6 +1263,14 @@ def main():
         help="The label for the outputted column based on matching the regex in comments.",
     )
 
+    parser.add_argument(
+        "-CPc",
+        "--comment-pattern-count",
+        dest="regex_comment_output_count",
+        action="store_true",
+        help="If present, this will cause the comment patterns to return counts of the number of matches instead of the text of the matches themselves. It will affect all comment patterns.",
+    )
+
     parser.add_argument(
         "-d",
         "--diff",
@@ -1306,14 +1329,6 @@ def main():
         help="Extract section headings from each revision.",
     )
 
-    parser.add_argument(
-        "--content-sizes",
-        dest="content_sizes",
-        action="store_true",
-        default=False,
-        help="Split each revision into content_chars, the information it carries, and markup_chars, the wikitext formatting around it. Content is what someone who did not know wikitext would have typed: words, link labels, and template parameter values. The two columns sum to the revision length.",
-    )
-
     parser.add_argument(
         "--redirect-aliases",
         dest="redirect_aliases",
@@ -1394,8 +1409,8 @@ def main():
 
     # Handle --print-schema: build and output schema, then exit
     if args.print_schema:
-        regex_revision_pairs = make_regex_pairs(args.regex_match_revision, args.regex_revision_label)
-        regex_comment_pairs = make_regex_pairs(args.regex_match_comment, args.regex_comment_label)
+        regex_revision_pairs = make_regex_pairs(args.regex_match_revision, args.regex_revision_label, args.regex_revision_output_count)
+        regex_comment_pairs = make_regex_pairs(args.regex_match_comment, args.regex_comment_label, args.regex_comment_output_count)
 
         table, _, _ = build_table(
             text=args.text,
@@ -1405,7 +1420,6 @@ def main():
             wikilinks=args.wikilinks,
             templates=args.templates,
             headings=args.headings,
-            content_sizes=args.content_sizes,
             redirect_aliases=redirect_aliases,
         )
         schema = build_schema(
@@ -1474,8 +1488,10 @@ def main():
                 revert_radius=args.revert_radius,
                 regex_match_revision=args.regex_match_revision,
                 regex_revision_label=args.regex_revision_label,
+                regex_revision_output_count=args.regex_revision_output_count,
                 regex_match_comment=args.regex_match_comment,
                 regex_comment_label=args.regex_comment_label,
+                regex_comment_output_count=args.regex_comment_output_count,
                 text=args.text,
                 diff=args.diff,
                 output_jsonl=output_jsonl,
@@ -1487,7 +1503,6 @@ def main():
                 wikilinks=args.wikilinks,
                 templates=args.templates,
                 headings=args.headings,
-                content_sizes=args.content_sizes,
                 redirect_aliases=redirect_aliases,
                 time_limit_seconds=time_limit_seconds,
                 input_filename=filename,
@@ -1532,8 +1547,10 @@ def main():
             revert_radius=args.revert_radius,
             regex_match_revision=args.regex_match_revision,
             regex_revision_label=args.regex_revision_label,
+            regex_revision_output_count=args.regex_revision_output_count,
             regex_match_comment=args.regex_match_comment,
             regex_comment_label=args.regex_comment_label,
+            regex_comment_output_count=args.regex_comment_output_count,
             diff=args.diff,
             text=args.text,
             batch_size=args.batch_size,
@@ -1543,7 +1560,6 @@ def main():
             wikilinks=args.wikilinks,
             templates=args.templates,
             headings=args.headings,
-            content_sizes=args.content_sizes,
             redirect_aliases=redirect_aliases,
             time_limit_seconds=time_limit_seconds,
         )

src/wikiq/tables.py

@@ -384,50 +384,6 @@ class RevisionHeadings(RevisionField[Union[list[dict], None]]):
         return self.wikitext_parser.extract_headings(revision.text)
 
 
-class RevisionContentChars(RevisionField[Union[int, None]]):
-    """Count the characters of information a revision carries.
-
-    The words, link labels and template parameter values -- what someone
-    who did not know wikitext would have typed. See
-    WikitextParser.content_sizes().
-    """
-
-    field = pa.field("content_chars", pa.int64(), nullable=True)
-
-    def __init__(self, wikitext_parser: "WikitextParser"):
-        super().__init__()
-        self.wikitext_parser = wikitext_parser
-
-    def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]:
-        revision = revisions[-1]
-        if revision.deleted.text:
-            return None
-        sizes = self.wikitext_parser.content_sizes(revision.text)
-        return None if sizes is None else sizes[0]
-
-
-class RevisionMarkupChars(RevisionField[Union[int, None]]):
-    """Count the characters of markup wrapped around a revision's content.
-
-    Braces, pipes, parameter names, tag names and attributes, link targets
-    and comments. Together with content_chars this sums to the revision
-    length.
-    """
-
-    field = pa.field("markup_chars", pa.int64(), nullable=True)
-
-    def __init__(self, wikitext_parser: "WikitextParser"):
-        super().__init__()
-        self.wikitext_parser = wikitext_parser
-
-    def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]:
-        revision = revisions[-1]
-        if revision.deleted.text:
-            return None
-        sizes = self.wikitext_parser.content_sizes(revision.text)
-        return None if sizes is None else sizes[1]
-
-
 class RevisionParserTimeout(RevisionField[bool]):
     """Track whether the wikitext parser timed out for this revision."""
 

src/wikiq/wikitext_parser.py

@@ -20,17 +20,6 @@ from __future__ import annotations
 import signal
 
 import mwparserfromhell
-from mwparserfromhell.nodes import (
-    Argument,
-    Comment,
-    ExternalLink,
-    Heading,
-    HTMLEntity,
-    Tag,
-    Template,
-    Text,
-    Wikilink,
-)
 
 PARSER_TIMEOUT = 60  # seconds
 
@@ -49,8 +38,6 @@ class WikitextParser:
         self._cached_text: str | None = None
         self._cached_wikicode = None
         self.last_parse_timed_out: bool = False
-        self._sizes_text: str | None = None
-        self._cached_sizes: tuple[int, int] | None = None
 
     def _timeout_handler(self, signum, frame):
         raise TimeoutError("mwparserfromhell parse exceeded timeout")
@@ -159,85 +146,6 @@ class WikitextParser:
         except Exception:
             return None
 
-    def _count_content(self, wikicode, counts: list[int]) -> None:
-        """Walk a wikicode tree adding the length of rendered content.
-
-        counts is a one-element list holding the running content total.
-        Anything not counted here is markup; content_sizes() derives the
-        markup total by subtraction, so every character lands in one column
-        or the other.
-        """
-        for node in wikicode.nodes:
-            if isinstance(node, Text):
-                counts[0] += len(node.value)
-            elif isinstance(node, HTMLEntity):
-                # an entity stands in for the character it renders as, so
-                # only that character is content and the escaping is markup
-                counts[0] += len(node.normalize())
-            elif isinstance(node, Comment):
-                # never reaches a reader, so it is not information
-                continue
-            elif isinstance(node, Template):
-                # the parameter values are the information; the template
-                # name, parameter names and delimiters are the polish that
-                # someone later wrapped around it
-                for param in node.params:
-                    self._count_content(param.value, counts)
-            elif isinstance(node, Tag):
-                # tag contents are information; the tag name and attributes
-                # are markup. This covers ''italic'' and '''bold''', which
-                # mwparserfromhell represents as i and b tags
-                if node.contents is not None:
-                    self._count_content(node.contents, counts)
-            elif isinstance(node, Wikilink):
-                # the label if present, otherwise the target, is what a
-                # reader sees; the brackets and the link target are markup
-                rendered = node.text if node.text is not None else node.title
-                self._count_content(rendered, counts)
-            elif isinstance(node, ExternalLink):
-                # both halves are information someone typed: a contributor
-                # who did not know wikitext would still have written the URL
-                counts[0] += len(str(node.url))
-                if node.title is not None:
-                    self._count_content(node.title, counts)
-            elif isinstance(node, Heading):
-                self._count_content(node.title, counts)
-            elif isinstance(node, Argument):
-                if node.default is not None:
-                    self._count_content(node.default, counts)
-
-    def content_sizes(self, text: str | None) -> tuple[int, int] | None:
-        """Split a revision into the information it carries and the markup
-        around it.
-
-        Returns (content_chars, markup_chars), which sum to len(text).
-
-        Content is what a contributor who did not know wikitext would have
-        typed: the words, the link labels, the values inside template
-        parameters. Markup is everything a later editor added to format it —
-        braces, pipes, parameter names, tag names and attributes, link
-        targets, quote marks, and comments.
-
-        So "Wonderful Hotel, +15555550123" is almost all content, while
-        "{{listing|name=Wonderful Hotel|phone=+15555550123}}" carries the
-        same information with two dozen characters of markup around it.
-        """
-        if text is None:
-            return None
-        if text == self._sizes_text:
-            return self._cached_sizes
-        try:
-            wikicode = self._get_wikicode(text)
-            counts = [0]
-            self._count_content(wikicode, counts)
-            content = min(counts[0], len(text))
-            result = (content, len(text) - content)
-        except Exception:
-            result = None
-        self._sizes_text = text
-        self._cached_sizes = result
-        return result
-
     def extract_headings(self, text: str | None) -> list[dict] | None:
         """Extract all section headings with their levels."""
         if text is None:

test/test_wikiq.py

@@ -12,7 +12,7 @@ import pytest
 from pandas import DataFrame
 from pandas.testing import assert_frame_equal, assert_series_equal
 
-from wikiq import build_table, build_schema, RegexPair, WikitextParser
+from wikiq import build_table, build_schema, RegexPair
 from wikiq_test_utils import (
     BASELINE_DIR,
     IKWIKI,
@@ -427,6 +427,55 @@ def test_regex_deleted_revisions():
     assert deleted["npov_npov"].isna().all()
     assert deleted["talk_talk"].isna().all()
 
+def test_regex_count_only():
+    # count mode returns match counts instead of the matched text, with 0
+    # (never None) for revisions with no matches or deleted content
+    pair = RegexPair(r"\bcat\b", "cats", count_only=True)
+    assert pair.matchmake("cat cat dog cat") == {"cats": 3}
+    assert pair.matchmake("dog") == {"cats": 0}
+    assert pair.matchmake(None) == {"cats": 0}
+    assert pair.get_pyarrow_fields()[0].type == pa.int64()
+
+    pair = RegexPair(r"(?P<a>\bcat\b)|(?P<b>\bdog\b)", "pets", count_only=True)
+    assert pair.matchmake("cat dog cat") == {"pets_a": 2, "pets_b": 1}
+    assert pair.matchmake("bird") == {"pets_a": 0, "pets_b": 0}
+    assert pair.matchmake(None) == {"pets_a": 0, "pets_b": 0}
+    assert all(f.type == pa.int64() for f in pair.get_pyarrow_fields())
+
+    # without count_only the original behavior is unchanged
+    pair = RegexPair(r"\bcat\b", "cats")
+    assert pair.matchmake("cat cat dog cat") == {"cats": "cat, cat, cat"}
+    assert pair.get_pyarrow_fields()[0].type == pa.string()
+
+def test_regex_count_e2e():
+    # -RPc/-CPc produce integer columns whose values equal the number of
+    # matches in the revision text and comment respectively
+    tester = WikiqTester(wiki=REGEXTEST, case_name="regex_count")
+
+    try:
+        tester.call_wikiq(
+            "--text",
+            "-RP 'TestCase' -RPl testcases -RPc",
+            "-CP '\\b[Cc]hevalier\\b' -CPl chev -CPc",
+        )
+    except subprocess.CalledProcessError as exc:
+        pytest.fail(exc.stderr.decode("utf8"))
+
+    import re
+    test = pd.read_table(tester.output)
+
+    assert test["testcases"].dtype == "int64"
+    assert test["chev"].dtype == "int64"
+
+    for _, row in test.iterrows():
+        text = row["text"] if isinstance(row["text"], str) else None
+        expected = len(re.findall("TestCase", text)) if text is not None else 0
+        assert row["testcases"] == expected, f"revid {row['revid']}: {row['testcases']} != {expected}"
+
+        comment = row["edit_summary"] if isinstance(row["edit_summary"], str) else None
+        expected = len(re.findall(r"\b[Cc]hevalier\b", comment)) if comment is not None else 0
+        assert row["chev"] == expected, f"revid {row['revid']}: {row['chev']} != {expected}"
+
 def test_redirect_detection():
     from wikiq.tables import RedirectDetector
 
@@ -484,71 +533,6 @@ def test_redirect_columns_e2e():
     assert len(redirects) > 0
     assert redirects["revision_redirect_target"].notna().all()
 
-def test_content_sizes():
-    # hand-computed fixtures: (text, content_chars, markup_chars)
-    cases = [
-        # plain text is all content
-        ("Hello world.", 12, 0),
-        # the same information written by someone who knows wikitext: the
-        # values survive as content, the scaffolding around them is markup
-        ("Wonderful Hotel, +15555550123", 29, 0),
-        ("{{listing|name=Wonderful Hotel|phone=+15555550123}}", 27, 24),
-        # emphasis is markup applied to content, not content itself
-        ("''whatever''", 8, 4),
-        ("Some '''bold''' prose.", 16, 6),
-        # tag contents are content; the tag name and attributes are not
-        ('Before <span class="x">inner text</span> after', 23, 23),
-        # a wikilink renders its label, so the target is markup
-        ("[[Main Page|the main page]]", 13, 14),
-        # an unlabeled wikilink renders its target, which is then content
-        ("[[Help]]", 4, 4),
-        # both halves of an external link are information someone typed
-        ("[http://x.com label]", 17, 3),
-        # comments never reach a reader, so they are entirely markup
-        ("Text <!-- hidden --> more", 10, 15),
-        # nesting: values at any depth are content
-        ("{{outer|a={{inner|b=deep}} shallow}}", 12, 24),
-    ]
-
-    parser = WikitextParser()
-    for text, content, markup in cases:
-        result = parser.content_sizes(text)
-        assert result == (content, markup), f"{text!r}: got {result}, want {(content, markup)}"
-        # every character lands in exactly one column
-        assert result[0] + result[1] == len(text), f"{text!r}: columns do not sum to length"
-
-    assert parser.content_sizes(None) is None
-
-def test_content_sizes_e2e():
-    # the ikwiki dump contains revisions with deleted text, which must get
-    # null content sizes; for every other revision the two columns must
-    # account for the whole revision
-    tester = WikiqTester(IKWIKI, "content_sizes")
-
-    try:
-        tester.call_wikiq("--content-sizes", "--text")
-    except subprocess.CalledProcessError as exc:
-        pytest.fail(exc.stderr.decode("utf8"))
-
-    test = pd.read_table(tester.output)
-
-    assert "content_chars" in test.columns
-    assert "markup_chars" in test.columns
-
-    deleted = test[test["deleted"]]
-    assert len(deleted) > 0
-    assert deleted["content_chars"].isna().all()
-    assert deleted["markup_chars"].isna().all()
-
-    present = test[~test["deleted"] & test["text"].notna() & test["content_chars"].notna()]
-    assert len(present) > 0
-    total = present["content_chars"] + present["markup_chars"]
-    assert (total == present["text"].str.len()).all()
-    # a real wiki has both kinds of revision: some almost pure text, some
-    # carrying a lot of markup
-    assert (present["markup_chars"] == 0).any()
-    assert (present["markup_chars"] > 0).any()
-
 def test_external_links_only():
     """Test that --external-links extracts external links correctly."""
     import mwparserfromhell