add --content-sizes: split revisions into content and markup

Wikitravel-style guides get written twice. Someone who does not know
wikitext contributes the information:

    Wonderful Hotel, +15555550123

and someone who does comes along later and polishes it:

    {{listing|name=Wonderful Hotel|phone=+15555550123}}

Both carry the same information. The second adds two dozen characters of
markup around it. --content-sizes measures that split, emitting two
nullable int64 columns from a single mwparserfromhell tree walk per
revision.

content_chars counts what a contributor who did not know wikitext would
have typed: text nodes, template parameter values, tag contents,
wikilink labels (or targets when unlabeled), external link URLs and
their labels, and heading titles. markup_chars is everything else --
braces, pipes, parameter names, tag names and attributes, link targets,
quote marks, and comments, which never reach a reader.

The two columns sum to the revision length, so their ratio reads
directly as how much of a revision is formatting. That invariant is what
the end-to-end test checks on a real dump, along with nulls for deleted
revisions, which come through the same machinery as the other
parser-based columns.

Emphasis needs no special handling: mwparserfromhell represents ''' and
'' as b and i tags, so the quote marks fall out as markup and the words
they wrap as content, which is the wanted answer.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -85,6 +85,13 @@ The most commonly useful options (`wikiq --help` describes them all):
 - `--external-links`, `--citations`, `--wikilinks`, `--templates`, and
   `--headings` parse each revision's wikitext and add a column with the
   extracted elements.
+- `--content-sizes` splits each revision into `content_chars`, the
+  information it carries, and `markup_chars`, the wikitext around it. A
+  contributor who does not know wikitext writes `Wonderful Hotel,
+  +15555550123`; someone polishing it later writes
+  `{{listing|name=Wonderful Hotel|phone=+15555550123}}`. Both carry the
+  same information, and the second adds two dozen characters of markup.
+  The two columns sum to the revision's length.
 - `-RP REGEX -RPl LABEL` searches revision text for a regular expression
   and reports the matches in a column named by the label; repeat the
   pair for multiple patterns. `-CP`/`-CPl` do the same for edit

src/wikiq/__init__.py

@@ -174,6 +174,7 @@ def build_table(
     wikilinks: bool = False,
     templates: bool = False,
     headings: bool = False,
+    content_sizes: bool = False,
     redirect_aliases: Union[list[str], None] = None,
 ):
     """Build the RevisionTable with appropriate columns based on flags.
@@ -213,7 +214,7 @@ def build_table(
         table.columns.append(tables.RevisionCollapsed())
 
     wikitext_parser = None
-    if external_links or citations or wikilinks or templates or headings:
+    if external_links or citations or wikilinks or templates or headings or content_sizes:
         wikitext_parser = WikitextParser()
 
         if external_links:
@@ -231,6 +232,10 @@ def build_table(
         if headings:
             table.columns.append(tables.RevisionHeadings(wikitext_parser))
 
+        if content_sizes:
+            table.columns.append(tables.RevisionContentChars(wikitext_parser))
+            table.columns.append(tables.RevisionMarkupChars(wikitext_parser))
+
         table.columns.append(tables.RevisionParserTimeout(wikitext_parser))
 
     return table, reverts_column, wikitext_parser
@@ -590,6 +595,7 @@ class WikiqParser:
         wikilinks: bool = False,
         templates: bool = False,
         headings: bool = False,
+        content_sizes: bool = False,
         redirect_aliases: Union[list[str], None] = None,
         time_limit_seconds: Union[float, None] = None,
         input_filename: Union[str, None] = None,
@@ -616,6 +622,7 @@ class WikiqParser:
         self.wikilinks = wikilinks
         self.templates = templates
         self.headings = headings
+        self.content_sizes = content_sizes
         self.redirect_aliases = redirect_aliases
         self.shutdown_requested = False
         self.time_limit_seconds = time_limit_seconds
@@ -742,6 +749,7 @@ class WikiqParser:
             wikilinks=self.wikilinks,
             templates=self.templates,
             headings=self.headings,
+            content_sizes=self.content_sizes,
             redirect_aliases=self.redirect_aliases,
         )
 
@@ -1298,6 +1306,14 @@ def main():
         help="Extract section headings from each revision.",
     )
 
+    parser.add_argument(
+        "--content-sizes",
+        dest="content_sizes",
+        action="store_true",
+        default=False,
+        help="Split each revision into content_chars, the information it carries, and markup_chars, the wikitext formatting around it. Content is what someone who did not know wikitext would have typed: words, link labels, and template parameter values. The two columns sum to the revision length.",
+    )
+
     parser.add_argument(
         "--redirect-aliases",
         dest="redirect_aliases",
@@ -1389,6 +1405,7 @@ def main():
             wikilinks=args.wikilinks,
             templates=args.templates,
             headings=args.headings,
+            content_sizes=args.content_sizes,
             redirect_aliases=redirect_aliases,
         )
         schema = build_schema(
@@ -1470,6 +1487,7 @@ def main():
                 wikilinks=args.wikilinks,
                 templates=args.templates,
                 headings=args.headings,
+                content_sizes=args.content_sizes,
                 redirect_aliases=redirect_aliases,
                 time_limit_seconds=time_limit_seconds,
                 input_filename=filename,
@@ -1525,6 +1543,7 @@ def main():
             wikilinks=args.wikilinks,
             templates=args.templates,
             headings=args.headings,
+            content_sizes=args.content_sizes,
             redirect_aliases=redirect_aliases,
             time_limit_seconds=time_limit_seconds,
         )

src/wikiq/tables.py

@@ -384,6 +384,50 @@ class RevisionHeadings(RevisionField[Union[list[dict], None]]):
         return self.wikitext_parser.extract_headings(revision.text)
 
 
+class RevisionContentChars(RevisionField[Union[int, None]]):
+    """Count the characters of information a revision carries.
+
+    The words, link labels and template parameter values -- what someone
+    who did not know wikitext would have typed. See
+    WikitextParser.content_sizes().
+    """
+
+    field = pa.field("content_chars", pa.int64(), nullable=True)
+
+    def __init__(self, wikitext_parser: "WikitextParser"):
+        super().__init__()
+        self.wikitext_parser = wikitext_parser
+
+    def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]:
+        revision = revisions[-1]
+        if revision.deleted.text:
+            return None
+        sizes = self.wikitext_parser.content_sizes(revision.text)
+        return None if sizes is None else sizes[0]
+
+
+class RevisionMarkupChars(RevisionField[Union[int, None]]):
+    """Count the characters of markup wrapped around a revision's content.
+
+    Braces, pipes, parameter names, tag names and attributes, link targets
+    and comments. Together with content_chars this sums to the revision
+    length.
+    """
+
+    field = pa.field("markup_chars", pa.int64(), nullable=True)
+
+    def __init__(self, wikitext_parser: "WikitextParser"):
+        super().__init__()
+        self.wikitext_parser = wikitext_parser
+
+    def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]:
+        revision = revisions[-1]
+        if revision.deleted.text:
+            return None
+        sizes = self.wikitext_parser.content_sizes(revision.text)
+        return None if sizes is None else sizes[1]
+
+
 class RevisionParserTimeout(RevisionField[bool]):
     """Track whether the wikitext parser timed out for this revision."""
 

src/wikiq/wikitext_parser.py

@@ -20,6 +20,17 @@ from __future__ import annotations
 import signal
 
 import mwparserfromhell
+from mwparserfromhell.nodes import (
+    Argument,
+    Comment,
+    ExternalLink,
+    Heading,
+    HTMLEntity,
+    Tag,
+    Template,
+    Text,
+    Wikilink,
+)
 
 PARSER_TIMEOUT = 60  # seconds
 
@@ -38,6 +49,8 @@ class WikitextParser:
         self._cached_text: str | None = None
         self._cached_wikicode = None
         self.last_parse_timed_out: bool = False
+        self._sizes_text: str | None = None
+        self._cached_sizes: tuple[int, int] | None = None
 
     def _timeout_handler(self, signum, frame):
         raise TimeoutError("mwparserfromhell parse exceeded timeout")
@@ -146,6 +159,85 @@ class WikitextParser:
         except Exception:
             return None
 
+    def _count_content(self, wikicode, counts: list[int]) -> None:
+        """Walk a wikicode tree adding the length of rendered content.
+
+        counts is a one-element list holding the running content total.
+        Anything not counted here is markup; content_sizes() derives the
+        markup total by subtraction, so every character lands in one column
+        or the other.
+        """
+        for node in wikicode.nodes:
+            if isinstance(node, Text):
+                counts[0] += len(node.value)
+            elif isinstance(node, HTMLEntity):
+                # an entity stands in for the character it renders as, so
+                # only that character is content and the escaping is markup
+                counts[0] += len(node.normalize())
+            elif isinstance(node, Comment):
+                # never reaches a reader, so it is not information
+                continue
+            elif isinstance(node, Template):
+                # the parameter values are the information; the template
+                # name, parameter names and delimiters are the polish that
+                # someone later wrapped around it
+                for param in node.params:
+                    self._count_content(param.value, counts)
+            elif isinstance(node, Tag):
+                # tag contents are information; the tag name and attributes
+                # are markup. This covers ''italic'' and '''bold''', which
+                # mwparserfromhell represents as i and b tags
+                if node.contents is not None:
+                    self._count_content(node.contents, counts)
+            elif isinstance(node, Wikilink):
+                # the label if present, otherwise the target, is what a
+                # reader sees; the brackets and the link target are markup
+                rendered = node.text if node.text is not None else node.title
+                self._count_content(rendered, counts)
+            elif isinstance(node, ExternalLink):
+                # both halves are information someone typed: a contributor
+                # who did not know wikitext would still have written the URL
+                counts[0] += len(str(node.url))
+                if node.title is not None:
+                    self._count_content(node.title, counts)
+            elif isinstance(node, Heading):
+                self._count_content(node.title, counts)
+            elif isinstance(node, Argument):
+                if node.default is not None:
+                    self._count_content(node.default, counts)
+
+    def content_sizes(self, text: str | None) -> tuple[int, int] | None:
+        """Split a revision into the information it carries and the markup
+        around it.
+
+        Returns (content_chars, markup_chars), which sum to len(text).
+
+        Content is what a contributor who did not know wikitext would have
+        typed: the words, the link labels, the values inside template
+        parameters. Markup is everything a later editor added to format it —
+        braces, pipes, parameter names, tag names and attributes, link
+        targets, quote marks, and comments.
+
+        So "Wonderful Hotel, +15555550123" is almost all content, while
+        "{{listing|name=Wonderful Hotel|phone=+15555550123}}" carries the
+        same information with two dozen characters of markup around it.
+        """
+        if text is None:
+            return None
+        if text == self._sizes_text:
+            return self._cached_sizes
+        try:
+            wikicode = self._get_wikicode(text)
+            counts = [0]
+            self._count_content(wikicode, counts)
+            content = min(counts[0], len(text))
+            result = (content, len(text) - content)
+        except Exception:
+            result = None
+        self._sizes_text = text
+        self._cached_sizes = result
+        return result
+
     def extract_headings(self, text: str | None) -> list[dict] | None:
         """Extract all section headings with their levels."""
         if text is None:

test/test_wikiq.py

@@ -12,7 +12,7 @@ import pytest
 from pandas import DataFrame
 from pandas.testing import assert_frame_equal, assert_series_equal
 
-from wikiq import build_table, build_schema, RegexPair
+from wikiq import build_table, build_schema, RegexPair, WikitextParser
 from wikiq_test_utils import (
     BASELINE_DIR,
     IKWIKI,
@@ -484,6 +484,71 @@ def test_redirect_columns_e2e():
     assert len(redirects) > 0
     assert redirects["revision_redirect_target"].notna().all()
 
+def test_content_sizes():
+    # hand-computed fixtures: (text, content_chars, markup_chars)
+    cases = [
+        # plain text is all content
+        ("Hello world.", 12, 0),
+        # the same information written by someone who knows wikitext: the
+        # values survive as content, the scaffolding around them is markup
+        ("Wonderful Hotel, +15555550123", 29, 0),
+        ("{{listing|name=Wonderful Hotel|phone=+15555550123}}", 27, 24),
+        # emphasis is markup applied to content, not content itself
+        ("''whatever''", 8, 4),
+        ("Some '''bold''' prose.", 16, 6),
+        # tag contents are content; the tag name and attributes are not
+        ('Before <span class="x">inner text</span> after', 23, 23),
+        # a wikilink renders its label, so the target is markup
+        ("[[Main Page|the main page]]", 13, 14),
+        # an unlabeled wikilink renders its target, which is then content
+        ("[[Help]]", 4, 4),
+        # both halves of an external link are information someone typed
+        ("[http://x.com label]", 17, 3),
+        # comments never reach a reader, so they are entirely markup
+        ("Text <!-- hidden --> more", 10, 15),
+        # nesting: values at any depth are content
+        ("{{outer|a={{inner|b=deep}} shallow}}", 12, 24),
+    ]
+
+    parser = WikitextParser()
+    for text, content, markup in cases:
+        result = parser.content_sizes(text)
+        assert result == (content, markup), f"{text!r}: got {result}, want {(content, markup)}"
+        # every character lands in exactly one column
+        assert result[0] + result[1] == len(text), f"{text!r}: columns do not sum to length"
+
+    assert parser.content_sizes(None) is None
+
+def test_content_sizes_e2e():
+    # the ikwiki dump contains revisions with deleted text, which must get
+    # null content sizes; for every other revision the two columns must
+    # account for the whole revision
+    tester = WikiqTester(IKWIKI, "content_sizes")
+
+    try:
+        tester.call_wikiq("--content-sizes", "--text")
+    except subprocess.CalledProcessError as exc:
+        pytest.fail(exc.stderr.decode("utf8"))
+
+    test = pd.read_table(tester.output)
+
+    assert "content_chars" in test.columns
+    assert "markup_chars" in test.columns
+
+    deleted = test[test["deleted"]]
+    assert len(deleted) > 0
+    assert deleted["content_chars"].isna().all()
+    assert deleted["markup_chars"].isna().all()
+
+    present = test[~test["deleted"] & test["text"].notna() & test["content_chars"].notna()]
+    assert len(present) > 0
+    total = present["content_chars"] + present["markup_chars"]
+    assert (total == present["text"].str.len()).all()
+    # a real wiki has both kinds of revision: some almost pure text, some
+    # carrying a lot of markup
+    assert (present["markup_chars"] == 0).any()
+    assert (present["markup_chars"] > 0).any()
+
 def test_external_links_only():
     """Test that --external-links extracts external links correctly."""
     import mwparserfromhell