add --content-sizes: split revisions into content and markup

Wikitravel-style guides get written twice. Someone who does not know
wikitext contributes the information:

    Wonderful Hotel, +15555550123

and someone who does comes along later and polishes it:

    {{listing|name=Wonderful Hotel|phone=+15555550123}}

Both carry the same information. The second adds two dozen characters of
markup around it. --content-sizes measures that split, emitting two
nullable int64 columns from a single mwparserfromhell tree walk per
revision.

content_chars counts what a contributor who did not know wikitext would
have typed: text nodes, template parameter values, tag contents,
wikilink labels (or targets when unlabeled), external link URLs and
their labels, and heading titles. markup_chars is everything else --
braces, pipes, parameter names, tag names and attributes, link targets,
quote marks, and comments, which never reach a reader.

The two columns sum to the revision length, so their ratio reads
directly as how much of a revision is formatting. That invariant is what
the end-to-end test checks on a real dump, along with nulls for deleted
revisions, which come through the same machinery as the other
parser-based columns.

Emphasis needs no special handling: mwparserfromhell represents ''' and
'' as b and i tags, so the quote marks fall out as markup and the words
they wrap as content, which is the wanted answer.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

test/test_wikiq.py

@@ -12,7 +12,7 @@ import pytest
 from pandas import DataFrame
 from pandas.testing import assert_frame_equal, assert_series_equal
 
-from wikiq import build_table, build_schema, RegexPair
+from wikiq import build_table, build_schema, RegexPair, WikitextParser
 from wikiq_test_utils import (
     BASELINE_DIR,
     IKWIKI,
@@ -484,6 +484,71 @@ def test_redirect_columns_e2e():
     assert len(redirects) > 0
     assert redirects["revision_redirect_target"].notna().all()
 
+def test_content_sizes():
+    # hand-computed fixtures: (text, content_chars, markup_chars)
+    cases = [
+        # plain text is all content
+        ("Hello world.", 12, 0),
+        # the same information written by someone who knows wikitext: the
+        # values survive as content, the scaffolding around them is markup
+        ("Wonderful Hotel, +15555550123", 29, 0),
+        ("{{listing|name=Wonderful Hotel|phone=+15555550123}}", 27, 24),
+        # emphasis is markup applied to content, not content itself
+        ("''whatever''", 8, 4),
+        ("Some '''bold''' prose.", 16, 6),
+        # tag contents are content; the tag name and attributes are not
+        ('Before <span class="x">inner text</span> after', 23, 23),
+        # a wikilink renders its label, so the target is markup
+        ("[[Main Page|the main page]]", 13, 14),
+        # an unlabeled wikilink renders its target, which is then content
+        ("[[Help]]", 4, 4),
+        # both halves of an external link are information someone typed
+        ("[http://x.com label]", 17, 3),
+        # comments never reach a reader, so they are entirely markup
+        ("Text <!-- hidden --> more", 10, 15),
+        # nesting: values at any depth are content
+        ("{{outer|a={{inner|b=deep}} shallow}}", 12, 24),
+    ]
+
+    parser = WikitextParser()
+    for text, content, markup in cases:
+        result = parser.content_sizes(text)
+        assert result == (content, markup), f"{text!r}: got {result}, want {(content, markup)}"
+        # every character lands in exactly one column
+        assert result[0] + result[1] == len(text), f"{text!r}: columns do not sum to length"
+
+    assert parser.content_sizes(None) is None
+
+def test_content_sizes_e2e():
+    # the ikwiki dump contains revisions with deleted text, which must get
+    # null content sizes; for every other revision the two columns must
+    # account for the whole revision
+    tester = WikiqTester(IKWIKI, "content_sizes")
+
+    try:
+        tester.call_wikiq("--content-sizes", "--text")
+    except subprocess.CalledProcessError as exc:
+        pytest.fail(exc.stderr.decode("utf8"))
+
+    test = pd.read_table(tester.output)
+
+    assert "content_chars" in test.columns
+    assert "markup_chars" in test.columns
+
+    deleted = test[test["deleted"]]
+    assert len(deleted) > 0
+    assert deleted["content_chars"].isna().all()
+    assert deleted["markup_chars"].isna().all()
+
+    present = test[~test["deleted"] & test["text"].notna() & test["content_chars"].notna()]
+    assert len(present) > 0
+    total = present["content_chars"] + present["markup_chars"]
+    assert (total == present["text"].str.len()).all()
+    # a real wiki has both kinds of revision: some almost pure text, some
+    # carrying a lot of markup
+    assert (present["markup_chars"] == 0).any()
+    assert (present["markup_chars"] > 0).any()
+
 def test_external_links_only():
     """Test that --external-links extracts external links correctly."""
     import mwparserfromhell