add --content-sizes: decompose revision content into prose and structured characters

A single mwparserfromhell tree walk per revision emits two nullable
int64 columns. prose_chars counts rendered content outside any template
or tag: text nodes, wikilink labels (or targets when unlabeled),
external link labels, and heading titles. structured_chars counts the
same kinds of content inside template parameter values or tag contents,
with each character assigned by its nearest enclosing container. Markup
syntax, template and tag names, parameter names, attributes, bare URLs,
and HTML comments count in neither, so syntax overhead is derivable as
the total size minus the two columns.

Deleted revisions and parse timeouts yield nulls through the same
machinery as the other parser-based columns and do not change which
revisions are processed. Note that mwparserfromhell parses bold and
italic markup as tags, so emphasized text counts as structured.

Includes hand-computed fixture tests for the counting rules and an
end-to-end test on a dump with deleted revisions verifying nulls and
the invariant prose_chars + structured_chars <= revision size.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

test/Wikiq_Unit_Test.py

@@ -12,7 +12,7 @@ import pytest
 from pandas import DataFrame
 from pandas.testing import assert_frame_equal, assert_series_equal
 
-from wikiq import build_table, build_schema, RegexPair
+from wikiq import build_table, build_schema, RegexPair, WikitextParser
 from wikiq_test_utils import (
     BASELINE_DIR,
     IKWIKI,
@@ -469,6 +469,64 @@ def test_regex_count_e2e():
         expected = len(re.findall(r"\b[Cc]hevalier\b", comment)) if comment is not None else 0
         assert row["chev"] == expected, f"revid {row['revid']}: {row['chev']} != {expected}"
 
+def test_content_sizes():
+    # hand-computed fixtures: (text, prose_chars, structured_chars)
+    cases = [
+        # pure prose
+        ("Hello world.", 12, 0),
+        # prose plus a listing-style template: parameter values count as
+        # structured, parameter names and braces do not
+        # prose: "Visit us. " = 10; structured: Hotel(5) + 555(3) + Nice place(10) = 18
+        ("Visit us. {{listing|name=Hotel|phone=555|description=Nice place}}", 10, 18),
+        # tag inner text counts, attributes do not
+        # prose: "Before "(7) + " after"(6) = 13; structured: "inner text" = 10
+        ('Before <span class="x">inner text</span> after', 13, 10),
+        # nesting: a template inside a template argument stays structured
+        # structured: deep(4) + " shallow"(8) = 12
+        ("{{outer|a={{inner|b=deep}} shallow}}", 0, 12),
+        # a labeled wikilink in prose counts its label; a wikilink inside a
+        # template argument counts its rendered text as structured
+        # prose: "See "(4) + "the main page"(13) + " and "(5) = 22; structured: Help(4)
+        ("See [[Main Page|the main page]] and {{box|link=[[Help]]}}", 22, 4),
+        # HTML comments count in neither column
+        # prose: "Text "(5) + " more"(5) = 10
+        ("Text <!-- hidden --> more", 10, 0),
+    ]
+
+    parser = WikitextParser()
+    for text, prose, structured in cases:
+        result = parser.content_sizes(text)
+        assert result == (prose, structured), f"{text!r}: got {result}, want {(prose, structured)}"
+        assert result[0] + result[1] <= len(text), f"{text!r}: content exceeds total size"
+
+    assert parser.content_sizes(None) is None
+
+def test_content_sizes_e2e():
+    # the ikwiki dump contains revisions with deleted text, which must get
+    # null content sizes; all other revisions must satisfy the invariant
+    # prose_chars + structured_chars <= len(text)
+    tester = WikiqTester(IKWIKI, "content_sizes")
+
+    try:
+        tester.call_wikiq("--content-sizes", "--text")
+    except subprocess.CalledProcessError as exc:
+        pytest.fail(exc.stderr.decode("utf8"))
+
+    test = pd.read_table(tester.output)
+
+    assert "prose_chars" in test.columns
+    assert "structured_chars" in test.columns
+
+    deleted = test[test["deleted"]]
+    assert len(deleted) > 0
+    assert deleted["prose_chars"].isna().all()
+    assert deleted["structured_chars"].isna().all()
+
+    present = test[~test["deleted"] & test["text"].notna() & test["prose_chars"].notna()]
+    assert len(present) > 0
+    total = present["prose_chars"] + present["structured_chars"]
+    assert (total <= present["text"].str.len()).all()
+
 def test_external_links_only():
     """Test that --external-links extracts external links correctly."""
     import mwparserfromhell