Merge branch 'mako_changes-20260806' into content-sizes

pip install . copies the source, so a branch switch leaves the installed
console script running the old code. Say so, since that is easy to miss.

Setup on klone differs enough that duplicating it here would go stale;
link to the collective's wiki rather than restating it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -28,6 +28,15 @@ depends on `7za`, `zcat`, and `bzcat` for those respective formats. On
 Debian or Ubuntu, `apt install 7zip` provides `7za`; the others are
 standard.
 
+Note that `pip install .` copies the source into the environment, so
+rerun it after switching branches or editing the code.
+
+Members of the Community Data Science Collective installing on the klone
+cluster should follow the collective's own instructions instead, since
+the Python environment and its dependencies are set up differently
+there. See [CommunityData:Hyak software
+installation](https://wiki.communitydata.science/CommunityData:Hyak_software_installation).
+
 ## Usage
 
     wikiq dump.xml.7z -o output/
@@ -65,6 +74,10 @@ The most commonly useful options (`wikiq --help` describes them all):
 - `--external-links`, `--citations`, `--wikilinks`, `--templates`, and
   `--headings` parse each revision's wikitext and add a column with the
   extracted elements.
+- `--content-sizes` counts each revision's rendered content characters
+  in two columns: `prose_chars` (content outside templates and tags) and
+  `structured_chars` (template parameter values and tag inner text).
+  Markup syntax and comments count in neither.
 - `-RP REGEX -RPl LABEL` searches revision text for a regular expression
   and reports the matches in a column named by the label; repeat the
   pair for multiple patterns. `-CP`/`-CPl` do the same for edit

src/wikiq/__init__.py

@@ -133,6 +133,7 @@ def build_table(
     wikilinks: bool = False,
     templates: bool = False,
     headings: bool = False,
+    content_sizes: bool = False,
     redirect_aliases: Union[list[str], None] = None,
 ):
     """Build the RevisionTable with appropriate columns based on flags.
@@ -172,7 +173,7 @@ def build_table(
         table.columns.append(tables.RevisionCollapsed())
 
     wikitext_parser = None
-    if external_links or citations or wikilinks or templates or headings:
+    if external_links or citations or wikilinks or templates or headings or content_sizes:
         wikitext_parser = WikitextParser()
 
         if external_links:
@@ -190,6 +191,10 @@ def build_table(
         if headings:
             table.columns.append(tables.RevisionHeadings(wikitext_parser))
 
+        if content_sizes:
+            table.columns.append(tables.RevisionProseChars(wikitext_parser))
+            table.columns.append(tables.RevisionStructuredChars(wikitext_parser))
+
         table.columns.append(tables.RevisionParserTimeout(wikitext_parser))
 
     return table, reverts_column, wikitext_parser
@@ -564,6 +569,7 @@ class WikiqParser:
         wikilinks: bool = False,
         templates: bool = False,
         headings: bool = False,
+        content_sizes: bool = False,
         redirect_aliases: Union[list[str], None] = None,
         time_limit_seconds: Union[float, None] = None,
         input_filename: Union[str, None] = None,
@@ -590,6 +596,7 @@ class WikiqParser:
         self.wikilinks = wikilinks
         self.templates = templates
         self.headings = headings
+        self.content_sizes = content_sizes
         self.redirect_aliases = redirect_aliases
         self.shutdown_requested = False
         self.time_limit_seconds = time_limit_seconds
@@ -716,6 +723,7 @@ class WikiqParser:
             wikilinks=self.wikilinks,
             templates=self.templates,
             headings=self.headings,
+            content_sizes=self.content_sizes,
             redirect_aliases=self.redirect_aliases,
         )
 
@@ -1285,6 +1293,14 @@ def main():
         help="Extract section headings from each revision.",
     )
 
+    parser.add_argument(
+        "--content-sizes",
+        dest="content_sizes",
+        action="store_true",
+        default=False,
+        help="Count characters of rendered content in each revision, split into prose_chars (outside templates and tags) and structured_chars (template parameter values and tag inner text). Markup syntax and comments count in neither.",
+    )
+
     parser.add_argument(
         "--redirect-aliases",
         dest="redirect_aliases",
@@ -1367,6 +1383,7 @@ def main():
             wikilinks=args.wikilinks,
             templates=args.templates,
             headings=args.headings,
+            content_sizes=args.content_sizes,
             redirect_aliases=redirect_aliases,
         )
         schema = build_schema(
@@ -1450,6 +1467,7 @@ def main():
                 wikilinks=args.wikilinks,
                 templates=args.templates,
                 headings=args.headings,
+                content_sizes=args.content_sizes,
                 redirect_aliases=redirect_aliases,
                 time_limit_seconds=time_limit_seconds,
                 input_filename=filename,
@@ -1507,6 +1525,7 @@ def main():
             wikilinks=args.wikilinks,
             templates=args.templates,
             headings=args.headings,
+            content_sizes=args.content_sizes,
             redirect_aliases=redirect_aliases,
             time_limit_seconds=time_limit_seconds,
         )

src/wikiq/tables.py

@@ -384,6 +384,40 @@ class RevisionHeadings(RevisionField[Union[list[dict], None]]):
         return self.wikitext_parser.extract_headings(revision.text)
 
 
+class RevisionProseChars(RevisionField[Union[int, None]]):
+    """Count characters of rendered content outside templates and tags."""
+
+    field = pa.field("prose_chars", pa.int64(), nullable=True)
+
+    def __init__(self, wikitext_parser: "WikitextParser"):
+        super().__init__()
+        self.wikitext_parser = wikitext_parser
+
+    def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]:
+        revision = revisions[-1]
+        if revision.deleted.text:
+            return None
+        sizes = self.wikitext_parser.content_sizes(revision.text)
+        return None if sizes is None else sizes[0]
+
+
+class RevisionStructuredChars(RevisionField[Union[int, None]]):
+    """Count characters of rendered content inside templates and tags."""
+
+    field = pa.field("structured_chars", pa.int64(), nullable=True)
+
+    def __init__(self, wikitext_parser: "WikitextParser"):
+        super().__init__()
+        self.wikitext_parser = wikitext_parser
+
+    def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]:
+        revision = revisions[-1]
+        if revision.deleted.text:
+            return None
+        sizes = self.wikitext_parser.content_sizes(revision.text)
+        return None if sizes is None else sizes[1]
+
+
 class RevisionParserTimeout(RevisionField[bool]):
     """Track whether the wikitext parser timed out for this revision."""
 

src/wikiq/wikitext_parser.py

@@ -20,6 +20,17 @@ from __future__ import annotations
 import signal
 
 import mwparserfromhell
+from mwparserfromhell.nodes import (
+    Argument,
+    Comment,
+    ExternalLink,
+    Heading,
+    HTMLEntity,
+    Tag,
+    Template,
+    Text,
+    Wikilink,
+)
 
 PARSER_TIMEOUT = 60  # seconds
 
@@ -38,6 +49,8 @@ class WikitextParser:
         self._cached_text: str | None = None
         self._cached_wikicode = None
         self.last_parse_timed_out: bool = False
+        self._sizes_text: str | None = None
+        self._cached_sizes: tuple[int, int] | None = None
 
     def _timeout_handler(self, signum, frame):
         raise TimeoutError("mwparserfromhell parse exceeded timeout")
@@ -146,6 +159,71 @@ class WikitextParser:
         except Exception:
             return None
 
+    def _count_content(self, wikicode, structured: bool, counts: list[int]) -> None:
+        """Walk a wikicode tree adding rendered content lengths to counts.
+
+        counts is a two-element list of [prose, structured] character
+        counts. structured is True once any enclosing template parameter
+        value or tag contents has been entered.
+        """
+        for node in wikicode.nodes:
+            if isinstance(node, Text):
+                counts[1 if structured else 0] += len(node.value)
+            elif isinstance(node, HTMLEntity):
+                counts[1 if structured else 0] += len(node.normalize())
+            elif isinstance(node, Comment):
+                continue
+            elif isinstance(node, Template):
+                # parameter values are structured content; names and
+                # delimiters are not counted
+                for param in node.params:
+                    self._count_content(param.value, True, counts)
+            elif isinstance(node, Tag):
+                # tag inner text is structured content; tag names and
+                # attributes are not counted
+                if node.contents is not None:
+                    self._count_content(node.contents, True, counts)
+            elif isinstance(node, Wikilink):
+                # only the rendered text counts: the label if present,
+                # otherwise the target
+                rendered = node.text if node.text is not None else node.title
+                self._count_content(rendered, structured, counts)
+            elif isinstance(node, ExternalLink):
+                # only the label renders as content; bare URLs count as
+                # neither
+                if node.title is not None:
+                    self._count_content(node.title, structured, counts)
+            elif isinstance(node, Heading):
+                self._count_content(node.title, structured, counts)
+            elif isinstance(node, Argument):
+                if node.default is not None:
+                    self._count_content(node.default, structured, counts)
+
+    def content_sizes(self, text: str | None) -> tuple[int, int] | None:
+        """Count characters of substantive content outside and inside
+        structured elements.
+
+        Returns (prose_chars, structured_chars). Prose is the rendered text
+        not enclosed by any template or tag; structured is the same kinds of
+        content inside template parameter values or tag contents. Markup
+        syntax, template and tag names, parameter names, attributes, bare
+        URLs, and comments count in neither.
+        """
+        if text is None:
+            return None
+        if text == self._sizes_text:
+            return self._cached_sizes
+        try:
+            wikicode = self._get_wikicode(text)
+            counts = [0, 0]
+            self._count_content(wikicode, False, counts)
+            result = (counts[0], counts[1])
+        except Exception:
+            result = None
+        self._sizes_text = text
+        self._cached_sizes = result
+        return result
+
     def extract_headings(self, text: str | None) -> list[dict] | None:
         """Extract all section headings with their levels."""
         if text is None:

test/Wikiq_Unit_Test.py

@@ -12,7 +12,7 @@ import pytest
 from pandas import DataFrame
 from pandas.testing import assert_frame_equal, assert_series_equal
 
-from wikiq import build_table, build_schema, RegexPair
+from wikiq import build_table, build_schema, RegexPair, WikitextParser
 from wikiq_test_utils import (
     BASELINE_DIR,
     IKWIKI,
@@ -469,6 +469,64 @@ def test_regex_count_e2e():
         expected = len(re.findall(r"\b[Cc]hevalier\b", comment)) if comment is not None else 0
         assert row["chev"] == expected, f"revid {row['revid']}: {row['chev']} != {expected}"
 
+def test_content_sizes():
+    # hand-computed fixtures: (text, prose_chars, structured_chars)
+    cases = [
+        # pure prose
+        ("Hello world.", 12, 0),
+        # prose plus a listing-style template: parameter values count as
+        # structured, parameter names and braces do not
+        # prose: "Visit us. " = 10; structured: Hotel(5) + 555(3) + Nice place(10) = 18
+        ("Visit us. {{listing|name=Hotel|phone=555|description=Nice place}}", 10, 18),
+        # tag inner text counts, attributes do not
+        # prose: "Before "(7) + " after"(6) = 13; structured: "inner text" = 10
+        ('Before <span class="x">inner text</span> after', 13, 10),
+        # nesting: a template inside a template argument stays structured
+        # structured: deep(4) + " shallow"(8) = 12
+        ("{{outer|a={{inner|b=deep}} shallow}}", 0, 12),
+        # a labeled wikilink in prose counts its label; a wikilink inside a
+        # template argument counts its rendered text as structured
+        # prose: "See "(4) + "the main page"(13) + " and "(5) = 22; structured: Help(4)
+        ("See [[Main Page|the main page]] and {{box|link=[[Help]]}}", 22, 4),
+        # HTML comments count in neither column
+        # prose: "Text "(5) + " more"(5) = 10
+        ("Text <!-- hidden --> more", 10, 0),
+    ]
+
+    parser = WikitextParser()
+    for text, prose, structured in cases:
+        result = parser.content_sizes(text)
+        assert result == (prose, structured), f"{text!r}: got {result}, want {(prose, structured)}"
+        assert result[0] + result[1] <= len(text), f"{text!r}: content exceeds total size"
+
+    assert parser.content_sizes(None) is None
+
+def test_content_sizes_e2e():
+    # the ikwiki dump contains revisions with deleted text, which must get
+    # null content sizes; all other revisions must satisfy the invariant
+    # prose_chars + structured_chars <= len(text)
+    tester = WikiqTester(IKWIKI, "content_sizes")
+
+    try:
+        tester.call_wikiq("--content-sizes", "--text")
+    except subprocess.CalledProcessError as exc:
+        pytest.fail(exc.stderr.decode("utf8"))
+
+    test = pd.read_table(tester.output)
+
+    assert "prose_chars" in test.columns
+    assert "structured_chars" in test.columns
+
+    deleted = test[test["deleted"]]
+    assert len(deleted) > 0
+    assert deleted["prose_chars"].isna().all()
+    assert deleted["structured_chars"].isna().all()
+
+    present = test[~test["deleted"] & test["text"].notna() & test["prose_chars"].notna()]
+    assert len(present) > 0
+    total = present["prose_chars"] + present["structured_chars"]
+    assert (total <= present["text"].str.len()).all()
+
 def test_redirect_detection():
     from wikiq.tables import RedirectDetector