From af62c4cea4ec01266a1d0755fa29573657fe51ec Mon Sep 17 00:00:00 2001 From: Benjamin Mako Hill Date: Thu, 6 Aug 2026 18:09:29 -0700 Subject: [PATCH] add --content-sizes: split revisions into content and markup Wikitravel-style guides get written twice. Someone who does not know wikitext contributes the information: Wonderful Hotel, +15555550123 and someone who does comes along later and polishes it: {{listing|name=Wonderful Hotel|phone=+15555550123}} Both carry the same information. The second adds two dozen characters of markup around it. --content-sizes measures that split, emitting two nullable int64 columns from a single mwparserfromhell tree walk per revision. content_chars counts what a contributor who did not know wikitext would have typed: text nodes, template parameter values, tag contents, wikilink labels (or targets when unlabeled), external link URLs and their labels, and heading titles. markup_chars is everything else -- braces, pipes, parameter names, tag names and attributes, link targets, quote marks, and comments, which never reach a reader. The two columns sum to the revision length, so their ratio reads directly as how much of a revision is formatting. That invariant is what the end-to-end test checks on a real dump, along with nulls for deleted revisions, which come through the same machinery as the other parser-based columns. Emphasis needs no special handling: mwparserfromhell represents ''' and '' as b and i tags, so the quote marks fall out as markup and the words they wrap as content, which is the wanted answer. Co-Authored-By: Claude Fable 5 Co-Authored-By: Claude Opus 5 --- README.md | 7 +++ src/wikiq/__init__.py | 21 +++++++- src/wikiq/tables.py | 44 +++++++++++++++++ src/wikiq/wikitext_parser.py | 92 ++++++++++++++++++++++++++++++++++++ test/test_wikiq.py | 67 +++++++++++++++++++++++++- 5 files changed, 229 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index be04d19..5cb9135 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,13 @@ The most commonly useful options (`wikiq --help` describes them all): - `--external-links`, `--citations`, `--wikilinks`, `--templates`, and `--headings` parse each revision's wikitext and add a column with the extracted elements. +- `--content-sizes` splits each revision into `content_chars`, the + information it carries, and `markup_chars`, the wikitext around it. A + contributor who does not know wikitext writes `Wonderful Hotel, + +15555550123`; someone polishing it later writes + `{{listing|name=Wonderful Hotel|phone=+15555550123}}`. Both carry the + same information, and the second adds two dozen characters of markup. + The two columns sum to the revision's length. - `-RP REGEX -RPl LABEL` searches revision text for a regular expression and reports the matches in a column named by the label; repeat the pair for multiple patterns. `-CP`/`-CPl` do the same for edit diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index 8a02c0c..5a0df58 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -174,6 +174,7 @@ def build_table( wikilinks: bool = False, templates: bool = False, headings: bool = False, + content_sizes: bool = False, redirect_aliases: Union[list[str], None] = None, ): """Build the RevisionTable with appropriate columns based on flags. @@ -213,7 +214,7 @@ def build_table( table.columns.append(tables.RevisionCollapsed()) wikitext_parser = None - if external_links or citations or wikilinks or templates or headings: + if external_links or citations or wikilinks or templates or headings or content_sizes: wikitext_parser = WikitextParser() if external_links: @@ -231,6 +232,10 @@ def build_table( if headings: table.columns.append(tables.RevisionHeadings(wikitext_parser)) + if content_sizes: + table.columns.append(tables.RevisionContentChars(wikitext_parser)) + table.columns.append(tables.RevisionMarkupChars(wikitext_parser)) + table.columns.append(tables.RevisionParserTimeout(wikitext_parser)) return table, reverts_column, wikitext_parser @@ -590,6 +595,7 @@ class WikiqParser: wikilinks: bool = False, templates: bool = False, headings: bool = False, + content_sizes: bool = False, redirect_aliases: Union[list[str], None] = None, time_limit_seconds: Union[float, None] = None, input_filename: Union[str, None] = None, @@ -616,6 +622,7 @@ class WikiqParser: self.wikilinks = wikilinks self.templates = templates self.headings = headings + self.content_sizes = content_sizes self.redirect_aliases = redirect_aliases self.shutdown_requested = False self.time_limit_seconds = time_limit_seconds @@ -742,6 +749,7 @@ class WikiqParser: wikilinks=self.wikilinks, templates=self.templates, headings=self.headings, + content_sizes=self.content_sizes, redirect_aliases=self.redirect_aliases, ) @@ -1298,6 +1306,14 @@ def main(): help="Extract section headings from each revision.", ) + parser.add_argument( + "--content-sizes", + dest="content_sizes", + action="store_true", + default=False, + help="Split each revision into content_chars, the information it carries, and markup_chars, the wikitext formatting around it. Content is what someone who did not know wikitext would have typed: words, link labels, and template parameter values. The two columns sum to the revision length.", + ) + parser.add_argument( "--redirect-aliases", dest="redirect_aliases", @@ -1389,6 +1405,7 @@ def main(): wikilinks=args.wikilinks, templates=args.templates, headings=args.headings, + content_sizes=args.content_sizes, redirect_aliases=redirect_aliases, ) schema = build_schema( @@ -1470,6 +1487,7 @@ def main(): wikilinks=args.wikilinks, templates=args.templates, headings=args.headings, + content_sizes=args.content_sizes, redirect_aliases=redirect_aliases, time_limit_seconds=time_limit_seconds, input_filename=filename, @@ -1525,6 +1543,7 @@ def main(): wikilinks=args.wikilinks, templates=args.templates, headings=args.headings, + content_sizes=args.content_sizes, redirect_aliases=redirect_aliases, time_limit_seconds=time_limit_seconds, ) diff --git a/src/wikiq/tables.py b/src/wikiq/tables.py index 019078b..a101208 100644 --- a/src/wikiq/tables.py +++ b/src/wikiq/tables.py @@ -384,6 +384,50 @@ class RevisionHeadings(RevisionField[Union[list[dict], None]]): return self.wikitext_parser.extract_headings(revision.text) +class RevisionContentChars(RevisionField[Union[int, None]]): + """Count the characters of information a revision carries. + + The words, link labels and template parameter values -- what someone + who did not know wikitext would have typed. See + WikitextParser.content_sizes(). + """ + + field = pa.field("content_chars", pa.int64(), nullable=True) + + def __init__(self, wikitext_parser: "WikitextParser"): + super().__init__() + self.wikitext_parser = wikitext_parser + + def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]: + revision = revisions[-1] + if revision.deleted.text: + return None + sizes = self.wikitext_parser.content_sizes(revision.text) + return None if sizes is None else sizes[0] + + +class RevisionMarkupChars(RevisionField[Union[int, None]]): + """Count the characters of markup wrapped around a revision's content. + + Braces, pipes, parameter names, tag names and attributes, link targets + and comments. Together with content_chars this sums to the revision + length. + """ + + field = pa.field("markup_chars", pa.int64(), nullable=True) + + def __init__(self, wikitext_parser: "WikitextParser"): + super().__init__() + self.wikitext_parser = wikitext_parser + + def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]: + revision = revisions[-1] + if revision.deleted.text: + return None + sizes = self.wikitext_parser.content_sizes(revision.text) + return None if sizes is None else sizes[1] + + class RevisionParserTimeout(RevisionField[bool]): """Track whether the wikitext parser timed out for this revision.""" diff --git a/src/wikiq/wikitext_parser.py b/src/wikiq/wikitext_parser.py index 56921b4..1e1a82c 100644 --- a/src/wikiq/wikitext_parser.py +++ b/src/wikiq/wikitext_parser.py @@ -20,6 +20,17 @@ from __future__ import annotations import signal import mwparserfromhell +from mwparserfromhell.nodes import ( + Argument, + Comment, + ExternalLink, + Heading, + HTMLEntity, + Tag, + Template, + Text, + Wikilink, +) PARSER_TIMEOUT = 60 # seconds @@ -38,6 +49,8 @@ class WikitextParser: self._cached_text: str | None = None self._cached_wikicode = None self.last_parse_timed_out: bool = False + self._sizes_text: str | None = None + self._cached_sizes: tuple[int, int] | None = None def _timeout_handler(self, signum, frame): raise TimeoutError("mwparserfromhell parse exceeded timeout") @@ -146,6 +159,85 @@ class WikitextParser: except Exception: return None + def _count_content(self, wikicode, counts: list[int]) -> None: + """Walk a wikicode tree adding the length of rendered content. + + counts is a one-element list holding the running content total. + Anything not counted here is markup; content_sizes() derives the + markup total by subtraction, so every character lands in one column + or the other. + """ + for node in wikicode.nodes: + if isinstance(node, Text): + counts[0] += len(node.value) + elif isinstance(node, HTMLEntity): + # an entity stands in for the character it renders as, so + # only that character is content and the escaping is markup + counts[0] += len(node.normalize()) + elif isinstance(node, Comment): + # never reaches a reader, so it is not information + continue + elif isinstance(node, Template): + # the parameter values are the information; the template + # name, parameter names and delimiters are the polish that + # someone later wrapped around it + for param in node.params: + self._count_content(param.value, counts) + elif isinstance(node, Tag): + # tag contents are information; the tag name and attributes + # are markup. This covers ''italic'' and '''bold''', which + # mwparserfromhell represents as i and b tags + if node.contents is not None: + self._count_content(node.contents, counts) + elif isinstance(node, Wikilink): + # the label if present, otherwise the target, is what a + # reader sees; the brackets and the link target are markup + rendered = node.text if node.text is not None else node.title + self._count_content(rendered, counts) + elif isinstance(node, ExternalLink): + # both halves are information someone typed: a contributor + # who did not know wikitext would still have written the URL + counts[0] += len(str(node.url)) + if node.title is not None: + self._count_content(node.title, counts) + elif isinstance(node, Heading): + self._count_content(node.title, counts) + elif isinstance(node, Argument): + if node.default is not None: + self._count_content(node.default, counts) + + def content_sizes(self, text: str | None) -> tuple[int, int] | None: + """Split a revision into the information it carries and the markup + around it. + + Returns (content_chars, markup_chars), which sum to len(text). + + Content is what a contributor who did not know wikitext would have + typed: the words, the link labels, the values inside template + parameters. Markup is everything a later editor added to format it — + braces, pipes, parameter names, tag names and attributes, link + targets, quote marks, and comments. + + So "Wonderful Hotel, +15555550123" is almost all content, while + "{{listing|name=Wonderful Hotel|phone=+15555550123}}" carries the + same information with two dozen characters of markup around it. + """ + if text is None: + return None + if text == self._sizes_text: + return self._cached_sizes + try: + wikicode = self._get_wikicode(text) + counts = [0] + self._count_content(wikicode, counts) + content = min(counts[0], len(text)) + result = (content, len(text) - content) + except Exception: + result = None + self._sizes_text = text + self._cached_sizes = result + return result + def extract_headings(self, text: str | None) -> list[dict] | None: """Extract all section headings with their levels.""" if text is None: diff --git a/test/test_wikiq.py b/test/test_wikiq.py index 802ff33..6b76917 100644 --- a/test/test_wikiq.py +++ b/test/test_wikiq.py @@ -12,7 +12,7 @@ import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal, assert_series_equal -from wikiq import build_table, build_schema, RegexPair +from wikiq import build_table, build_schema, RegexPair, WikitextParser from wikiq_test_utils import ( BASELINE_DIR, IKWIKI, @@ -484,6 +484,71 @@ def test_redirect_columns_e2e(): assert len(redirects) > 0 assert redirects["revision_redirect_target"].notna().all() +def test_content_sizes(): + # hand-computed fixtures: (text, content_chars, markup_chars) + cases = [ + # plain text is all content + ("Hello world.", 12, 0), + # the same information written by someone who knows wikitext: the + # values survive as content, the scaffolding around them is markup + ("Wonderful Hotel, +15555550123", 29, 0), + ("{{listing|name=Wonderful Hotel|phone=+15555550123}}", 27, 24), + # emphasis is markup applied to content, not content itself + ("''whatever''", 8, 4), + ("Some '''bold''' prose.", 16, 6), + # tag contents are content; the tag name and attributes are not + ('Before inner text after', 23, 23), + # a wikilink renders its label, so the target is markup + ("[[Main Page|the main page]]", 13, 14), + # an unlabeled wikilink renders its target, which is then content + ("[[Help]]", 4, 4), + # both halves of an external link are information someone typed + ("[http://x.com label]", 17, 3), + # comments never reach a reader, so they are entirely markup + ("Text more", 10, 15), + # nesting: values at any depth are content + ("{{outer|a={{inner|b=deep}} shallow}}", 12, 24), + ] + + parser = WikitextParser() + for text, content, markup in cases: + result = parser.content_sizes(text) + assert result == (content, markup), f"{text!r}: got {result}, want {(content, markup)}" + # every character lands in exactly one column + assert result[0] + result[1] == len(text), f"{text!r}: columns do not sum to length" + + assert parser.content_sizes(None) is None + +def test_content_sizes_e2e(): + # the ikwiki dump contains revisions with deleted text, which must get + # null content sizes; for every other revision the two columns must + # account for the whole revision + tester = WikiqTester(IKWIKI, "content_sizes") + + try: + tester.call_wikiq("--content-sizes", "--text") + except subprocess.CalledProcessError as exc: + pytest.fail(exc.stderr.decode("utf8")) + + test = pd.read_table(tester.output) + + assert "content_chars" in test.columns + assert "markup_chars" in test.columns + + deleted = test[test["deleted"]] + assert len(deleted) > 0 + assert deleted["content_chars"].isna().all() + assert deleted["markup_chars"].isna().all() + + present = test[~test["deleted"] & test["text"].notna() & test["content_chars"].notna()] + assert len(present) > 0 + total = present["content_chars"] + present["markup_chars"] + assert (total == present["text"].str.len()).all() + # a real wiki has both kinds of revision: some almost pure text, some + # carrying a lot of markup + assert (present["markup_chars"] == 0).any() + assert (present["markup_chars"] > 0).any() + def test_external_links_only(): """Test that --external-links extracts external links correctly.""" import mwparserfromhell