diff --git a/README.md b/README.md index 720e92c..a67cda4 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,10 @@ The most commonly useful options (`wikiq --help` describes them all): - `--external-links`, `--citations`, `--wikilinks`, `--templates`, and `--headings` parse each revision's wikitext and add a column with the extracted elements. +- `--content-sizes` counts each revision's rendered content characters + in two columns: `prose_chars` (content outside templates and tags) and + `structured_chars` (template parameter values and tag inner text). + Markup syntax and comments count in neither. - `-RP REGEX -RPl LABEL` searches revision text for a regular expression and reports the matches in a column named by the label; repeat the pair for multiple patterns. `-CP`/`-CPl` do the same for edit diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index 70fff19..22a9385 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -133,6 +133,7 @@ def build_table( wikilinks: bool = False, templates: bool = False, headings: bool = False, + content_sizes: bool = False, ): """Build the RevisionTable with appropriate columns based on flags. @@ -169,7 +170,7 @@ def build_table( table.columns.append(tables.RevisionCollapsed()) wikitext_parser = None - if external_links or citations or wikilinks or templates or headings: + if external_links or citations or wikilinks or templates or headings or content_sizes: wikitext_parser = WikitextParser() if external_links: @@ -187,6 +188,10 @@ def build_table( if headings: table.columns.append(tables.RevisionHeadings(wikitext_parser)) + if content_sizes: + table.columns.append(tables.RevisionProseChars(wikitext_parser)) + table.columns.append(tables.RevisionStructuredChars(wikitext_parser)) + table.columns.append(tables.RevisionParserTimeout(wikitext_parser)) return table, reverts_column, wikitext_parser @@ -561,6 +566,7 @@ class WikiqParser: wikilinks: bool = False, templates: bool = False, headings: bool = False, + content_sizes: bool = False, time_limit_seconds: Union[float, None] = None, input_filename: Union[str, None] = None, ): @@ -586,6 +592,7 @@ class WikiqParser: self.wikilinks = wikilinks self.templates = templates self.headings = headings + self.content_sizes = content_sizes self.shutdown_requested = False self.time_limit_seconds = time_limit_seconds if namespaces is not None: @@ -711,6 +718,7 @@ class WikiqParser: wikilinks=self.wikilinks, templates=self.templates, headings=self.headings, + content_sizes=self.content_sizes, ) # Extract list of namespaces @@ -1279,6 +1287,14 @@ def main(): help="Extract section headings from each revision.", ) + parser.add_argument( + "--content-sizes", + dest="content_sizes", + action="store_true", + default=False, + help="Count characters of rendered content in each revision, split into prose_chars (outside templates and tags) and structured_chars (template parameter values and tag inner text). Markup syntax and comments count in neither.", + ) + parser.add_argument( "--fandom-2020", dest="fandom_2020", @@ -1342,6 +1358,7 @@ def main(): wikilinks=args.wikilinks, templates=args.templates, headings=args.headings, + content_sizes=args.content_sizes, ) schema = build_schema( table, @@ -1424,6 +1441,7 @@ def main(): wikilinks=args.wikilinks, templates=args.templates, headings=args.headings, + content_sizes=args.content_sizes, time_limit_seconds=time_limit_seconds, input_filename=filename, ) @@ -1480,6 +1498,7 @@ def main(): wikilinks=args.wikilinks, templates=args.templates, headings=args.headings, + content_sizes=args.content_sizes, time_limit_seconds=time_limit_seconds, ) diff --git a/src/wikiq/tables.py b/src/wikiq/tables.py index 31bb60e..a77b489 100644 --- a/src/wikiq/tables.py +++ b/src/wikiq/tables.py @@ -321,6 +321,40 @@ class RevisionHeadings(RevisionField[Union[list[dict], None]]): return self.wikitext_parser.extract_headings(revision.text) +class RevisionProseChars(RevisionField[Union[int, None]]): + """Count characters of rendered content outside templates and tags.""" + + field = pa.field("prose_chars", pa.int64(), nullable=True) + + def __init__(self, wikitext_parser: "WikitextParser"): + super().__init__() + self.wikitext_parser = wikitext_parser + + def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]: + revision = revisions[-1] + if revision.deleted.text: + return None + sizes = self.wikitext_parser.content_sizes(revision.text) + return None if sizes is None else sizes[0] + + +class RevisionStructuredChars(RevisionField[Union[int, None]]): + """Count characters of rendered content inside templates and tags.""" + + field = pa.field("structured_chars", pa.int64(), nullable=True) + + def __init__(self, wikitext_parser: "WikitextParser"): + super().__init__() + self.wikitext_parser = wikitext_parser + + def extract(self, page: mwtypes.Page, revisions: list[mwxml.Revision]) -> Union[int, None]: + revision = revisions[-1] + if revision.deleted.text: + return None + sizes = self.wikitext_parser.content_sizes(revision.text) + return None if sizes is None else sizes[1] + + class RevisionParserTimeout(RevisionField[bool]): """Track whether the wikitext parser timed out for this revision.""" diff --git a/src/wikiq/wikitext_parser.py b/src/wikiq/wikitext_parser.py index 56921b4..9505e5c 100644 --- a/src/wikiq/wikitext_parser.py +++ b/src/wikiq/wikitext_parser.py @@ -20,6 +20,17 @@ from __future__ import annotations import signal import mwparserfromhell +from mwparserfromhell.nodes import ( + Argument, + Comment, + ExternalLink, + Heading, + HTMLEntity, + Tag, + Template, + Text, + Wikilink, +) PARSER_TIMEOUT = 60 # seconds @@ -38,6 +49,8 @@ class WikitextParser: self._cached_text: str | None = None self._cached_wikicode = None self.last_parse_timed_out: bool = False + self._sizes_text: str | None = None + self._cached_sizes: tuple[int, int] | None = None def _timeout_handler(self, signum, frame): raise TimeoutError("mwparserfromhell parse exceeded timeout") @@ -146,6 +159,71 @@ class WikitextParser: except Exception: return None + def _count_content(self, wikicode, structured: bool, counts: list[int]) -> None: + """Walk a wikicode tree adding rendered content lengths to counts. + + counts is a two-element list of [prose, structured] character + counts. structured is True once any enclosing template parameter + value or tag contents has been entered. + """ + for node in wikicode.nodes: + if isinstance(node, Text): + counts[1 if structured else 0] += len(node.value) + elif isinstance(node, HTMLEntity): + counts[1 if structured else 0] += len(node.normalize()) + elif isinstance(node, Comment): + continue + elif isinstance(node, Template): + # parameter values are structured content; names and + # delimiters are not counted + for param in node.params: + self._count_content(param.value, True, counts) + elif isinstance(node, Tag): + # tag inner text is structured content; tag names and + # attributes are not counted + if node.contents is not None: + self._count_content(node.contents, True, counts) + elif isinstance(node, Wikilink): + # only the rendered text counts: the label if present, + # otherwise the target + rendered = node.text if node.text is not None else node.title + self._count_content(rendered, structured, counts) + elif isinstance(node, ExternalLink): + # only the label renders as content; bare URLs count as + # neither + if node.title is not None: + self._count_content(node.title, structured, counts) + elif isinstance(node, Heading): + self._count_content(node.title, structured, counts) + elif isinstance(node, Argument): + if node.default is not None: + self._count_content(node.default, structured, counts) + + def content_sizes(self, text: str | None) -> tuple[int, int] | None: + """Count characters of substantive content outside and inside + structured elements. + + Returns (prose_chars, structured_chars). Prose is the rendered text + not enclosed by any template or tag; structured is the same kinds of + content inside template parameter values or tag contents. Markup + syntax, template and tag names, parameter names, attributes, bare + URLs, and comments count in neither. + """ + if text is None: + return None + if text == self._sizes_text: + return self._cached_sizes + try: + wikicode = self._get_wikicode(text) + counts = [0, 0] + self._count_content(wikicode, False, counts) + result = (counts[0], counts[1]) + except Exception: + result = None + self._sizes_text = text + self._cached_sizes = result + return result + def extract_headings(self, text: str | None) -> list[dict] | None: """Extract all section headings with their levels.""" if text is None: diff --git a/test/Wikiq_Unit_Test.py b/test/Wikiq_Unit_Test.py index e987028..d231cd6 100644 --- a/test/Wikiq_Unit_Test.py +++ b/test/Wikiq_Unit_Test.py @@ -12,7 +12,7 @@ import pytest from pandas import DataFrame from pandas.testing import assert_frame_equal, assert_series_equal -from wikiq import build_table, build_schema, RegexPair +from wikiq import build_table, build_schema, RegexPair, WikitextParser from wikiq_test_utils import ( BASELINE_DIR, IKWIKI, @@ -469,6 +469,64 @@ def test_regex_count_e2e(): expected = len(re.findall(r"\b[Cc]hevalier\b", comment)) if comment is not None else 0 assert row["chev"] == expected, f"revid {row['revid']}: {row['chev']} != {expected}" +def test_content_sizes(): + # hand-computed fixtures: (text, prose_chars, structured_chars) + cases = [ + # pure prose + ("Hello world.", 12, 0), + # prose plus a listing-style template: parameter values count as + # structured, parameter names and braces do not + # prose: "Visit us. " = 10; structured: Hotel(5) + 555(3) + Nice place(10) = 18 + ("Visit us. {{listing|name=Hotel|phone=555|description=Nice place}}", 10, 18), + # tag inner text counts, attributes do not + # prose: "Before "(7) + " after"(6) = 13; structured: "inner text" = 10 + ('Before inner text after', 13, 10), + # nesting: a template inside a template argument stays structured + # structured: deep(4) + " shallow"(8) = 12 + ("{{outer|a={{inner|b=deep}} shallow}}", 0, 12), + # a labeled wikilink in prose counts its label; a wikilink inside a + # template argument counts its rendered text as structured + # prose: "See "(4) + "the main page"(13) + " and "(5) = 22; structured: Help(4) + ("See [[Main Page|the main page]] and {{box|link=[[Help]]}}", 22, 4), + # HTML comments count in neither column + # prose: "Text "(5) + " more"(5) = 10 + ("Text more", 10, 0), + ] + + parser = WikitextParser() + for text, prose, structured in cases: + result = parser.content_sizes(text) + assert result == (prose, structured), f"{text!r}: got {result}, want {(prose, structured)}" + assert result[0] + result[1] <= len(text), f"{text!r}: content exceeds total size" + + assert parser.content_sizes(None) is None + +def test_content_sizes_e2e(): + # the ikwiki dump contains revisions with deleted text, which must get + # null content sizes; all other revisions must satisfy the invariant + # prose_chars + structured_chars <= len(text) + tester = WikiqTester(IKWIKI, "content_sizes") + + try: + tester.call_wikiq("--content-sizes", "--text") + except subprocess.CalledProcessError as exc: + pytest.fail(exc.stderr.decode("utf8")) + + test = pd.read_table(tester.output) + + assert "prose_chars" in test.columns + assert "structured_chars" in test.columns + + deleted = test[test["deleted"]] + assert len(deleted) > 0 + assert deleted["prose_chars"].isna().all() + assert deleted["structured_chars"].isna().all() + + present = test[~test["deleted"] & test["text"].notna() & test["prose_chars"].notna()] + assert len(present) > 0 + total = present["prose_chars"] + present["structured_chars"] + assert (total <= present["text"].str.len()).all() + def test_external_links_only(): """Test that --external-links extracts external links correctly.""" import mwparserfromhell