Merge branch 'mako_changes-20260806' into content-sizes

Brings in revision-level redirect detection and its regenerated
baselines. Conflicts were the expected adjacent insertions where both
branches extended the same seams (build_table and WikiqParser
signatures, the argparse block, the build_table call sites, and the
test file); resolved by keeping both branches' additions.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-06 23:52:31 -07:00
22 changed files with 137498 additions and 137340 deletions

View File

@@ -527,6 +527,63 @@ def test_content_sizes_e2e():
total = present["prose_chars"] + present["structured_chars"]
assert (total <= present["text"].str.len()).all()
def test_redirect_detection():
from wikiq.tables import RedirectDetector
detector = RedirectDetector()
# a plain redirect directive
assert detector.detect("#REDIRECT [[Target]]") == (True, "Target")
# localized aliases are only recognized when configured
assert detector.detect("#OMDIRIGERING [[Mål]]") == (False, None)
swedish = RedirectDetector(["OMDIRIGERING"])
assert swedish.detect("#OMDIRIGERING [[Mål]]") == (True, "Mål")
assert swedish.detect("#REDIRECT [[Target]]") == (True, "Target")
# fragment and label are stripped from the target
assert detector.detect("#REDIRECT [[Target#Section|label]]") == (True, "Target")
# leading whitespace and lowercase are accepted
assert detector.detect(" \n#redirect [[Target]]") == (True, "Target")
# each revision is classified by its own text
history = ["#REDIRECT [[A]]", "An article now.", "#REDIRECT [[B]]"]
assert [detector.detect(t) for t in history] == [
(True, "A"), (False, None), (True, "B"),
]
# a directive that is not at the start of the text is not a redirect
assert detector.detect("Some text. #REDIRECT [[Target]]") == (False, None)
def test_redirect_columns_e2e():
# revision-level redirect columns on a real dump: the ikwiki dump
# contains both redirect revisions and revisions with deleted text
tester = WikiqTester(IKWIKI, "redirect_columns")
try:
tester.call_wikiq()
except subprocess.CalledProcessError as exc:
pytest.fail(exc.stderr.decode("utf8"))
test = pd.read_table(tester.output)
assert "revision_is_redirect" in test.columns
assert "revision_redirect_target" in test.columns
assert "redirect_target" not in test.columns
# revisions with deleted text have null values in both columns
deleted = test[test["deleted"]]
assert len(deleted) > 0
assert deleted["revision_is_redirect"].isna().all()
assert deleted["revision_redirect_target"].isna().all()
# the dump contains real redirect revisions, and every detected
# redirect has a target
redirects = test[test["revision_is_redirect"] == True]
assert len(redirects) > 0
assert redirects["revision_redirect_target"].notna().all()
def test_external_links_only():
"""Test that --external-links extracts external links correctly."""
import mwparserfromhell