Merge branch 'mako_changes-20260806' into content-sizes
Brings in revision-level redirect detection and its regenerated baselines. Conflicts were the expected adjacent insertions where both branches extended the same seams (build_table and WikiqParser signatures, the argparse block, the build_table call sites, and the test file); resolved by keeping both branches' additions. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -527,6 +527,63 @@ def test_content_sizes_e2e():
|
||||
total = present["prose_chars"] + present["structured_chars"]
|
||||
assert (total <= present["text"].str.len()).all()
|
||||
|
||||
def test_redirect_detection():
|
||||
from wikiq.tables import RedirectDetector
|
||||
|
||||
detector = RedirectDetector()
|
||||
|
||||
# a plain redirect directive
|
||||
assert detector.detect("#REDIRECT [[Target]]") == (True, "Target")
|
||||
|
||||
# localized aliases are only recognized when configured
|
||||
assert detector.detect("#OMDIRIGERING [[Mål]]") == (False, None)
|
||||
swedish = RedirectDetector(["OMDIRIGERING"])
|
||||
assert swedish.detect("#OMDIRIGERING [[Mål]]") == (True, "Mål")
|
||||
assert swedish.detect("#REDIRECT [[Target]]") == (True, "Target")
|
||||
|
||||
# fragment and label are stripped from the target
|
||||
assert detector.detect("#REDIRECT [[Target#Section|label]]") == (True, "Target")
|
||||
|
||||
# leading whitespace and lowercase are accepted
|
||||
assert detector.detect(" \n#redirect [[Target]]") == (True, "Target")
|
||||
|
||||
# each revision is classified by its own text
|
||||
history = ["#REDIRECT [[A]]", "An article now.", "#REDIRECT [[B]]"]
|
||||
assert [detector.detect(t) for t in history] == [
|
||||
(True, "A"), (False, None), (True, "B"),
|
||||
]
|
||||
|
||||
# a directive that is not at the start of the text is not a redirect
|
||||
assert detector.detect("Some text. #REDIRECT [[Target]]") == (False, None)
|
||||
|
||||
def test_redirect_columns_e2e():
|
||||
# revision-level redirect columns on a real dump: the ikwiki dump
|
||||
# contains both redirect revisions and revisions with deleted text
|
||||
tester = WikiqTester(IKWIKI, "redirect_columns")
|
||||
|
||||
try:
|
||||
tester.call_wikiq()
|
||||
except subprocess.CalledProcessError as exc:
|
||||
pytest.fail(exc.stderr.decode("utf8"))
|
||||
|
||||
test = pd.read_table(tester.output)
|
||||
|
||||
assert "revision_is_redirect" in test.columns
|
||||
assert "revision_redirect_target" in test.columns
|
||||
assert "redirect_target" not in test.columns
|
||||
|
||||
# revisions with deleted text have null values in both columns
|
||||
deleted = test[test["deleted"]]
|
||||
assert len(deleted) > 0
|
||||
assert deleted["revision_is_redirect"].isna().all()
|
||||
assert deleted["revision_redirect_target"].isna().all()
|
||||
|
||||
# the dump contains real redirect revisions, and every detected
|
||||
# redirect has a target
|
||||
redirects = test[test["revision_is_redirect"] == True]
|
||||
assert len(redirects) > 0
|
||||
assert redirects["revision_redirect_target"].notna().all()
|
||||
|
||||
def test_external_links_only():
|
||||
"""Test that --external-links extracts external links correctly."""
|
||||
import mwparserfromhell
|
||||
|
||||
Reference in New Issue
Block a user