Prepare wikiq for release: fixes, dependency cleanup, and packaging #2

Open
mako wants to merge 33 commits from release_review into jsonl-output
4 changed files with 36 additions and 0 deletions
Showing only changes of commit b10a7d0130 - Show all commits

View File

@@ -23,6 +23,8 @@ from wikiq_test_utils import (
TWINPEAKS,
WIKIQ,
WikiqTester,
requires_mediawiki_utilities,
requires_pywikidiff2,
)
@@ -205,6 +207,7 @@ def test_collapse_user():
baseline = pd.read_table(tester.baseline_file)
assert_frame_equal(test, baseline, check_like=True)
@requires_pywikidiff2
def test_pwr_wikidiff2():
tester = WikiqTester(SAILORMOON, "persistence_wikidiff2", in_compression="7z")
@@ -229,6 +232,7 @@ def test_pwr_segment():
baseline = pd.read_table(tester.baseline_file)
assert_frame_equal(test, baseline, check_like=True)
@requires_mediawiki_utilities
def test_pwr_legacy():
tester = WikiqTester(SAILORMOON, "persistence_legacy", in_compression="7z")
@@ -255,6 +259,7 @@ def test_pwr():
test = test.reindex(columns=sorted(test.columns))
assert_frame_equal(test, baseline, check_like=True)
@requires_pywikidiff2
def test_diff():
tester = WikiqTester(SAILORMOON, "diff", in_compression="7z", out_format='jsonl')
@@ -268,6 +273,7 @@ def test_diff():
assert "diff_timeout" in test.columns, "diff_timeout column should exist"
assert len(test) > 0, "Should have output rows"
@requires_pywikidiff2
def test_diff_plus_pwr():
tester = WikiqTester(SAILORMOON, "diff_pwr", in_compression="7z", out_format='jsonl')
@@ -281,6 +287,7 @@ def test_diff_plus_pwr():
assert "token_revs" in test.columns, "token_revs column should exist"
assert len(test) > 0, "Should have output rows"
@requires_pywikidiff2
def test_text():
tester = WikiqTester(SAILORMOON, "text", in_compression="7z", out_format='jsonl')

View File

@@ -15,6 +15,7 @@ from wikiq_test_utils import (
TEST_OUTPUT_DIR,
WIKIQ,
WikiqTester,
requires_pywikidiff2,
)
@@ -65,6 +66,7 @@ def test_resume():
assert_frame_equal(df_full, df_resumed)
@requires_pywikidiff2
def test_resume_with_diff():
"""Test that --resume correctly computes diff values after resume.
@@ -305,6 +307,7 @@ def test_resume_data_equivalence():
assert_frame_equal(df_full, df_resumed)
@requires_pywikidiff2
def test_resume_with_persistence():
"""Test that --resume correctly handles persistence state after resume.
@@ -403,6 +406,7 @@ def test_resume_corrupted_jsonl_last_line():
assert_frame_equal(df_full, df_resumed)
@requires_pywikidiff2
def test_resume_diff_persistence_combined():
"""Test that --resume correctly handles both diff and persistence state together.
@@ -448,6 +452,7 @@ def test_resume_diff_persistence_combined():
assert_frame_equal(df_full, df_resumed)
@requires_pywikidiff2
def test_resume_mid_page():
"""Test resume from the middle of a page with many revisions.
@@ -501,6 +506,7 @@ def test_resume_mid_page():
assert_frame_equal(df_full, df_resumed)
@requires_pywikidiff2
def test_resume_page_boundary():
"""Test resume at the exact start of a new page.

View File

@@ -7,6 +7,10 @@ from typing import List
from deltas import Delete, Equal, Insert, wikitext_split
from mwpersistence import Token
from wikiq.wiki_diff_matcher import WikiDiffMatcher
from wikiq_test_utils import requires_pywikidiff2
# every test here drives wikidiff2 directly
pytestmark = requires_pywikidiff2
def _replace_whitespace(match):
if match.group(1): # If spaces matched (e.g., ' ')

View File

@@ -1,8 +1,27 @@
import importlib.util
import os
import shutil
import subprocess
from typing import Final, Union
import pytest
def _installed(module: str) -> bool:
return importlib.util.find_spec(module) is not None
# wikiq's two optional dependencies. Tests exercising --diff, -p wikidiff2, or
# -p legacy cannot run on a base install, so they skip rather than fail.
requires_pywikidiff2 = pytest.mark.skipif(
not _installed("pywikidiff2"),
reason="needs the optional pywikidiff2 extension (--diff, -p wikidiff2)",
)
requires_mediawiki_utilities = pytest.mark.skipif(
not _installed("mw"),
reason="needs the optional mediawiki-utilities package (-p legacy)",
)
TEST_DIR: Final[str] = os.path.dirname(os.path.realpath(__file__))
WIKIQ: Final[str] = os.path.join(os.path.join(TEST_DIR, ".."), "src/wikiq/__init__.py")
TEST_OUTPUT_DIR: Final[str] = os.path.join(TEST_DIR, "test_output")