Prepare wikiq for release: fixes, dependency cleanup, and packaging #2
@@ -23,6 +23,8 @@ from wikiq_test_utils import (
|
|||||||
TWINPEAKS,
|
TWINPEAKS,
|
||||||
WIKIQ,
|
WIKIQ,
|
||||||
WikiqTester,
|
WikiqTester,
|
||||||
|
requires_mediawiki_utilities,
|
||||||
|
requires_pywikidiff2,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -205,6 +207,7 @@ def test_collapse_user():
|
|||||||
baseline = pd.read_table(tester.baseline_file)
|
baseline = pd.read_table(tester.baseline_file)
|
||||||
assert_frame_equal(test, baseline, check_like=True)
|
assert_frame_equal(test, baseline, check_like=True)
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_pwr_wikidiff2():
|
def test_pwr_wikidiff2():
|
||||||
tester = WikiqTester(SAILORMOON, "persistence_wikidiff2", in_compression="7z")
|
tester = WikiqTester(SAILORMOON, "persistence_wikidiff2", in_compression="7z")
|
||||||
|
|
||||||
@@ -229,6 +232,7 @@ def test_pwr_segment():
|
|||||||
baseline = pd.read_table(tester.baseline_file)
|
baseline = pd.read_table(tester.baseline_file)
|
||||||
assert_frame_equal(test, baseline, check_like=True)
|
assert_frame_equal(test, baseline, check_like=True)
|
||||||
|
|
||||||
|
@requires_mediawiki_utilities
|
||||||
def test_pwr_legacy():
|
def test_pwr_legacy():
|
||||||
tester = WikiqTester(SAILORMOON, "persistence_legacy", in_compression="7z")
|
tester = WikiqTester(SAILORMOON, "persistence_legacy", in_compression="7z")
|
||||||
|
|
||||||
@@ -255,6 +259,7 @@ def test_pwr():
|
|||||||
test = test.reindex(columns=sorted(test.columns))
|
test = test.reindex(columns=sorted(test.columns))
|
||||||
assert_frame_equal(test, baseline, check_like=True)
|
assert_frame_equal(test, baseline, check_like=True)
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_diff():
|
def test_diff():
|
||||||
tester = WikiqTester(SAILORMOON, "diff", in_compression="7z", out_format='jsonl')
|
tester = WikiqTester(SAILORMOON, "diff", in_compression="7z", out_format='jsonl')
|
||||||
|
|
||||||
@@ -268,6 +273,7 @@ def test_diff():
|
|||||||
assert "diff_timeout" in test.columns, "diff_timeout column should exist"
|
assert "diff_timeout" in test.columns, "diff_timeout column should exist"
|
||||||
assert len(test) > 0, "Should have output rows"
|
assert len(test) > 0, "Should have output rows"
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_diff_plus_pwr():
|
def test_diff_plus_pwr():
|
||||||
tester = WikiqTester(SAILORMOON, "diff_pwr", in_compression="7z", out_format='jsonl')
|
tester = WikiqTester(SAILORMOON, "diff_pwr", in_compression="7z", out_format='jsonl')
|
||||||
|
|
||||||
@@ -281,6 +287,7 @@ def test_diff_plus_pwr():
|
|||||||
assert "token_revs" in test.columns, "token_revs column should exist"
|
assert "token_revs" in test.columns, "token_revs column should exist"
|
||||||
assert len(test) > 0, "Should have output rows"
|
assert len(test) > 0, "Should have output rows"
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_text():
|
def test_text():
|
||||||
tester = WikiqTester(SAILORMOON, "text", in_compression="7z", out_format='jsonl')
|
tester = WikiqTester(SAILORMOON, "text", in_compression="7z", out_format='jsonl')
|
||||||
|
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from wikiq_test_utils import (
|
|||||||
TEST_OUTPUT_DIR,
|
TEST_OUTPUT_DIR,
|
||||||
WIKIQ,
|
WIKIQ,
|
||||||
WikiqTester,
|
WikiqTester,
|
||||||
|
requires_pywikidiff2,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -65,6 +66,7 @@ def test_resume():
|
|||||||
assert_frame_equal(df_full, df_resumed)
|
assert_frame_equal(df_full, df_resumed)
|
||||||
|
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_resume_with_diff():
|
def test_resume_with_diff():
|
||||||
"""Test that --resume correctly computes diff values after resume.
|
"""Test that --resume correctly computes diff values after resume.
|
||||||
|
|
||||||
@@ -305,6 +307,7 @@ def test_resume_data_equivalence():
|
|||||||
assert_frame_equal(df_full, df_resumed)
|
assert_frame_equal(df_full, df_resumed)
|
||||||
|
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_resume_with_persistence():
|
def test_resume_with_persistence():
|
||||||
"""Test that --resume correctly handles persistence state after resume.
|
"""Test that --resume correctly handles persistence state after resume.
|
||||||
|
|
||||||
@@ -403,6 +406,7 @@ def test_resume_corrupted_jsonl_last_line():
|
|||||||
assert_frame_equal(df_full, df_resumed)
|
assert_frame_equal(df_full, df_resumed)
|
||||||
|
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_resume_diff_persistence_combined():
|
def test_resume_diff_persistence_combined():
|
||||||
"""Test that --resume correctly handles both diff and persistence state together.
|
"""Test that --resume correctly handles both diff and persistence state together.
|
||||||
|
|
||||||
@@ -448,6 +452,7 @@ def test_resume_diff_persistence_combined():
|
|||||||
assert_frame_equal(df_full, df_resumed)
|
assert_frame_equal(df_full, df_resumed)
|
||||||
|
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_resume_mid_page():
|
def test_resume_mid_page():
|
||||||
"""Test resume from the middle of a page with many revisions.
|
"""Test resume from the middle of a page with many revisions.
|
||||||
|
|
||||||
@@ -501,6 +506,7 @@ def test_resume_mid_page():
|
|||||||
assert_frame_equal(df_full, df_resumed)
|
assert_frame_equal(df_full, df_resumed)
|
||||||
|
|
||||||
|
|
||||||
|
@requires_pywikidiff2
|
||||||
def test_resume_page_boundary():
|
def test_resume_page_boundary():
|
||||||
"""Test resume at the exact start of a new page.
|
"""Test resume at the exact start of a new page.
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ from typing import List
|
|||||||
from deltas import Delete, Equal, Insert, wikitext_split
|
from deltas import Delete, Equal, Insert, wikitext_split
|
||||||
from mwpersistence import Token
|
from mwpersistence import Token
|
||||||
from wikiq.wiki_diff_matcher import WikiDiffMatcher
|
from wikiq.wiki_diff_matcher import WikiDiffMatcher
|
||||||
|
from wikiq_test_utils import requires_pywikidiff2
|
||||||
|
|
||||||
|
# every test here drives wikidiff2 directly
|
||||||
|
pytestmark = requires_pywikidiff2
|
||||||
|
|
||||||
def _replace_whitespace(match):
|
def _replace_whitespace(match):
|
||||||
if match.group(1): # If spaces matched (e.g., ' ')
|
if match.group(1): # If spaces matched (e.g., ' ')
|
||||||
|
|||||||
@@ -1,8 +1,27 @@
|
|||||||
|
import importlib.util
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
from typing import Final, Union
|
from typing import Final, Union
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
def _installed(module: str) -> bool:
|
||||||
|
return importlib.util.find_spec(module) is not None
|
||||||
|
|
||||||
|
|
||||||
|
# wikiq's two optional dependencies. Tests exercising --diff, -p wikidiff2, or
|
||||||
|
# -p legacy cannot run on a base install, so they skip rather than fail.
|
||||||
|
requires_pywikidiff2 = pytest.mark.skipif(
|
||||||
|
not _installed("pywikidiff2"),
|
||||||
|
reason="needs the optional pywikidiff2 extension (--diff, -p wikidiff2)",
|
||||||
|
)
|
||||||
|
requires_mediawiki_utilities = pytest.mark.skipif(
|
||||||
|
not _installed("mw"),
|
||||||
|
reason="needs the optional mediawiki-utilities package (-p legacy)",
|
||||||
|
)
|
||||||
|
|
||||||
TEST_DIR: Final[str] = os.path.dirname(os.path.realpath(__file__))
|
TEST_DIR: Final[str] = os.path.dirname(os.path.realpath(__file__))
|
||||||
WIKIQ: Final[str] = os.path.join(os.path.join(TEST_DIR, ".."), "src/wikiq/__init__.py")
|
WIKIQ: Final[str] = os.path.join(os.path.join(TEST_DIR, ".."), "src/wikiq/__init__.py")
|
||||||
TEST_OUTPUT_DIR: Final[str] = os.path.join(TEST_DIR, "test_output")
|
TEST_OUTPUT_DIR: Final[str] = os.path.join(TEST_DIR, "test_output")
|
||||||
|
|||||||
Reference in New Issue
Block a user