diff --git a/pyproject.toml b/pyproject.toml index bc3e709..c30bcae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,6 @@ dependencies = [ "mwtypes>=0.4.0", "mwxml>=0.3.6", "pyarrow>=20.0.0", - "pywikidiff2", "sortedcontainers>=2.4.0", "yamlconf>=0.2.6", ] @@ -34,7 +33,6 @@ packages = ["src/wikiq"] yamlconf = { git = "https://github.com/groceryheist/yamlconf" } mwxml = { git = "https://github.com/groceryheist/python-mwxml" } deltas = { git = "https://github.com/groceryheist/deltas" } -pywikidiff2 = { git = "ssh://gitea@gitea.communitydata.science:2200/groceryheist/pywikidiff2.git"} [dependency-groups] dev = [ diff --git a/src/wikiq/__init__.py b/src/wikiq/__init__.py index a9215f2..0cd9dcc 100755 --- a/src/wikiq/__init__.py +++ b/src/wikiq/__init__.py @@ -22,13 +22,11 @@ from typing import IO, Any, Generator, TextIO, Union import mwpersistence import mwreverts import mwxml -import pywikidiff2 from deltas.tokenizers import wikitext_split from more_itertools import peekable from mwxml import Dump import wikiq.tables as tables from wikiq.tables import RevisionTable -from wikiq.wiki_diff_matcher import WikiDiffMatcher from wikiq.wikitext_parser import WikitextParser from wikiq.resume import get_resume_point @@ -42,6 +40,33 @@ import pyarrow.csv as pacsv from deltas import SegmentMatcher, SequenceMatcher +# Some dependencies serve a single feature and are awkward enough to install +# that wikiq leaves them out of the base install. Each is imported inside the +# code path that needs it, so importing wikiq works without them and a plain +# `pip install` needs no C++ compiler. These helpers do that import and, when +# it fails, explain what to install rather than raising an ImportError. + +def require_pywikidiff2(feature: str): + """Import and return pywikidiff2, or exit explaining how to install it. + + feature names the wikiq option that needs it, so the message points at + whatever the user actually asked for. + """ + try: + import pywikidiff2 + except ImportError: + raise SystemExit( + f"{feature} requires pywikidiff2, which wikiq does not install by " + "default because it compiles a C++ extension.\n" + "Install it with:\n" + " pip install 'pywikidiff2 @ git+" + "https://gitea.communitydata.science/groceryheist/pywikidiff2.git'\n" + "A C++ compiler and libthai must be available. Other persistence " + "methods (-p sequence, -p segment, -p legacy) do not need it." + ) + return pywikidiff2 + + def pyarrow_type_to_spark(pa_type): """Convert a PyArrow type to Spark JSON schema format.""" if pa.types.is_int64(pa_type): @@ -725,6 +750,7 @@ class WikiqParser: differ = None fast_differ = None if self.diff: + pywikidiff2 = require_pywikidiff2("--diff") differ = pywikidiff2.pywikidiff2( num_context_lines=1000000, max_word_level_diff_complexity=-1, @@ -812,6 +838,8 @@ class WikiqParser: revert_radius=PERSISTENCE_RADIUS, ) elif self.persist == PersistMethod.wikidiff2: + require_pywikidiff2("-p wikidiff2") + from wikiq.wiki_diff_matcher import WikiDiffMatcher wikidiff_matcher = WikiDiffMatcher(tokenizer=wikitext_split) persist_state = mwpersistence.DiffState( wikidiff_matcher, revert_radius=PERSISTENCE_RADIUS @@ -1264,6 +1292,13 @@ def main(): else: persist = PersistMethod.sequence + # Check for the optional dependencies up front. Both are used deep in the + # per-revision loop, and a run can stream for hours before reaching them. + if args.diff: + require_pywikidiff2("--diff") + if persist == PersistMethod.wikidiff2: + require_pywikidiff2("-p wikidiff2") + if args.namespace_filter is not None: namespaces = args.namespace_filter else: diff --git a/src/wikiq/wiki_diff_matcher.py b/src/wikiq/wiki_diff_matcher.py index a4fdc0a..c39cdf6 100644 --- a/src/wikiq/wiki_diff_matcher.py +++ b/src/wikiq/wiki_diff_matcher.py @@ -10,7 +10,6 @@ from mwpersistence import Token from sortedcontainers import SortedDict TOKENIZER = tokenizers.wikitext_split -import pywikidiff2 class DiffToOperationMap: @@ -332,6 +331,11 @@ class WikiDiffMatcher: class Processor(DiffEngine.Processor): def __init__(self, tokenizer=None): + # imported here rather than at module scope so that importing + # wikiq does not require the pywikidiff2 C++ extension + from wikiq import require_pywikidiff2 + + pywikidiff2 = require_pywikidiff2("-p wikidiff2") self.tokenizer = tokenizer or TOKENIZER self.last_tokens = [] self.previous_text = ""