pyproject.toml

@@ -14,7 +14,6 @@ dependencies = [
     "mwtypes>=0.4.0",
     "mwxml>=0.3.6",
     "pyarrow>=20.0.0",
-    "pywikidiff2",
     "sortedcontainers>=2.4.0",
     "yamlconf>=0.2.6",
 ]
@@ -34,7 +33,6 @@ packages = ["src/wikiq"]
 yamlconf = { git = "https://github.com/groceryheist/yamlconf" }
 mwxml = { git = "https://github.com/groceryheist/python-mwxml" }
 deltas = { git = "https://github.com/groceryheist/deltas" }
-pywikidiff2 = { git = "ssh://gitea@gitea.communitydata.science:2200/groceryheist/pywikidiff2.git"}
 
 [dependency-groups]
 dev = [

src/wikiq/__init__.py

@@ -22,13 +22,11 @@ from typing import IO, Any, Generator, TextIO, Union
 import mwpersistence
 import mwreverts
 import mwxml
-import pywikidiff2
 from deltas.tokenizers import wikitext_split
 from more_itertools import peekable
 from mwxml import Dump
 import wikiq.tables as tables
 from wikiq.tables import RevisionTable
-from wikiq.wiki_diff_matcher import WikiDiffMatcher
 from wikiq.wikitext_parser import WikitextParser
 from wikiq.resume import get_resume_point
 
@@ -42,6 +40,33 @@ import pyarrow.csv as pacsv
 from deltas import SegmentMatcher, SequenceMatcher
 
 
+# Some dependencies serve a single feature and are awkward enough to install
+# that wikiq leaves them out of the base install. Each is imported inside the
+# code path that needs it, so importing wikiq works without them and a plain
+# `pip install` needs no C++ compiler. These helpers do that import and, when
+# it fails, explain what to install rather than raising an ImportError.
+
+def require_pywikidiff2(feature: str):
+    """Import and return pywikidiff2, or exit explaining how to install it.
+
+    feature names the wikiq option that needs it, so the message points at
+    whatever the user actually asked for.
+    """
+    try:
+        import pywikidiff2
+    except ImportError:
+        raise SystemExit(
+            f"{feature} requires pywikidiff2, which wikiq does not install by "
+            "default because it compiles a C++ extension.\n"
+            "Install it with:\n"
+            "    pip install 'pywikidiff2 @ git+"
+            "https://gitea.communitydata.science/groceryheist/pywikidiff2.git'\n"
+            "A C++ compiler and libthai must be available. Other persistence "
+            "methods (-p sequence, -p segment, -p legacy) do not need it."
+        )
+    return pywikidiff2
+
+
 def pyarrow_type_to_spark(pa_type):
     """Convert a PyArrow type to Spark JSON schema format."""
     if pa.types.is_int64(pa_type):
@@ -725,6 +750,7 @@ class WikiqParser:
         differ = None
         fast_differ = None
         if self.diff:
+            pywikidiff2 = require_pywikidiff2("--diff")
             differ = pywikidiff2.pywikidiff2(
                 num_context_lines=1000000,
                 max_word_level_diff_complexity=-1,
@@ -812,6 +838,8 @@ class WikiqParser:
                         revert_radius=PERSISTENCE_RADIUS,
                     )
                 elif self.persist == PersistMethod.wikidiff2:
+                    require_pywikidiff2("-p wikidiff2")
+                    from wikiq.wiki_diff_matcher import WikiDiffMatcher
                     wikidiff_matcher = WikiDiffMatcher(tokenizer=wikitext_split)
                     persist_state = mwpersistence.DiffState(
                         wikidiff_matcher, revert_radius=PERSISTENCE_RADIUS
@@ -1264,6 +1292,13 @@ def main():
     else:
         persist = PersistMethod.sequence
 
+    # Check for the optional dependencies up front. Both are used deep in the
+    # per-revision loop, and a run can stream for hours before reaching them.
+    if args.diff:
+        require_pywikidiff2("--diff")
+    if persist == PersistMethod.wikidiff2:
+        require_pywikidiff2("-p wikidiff2")
+
     if args.namespace_filter is not None:
         namespaces = args.namespace_filter
     else:

src/wikiq/wiki_diff_matcher.py

@@ -10,7 +10,6 @@ from mwpersistence import Token
 from sortedcontainers import SortedDict
 
 TOKENIZER = tokenizers.wikitext_split
-import pywikidiff2
 
 
 class DiffToOperationMap:
@@ -332,6 +331,11 @@ class WikiDiffMatcher:
 
     class Processor(DiffEngine.Processor):
         def __init__(self, tokenizer=None):
+            # imported here rather than at module scope so that importing
+            # wikiq does not require the pywikidiff2 C++ extension
+            from wikiq import require_pywikidiff2
+
+            pywikidiff2 = require_pywikidiff2("-p wikidiff2")
             self.tokenizer = tokenizer or TOKENIZER
             self.last_tokens = []
             self.previous_text = ""