make pywikidiff2 an optional dependency

pywikidiff2 was imported at module scope, so every invocation of wikiq
required it -- including `wikiq --help`. It is not on PyPI, compiles a
C++ extension, and needs libthai, which made a compiler a hard
requirement for installing a tool that mostly does not need one.

Only --diff and -p wikidiff2 actually use it. Import it inside those two
code paths instead, and report what to install when it is missing rather
than failing with an ImportError traceback. The check also runs once at
startup, since both use sites sit deep in the per-revision loop and a run
can stream for hours before reaching them.

Dropping the dependency also removes the PEP 508 direct reference from
the package metadata, and with it the need for hatchling's
allow-direct-references. PyPI rejects uploads whose metadata contains a
direct URL, so this is a prerequisite for publishing wikiq there.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
(cherry picked from commit 12325afd46670e949abf1c0e14799e212d7ff6ce)

pyproject.toml

@@ -14,7 +14,6 @@ dependencies = [
     "mwtypes>=0.4.0",
     "mwxml>=0.3.6",
     "pyarrow>=20.0.0",
-    "pywikidiff2",
     "sortedcontainers>=2.4.0",
     "yamlconf>=0.2.6",
 ]
@@ -34,7 +33,6 @@ packages = ["src/wikiq"]
 yamlconf = { git = "https://github.com/groceryheist/yamlconf" }
 mwxml = { git = "https://github.com/groceryheist/python-mwxml" }
 deltas = { git = "https://github.com/groceryheist/deltas" }
-pywikidiff2 = { git = "ssh://gitea@gitea.communitydata.science:2200/groceryheist/pywikidiff2.git"}
 
 [dependency-groups]
 dev = [

src/wikiq/__init__.py

@@ -22,13 +22,11 @@ from typing import IO, Any, Generator, TextIO, Union
 import mwpersistence
 import mwreverts
 import mwxml
-import pywikidiff2
 from deltas.tokenizers import wikitext_split
 from more_itertools import peekable
 from mwxml import Dump
 import wikiq.tables as tables
 from wikiq.tables import RevisionTable
-from wikiq.wiki_diff_matcher import WikiDiffMatcher
 from wikiq.wikitext_parser import WikitextParser
 from wikiq.resume import get_resume_point
 
@@ -42,6 +40,33 @@ import pyarrow.csv as pacsv
 from deltas import SegmentMatcher, SequenceMatcher
 
 
+# Some dependencies serve a single feature and are awkward enough to install
+# that wikiq leaves them out of the base install. Each is imported inside the
+# code path that needs it, so importing wikiq works without them and a plain
+# `pip install` needs no C++ compiler. These helpers do that import and, when
+# it fails, explain what to install rather than raising an ImportError.
+
+def require_pywikidiff2(feature: str):
+    """Import and return pywikidiff2, or exit explaining how to install it.
+
+    feature names the wikiq option that needs it, so the message points at
+    whatever the user actually asked for.
+    """
+    try:
+        import pywikidiff2
+    except ImportError:
+        raise SystemExit(
+            f"{feature} requires pywikidiff2, which wikiq does not install by "
+            "default because it compiles a C++ extension.\n"
+            "Install it with:\n"
+            "    pip install 'pywikidiff2 @ git+"
+            "https://gitea.communitydata.science/groceryheist/pywikidiff2.git'\n"
+            "A C++ compiler and libthai must be available. Other persistence "
+            "methods (-p sequence, -p segment, -p legacy) do not need it."
+        )
+    return pywikidiff2
+
+
 def pyarrow_type_to_spark(pa_type):
     """Convert a PyArrow type to Spark JSON schema format."""
     if pa.types.is_int64(pa_type):
@@ -725,6 +750,7 @@ class WikiqParser:
         differ = None
         fast_differ = None
         if self.diff:
+            pywikidiff2 = require_pywikidiff2("--diff")
             differ = pywikidiff2.pywikidiff2(
                 num_context_lines=1000000,
                 max_word_level_diff_complexity=-1,
@@ -812,6 +838,8 @@ class WikiqParser:
                         revert_radius=PERSISTENCE_RADIUS,
                     )
                 elif self.persist == PersistMethod.wikidiff2:
+                    require_pywikidiff2("-p wikidiff2")
+                    from wikiq.wiki_diff_matcher import WikiDiffMatcher
                     wikidiff_matcher = WikiDiffMatcher(tokenizer=wikitext_split)
                     persist_state = mwpersistence.DiffState(
                         wikidiff_matcher, revert_radius=PERSISTENCE_RADIUS
@@ -1264,6 +1292,13 @@ def main():
     else:
         persist = PersistMethod.sequence
 
+    # Check for the optional dependencies up front. Both are used deep in the
+    # per-revision loop, and a run can stream for hours before reaching them.
+    if args.diff:
+        require_pywikidiff2("--diff")
+    if persist == PersistMethod.wikidiff2:
+        require_pywikidiff2("-p wikidiff2")
+
     if args.namespace_filter is not None:
         namespaces = args.namespace_filter
     else:

src/wikiq/wiki_diff_matcher.py

@@ -10,7 +10,6 @@ from mwpersistence import Token
 from sortedcontainers import SortedDict
 
 TOKENIZER = tokenizers.wikitext_split
-import pywikidiff2
 
 
 class DiffToOperationMap:
@@ -332,6 +331,11 @@ class WikiDiffMatcher:
 
     class Processor(DiffEngine.Processor):
         def __init__(self, tokenizer=None):
+            # imported here rather than at module scope so that importing
+            # wikiq does not require the pywikidiff2 C++ extension
+            from wikiq import require_pywikidiff2
+
+            pywikidiff2 = require_pywikidiff2("-p wikidiff2")
             self.tokenizer = tokenizer or TOKENIZER
             self.last_tokens = []
             self.previous_text = ""