Start interoperability between wikidiff2 and deltas

The big challenges here (and remaining) are as follows:

1. Deltas requires changes to be given at the token level,
whereas wikidiff2 reports changes at the byte level. Thus,
it is often required to tokenize sequences of text to convert
to the desired token indices. As-is this is done inefficiently,
often requiring re-tokenization of previously-tokenized sequences.
A better implementation would incrementally tokenize, or
automatically find the referenced sequences.

2. Deltas only allows for Equal/Insert/Delete operations,
while wikidiff2 also detects paragraph moves. These paragraph
moves are NOT equivalent to Equal, as the moved paragraphs
are not guaranteed to be equivalent, just very similar.
Wikidiff2 does not report changes to moved paragraphs, so
to preserve token persistence, a difference algorithm
would need to be performed on the before/after sequences.
A stopgap (currently implemented) is to turn these
into strict deletions/insertions.

3. There appears to be a lot of memory consumption, and
sometimes this results in memory overflow. I am unsure
if this is a memory leak or simply that re-tokenizing
causes significant enough memory throughput that
my machine can't handle it.

4. Deltas expects all tokens in the before/after text to
be covered by segment ranges of Equal/Insert/Delete, but
wikidiff2 does not appear to ever emit any Equal ranges,
instead skipping them. These ranges must be computed
and inserted in sequence. As-is the code does not correctly
handle unchanged text at the end of pages.

Signed-off-by: Will Beason <willbeason@gmail.com>

wikiq

@@ -17,7 +17,6 @@ from hashlib import sha1
 from typing import Any, IO, TextIO, Generator, Union
 
 import mwxml
-import requests
 from mwxml import Dump
 
 from deltas.tokenizers import wikitext_split
@@ -26,6 +25,7 @@ import mwreverts
 
 import tables
 from tables import RevisionTable
+from wiki_diff_matcher import WikiDiffMatcher
 
 TO_ENCODE = ('title', 'editor')
 PERSISTENCE_RADIUS = 7
@@ -43,6 +43,7 @@ class PersistMethod:
     sequence = 1
     segment = 2
     legacy = 3
+    wikidiff = 4
 
 
 def calculate_persistence(tokens_added):
@@ -218,7 +219,7 @@ class WikiqParser:
                  revert_radius: int = 15,
                  output_parquet: bool = True,
                  parquet_buffer_size: int = 2000,
-                 compute_incremental_diffs: bool = False,
+                 wikidiff_url: str = "",
                  ):
 
         """ 
@@ -231,7 +232,7 @@ class WikiqParser:
         self.persist: int = persist
         self.namespaces = []
         self.revert_radius = revert_radius
-        self.compute_incremental_diffs: bool = compute_incremental_diffs
+        self.wikidiff_url: str = wikidiff_url
 
         if namespaces is not None:
             self.namespace_filter = set(namespaces)
@@ -367,9 +368,6 @@ class WikiqParser:
             schema = schema.append(pa.field('tokens_removed', pa.int64(), nullable=True))
             schema = schema.append(pa.field('tokens_window', pa.int64(), nullable=True))
 
-        if self.compute_incremental_diffs:
-            schema = schema.append(pa.field('incremental diffs', pa.string()))
-
         if self.output_parquet:
             writer = pq.ParquetWriter(self.output_file, schema, flavor='spark')
         else:
@@ -379,7 +377,7 @@ class WikiqParser:
 
         # Iterate through pages
         for page in dump:
-            payload = []
+            revision_texts = []
 
             # skip namespaces not in the filter
             if self.namespace_filter is not None:
@@ -419,36 +417,10 @@ class WikiqParser:
                         regex_matches[k] = []
                     regex_matches[k].append(v)
 
-                if self.compute_incremental_diffs:
-                    payload.append(rev.text)
+                revision_texts.append(rev.text)
 
             # Collect the set of pages currently buffered in the table so we can run multi-page functions on them.
             row_buffer = table.pop()
-            if self.compute_incremental_diffs:
-                try:
-                    response = requests.post(DIFFS_URL, json=payload)
-                    response.raise_for_status()
-                    incremental_diffs = response.json()
-                except requests.exceptions.ConnectionError as e:
-                    print(
-                        f"Connection Error: Could not connect to the server at {DIFFS_URL}. Make sure your local server is running.")
-                    print(e)
-                    raise e
-                except requests.exceptions.HTTPError as e:
-                    print(f"HTTP Error: {e}")
-                    print(f"Response Body: {response.text}")
-                    raise e
-                except requests.exceptions.JSONDecodeError as e:
-                    # Must come before RequestException as JSONDecodeError is
-                    # a subclass.
-                    print(f"JSON Decode Error: {e}", file=sys.stderr)
-                    print(f"Response Body: {response.text}", file=sys.stderr)
-                    raise e
-                except requests.exceptions.RequestException as e:
-                    print(f"An unexpected error occurred: {e}")
-                    raise e
-
-                row_buffer['incremental diffs'] = incremental_diffs
 
             is_revert_column: list[Union[bool, None]] = []
             for r, d in zip(row_buffer['reverteds'], row_buffer['deleted']):
@@ -477,6 +449,11 @@ class WikiqParser:
                 elif self.persist == PersistMethod.segment:
                     state = mwpersistence.DiffState(SegmentMatcher(tokenizer=wikitext_split),
                                                     revert_radius=PERSISTENCE_RADIUS)
+                elif self.persist == PersistMethod.wikidiff:
+                    state = mwpersistence.DiffState(WikiDiffMatcher(self.wikidiff_url,
+                                                                    revision_texts,
+                                                                    tokenizer=wikitext_split),
+                                                    revert_radius=PERSISTENCE_RADIUS)
                 else:
                     from mw.lib import persistence
                     state = persistence.State()
@@ -614,20 +591,22 @@ def main():
                         action='store_true',
                         help="Whether the archive is from the fandom 2020 dumps by Wikiteam. These dumps can have multiple .xml files in their archives.")
 
-    parser.add_argument('--compute-incremental-diffs', dest="compute_incremental_diffs",
-                        action='store_true',
-                        help="Compute and store incremental diffs by edit session.")
+    parser.add_argument('--wikidiff-url', dest="wikidiff_url",
+                        action='store',
+                        help="The URL to a server running WikiDiff2.")
 
     args = parser.parse_args()
 
     # set persistence method
 
-    if args.persist is None:
+    if args.persist is None and not args.wikidiff_url:
         persist = PersistMethod.none
     elif args.persist == "segment":
         persist = PersistMethod.segment
     elif args.persist == "legacy":
         persist = PersistMethod.legacy
+    elif args.wikidiff_url:
+        persist = PersistMethod.wikidiff
     else:
         persist = PersistMethod.sequence
 
@@ -670,7 +649,7 @@ def main():
                                 regex_match_comment=args.regex_match_comment,
                                 regex_comment_label=args.regex_comment_label,
                                 output_parquet=output_parquet,
-                                compute_incremental_diffs=args.compute_incremental_diffs,
+                                wikidiff_url=args.wikidiff_url,
                                 )
 
             wikiq.process()