wikidiff2 integration: pwr complete.

test for pwr based on wikidiff2.

wiki_diff_matcher.py

@@ -6,11 +6,13 @@ from typing import Dict, Generator, List, Optional, Tuple
 
 from deltas import (Delete, DiffEngine, Equal, Insert, Operation,
                     RegexTokenizer, Token, tokenizers)
+from mwpersistence import Token
 from sortedcontainers import SortedDict
 
 TOKENIZER = tokenizers.wikitext_split
 import pywikidiff2
 
+
 class DiffToOperationMap:
     def __init__(self, diff, tokenizer):
         self.tokenizer = tokenizer
@@ -23,7 +25,7 @@ class DiffToOperationMap:
         self.from_linenumber_bytes_map: SortedDict[int, int] = SortedDict()
 
     def tokenize(self, bytes):
-        return self.tokenizer.tokenize(bytes.decode("utf-8"))
+        return self.tokenizer.tokenize(bytes.decode("utf-8"), token_class=Token)
 
     def to_operations(self):
         for entry in self.diff["diff"]:
@@ -148,8 +150,8 @@ class DiffToOperationMap:
                 {
                     "text": to_diff["text"],
                     "highlightRanges": to_diff["highlightRanges"],
-                    'offset': offset,
-                    'lineNumber': to_diff["lineNumber"],
+                    "offset": offset,
+                    "lineNumber": to_diff["lineNumber"],
                 }
             )
             diffops = [
@@ -247,7 +249,9 @@ class DiffToOperationMap:
             raise ValueError(delete_segment)
         tokens = self.tokenize(delete_bytes)
         if lineNumber is not None:
-            self.from_linenumber_bytes_map[lineNumber] = offset["from"] + len(delete_bytes)
+            self.from_linenumber_bytes_map[lineNumber] = offset["from"] + len(
+                delete_bytes
+            )
 
         yield (
             Delete(offset["from"], None, None, None),
@@ -355,8 +359,8 @@ class DiffToOperationMap:
             {
                 "text": to_diff["text"],
                 "highlightRanges": to_diff["highlightRanges"],
-                'offset': offset,
-                'lineNumber': to_diff["lineNumber"],
+                "offset": offset,
+                "lineNumber": to_diff["lineNumber"],
             }
         )
 
@@ -367,11 +371,11 @@ class WikiDiffMatcher:
         texts: list[str] = None,
         tokenizer: Optional[RegexTokenizer] = None,
     ):
-        differ = pywikidiff2.pywikidiff2(numContextLines=1000000,
-                                         moved_paragraph_detection_cutoff=200000)
+        differ = pywikidiff2.pywikidiff2(
+            numContextLines=1000000, moved_paragraph_detection_cutoff=200000
+        )
         # Pre-compute diffs to reduce traffic overhead.
         self.diffs = differ.inline_json_diff_sequence(list(texts))
-
         self.tokenizer = tokenizer or TOKENIZER
 
     class Processor(DiffEngine.Processor):
@@ -394,7 +398,7 @@ class WikiDiffMatcher:
 
             # this happens when revisions are actually equal.
             if len(diffops) == 0:
-                self.last_tokens = self.tokenizer.tokenize(text)
+                self.last_tokens = self.tokenizer.tokenize(text, token_class=Token)
                 ops = [Equal(0, len(self.last_tokens), 0, len(self.last_tokens))]
                 return ops, self.last_tokens, self.last_tokens