Benjamin Mako Hill
90a14cf999
Add the full license text as COPYING (the same text Debian ships in common-licenses), declare the license and license file in pyproject.toml, add copyright and permission notices to the source files, and describe the license in the README. Also replace the placeholder package description in pyproject.toml. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
429 lines
16 KiB
Python
429 lines
16 KiB
Python
# Copyright (C) 2015-2026 Benjamin Mako Hill, Nathan TeBlunthuis, Will
|
|
# Beason, Sohyeon Hwang, Kaylea Champion, and other contributors
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify it
|
|
# under the terms of the GNU General Public License as published by the
|
|
# Free Software Foundation, either version 3 of the License, or (at your
|
|
# option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License along
|
|
# with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
|
|
import json
|
|
import sys
|
|
from collections import namedtuple
|
|
from itertools import chain
|
|
from typing import Dict, Generator, List, Optional, Tuple
|
|
|
|
from deltas import (Delete, DiffEngine, Equal, Insert, Operation,
|
|
RegexTokenizer, Token, tokenizers)
|
|
from mwpersistence import Token
|
|
from sortedcontainers import SortedDict
|
|
|
|
TOKENIZER = tokenizers.wikitext_split
|
|
import pywikidiff2
|
|
|
|
|
|
class DiffToOperationMap:
|
|
def __init__(self, diff, tokenizer):
|
|
self.tokenizer = tokenizer
|
|
self.from_par_move_dict = {}
|
|
self.to_par_move_dict = {}
|
|
self.highlights_without_offset = []
|
|
self.diff = diff
|
|
# we need to keep track of the bytes of line numbers to recover when wikidiff2 loses offsets.
|
|
self.to_linenumber_bytes_map: SortedDict[int, int] = SortedDict()
|
|
self.from_linenumber_bytes_map: SortedDict[int, int] = SortedDict()
|
|
|
|
def tokenize(self, bytes):
|
|
return self.tokenizer.tokenize(bytes.decode("utf-8"), token_class=Token)
|
|
|
|
def to_operations(self):
|
|
for entry in self.diff["diff"]:
|
|
# add back the newline
|
|
entry["text"] += "\n"
|
|
text = entry["text"]
|
|
offset = entry["offset"]
|
|
# this is the first byte of the line in the 'from' revision.
|
|
from_start_line = entry["offset"]["from"]
|
|
# this is the first byte of the line in the 'to' revision.
|
|
to_start_line = entry["offset"]["to"]
|
|
|
|
if entry["type"] == 0:
|
|
yield from self.doEqual(entry)
|
|
|
|
# a line included in the 'to' revision, but not in the 'from' revision
|
|
elif entry["type"] == 1:
|
|
yield from self.doInsert(entry)
|
|
|
|
# a line included in the 'from' revision, but not in the 'to' revision
|
|
elif entry["type"] == 2:
|
|
yield from self.doDelete(entry)
|
|
|
|
elif entry["type"] == 3:
|
|
# sometimes, for some reason we don't have a 'to' index here. we'll save these for later
|
|
if entry["offset"]["to"] is None:
|
|
self.highlights_without_offset.append(entry)
|
|
else:
|
|
yield from self.doHighlightRange(entry)
|
|
|
|
elif entry["type"] == 4:
|
|
linkId = entry["moveInfo"]["linkId"]
|
|
|
|
if linkId in self.to_par_move_dict:
|
|
yield from self.doParMove(entry, self.to_par_move_dict.pop(linkId))
|
|
else:
|
|
self.from_par_move_dict[entry["moveInfo"]["id"]] = entry
|
|
|
|
elif entry["type"] == 5:
|
|
linkId = entry["moveInfo"]["linkId"]
|
|
if linkId in self.from_par_move_dict:
|
|
yield from self.doParMove(
|
|
self.from_par_move_dict.pop(linkId), entry
|
|
)
|
|
else:
|
|
self.to_par_move_dict[entry["moveInfo"]["id"]] = entry
|
|
else:
|
|
# The 'type' isn't one of the known
|
|
raise ValueError(d)
|
|
|
|
# now we should be able to apply highlights
|
|
|
|
for entry in self.highlights_without_offset:
|
|
yield from self.doHighlightRange(entry)
|
|
|
|
if len(self.from_par_move_dict) > 0 or len(self.to_par_move_dict) > 0:
|
|
print("PROBLEM! Unmatched parmoves!")
|
|
print(self.from_par_move_dict)
|
|
print(self.to_par_move_dict)
|
|
# We can try to match them:
|
|
for lkey in self.from_par_move_dict.keys():
|
|
for rkey in self.to_par_move_dict.keys():
|
|
from_diff = self.from_par_move_dict[lkey]
|
|
to_diff = self.to_par_move_dict[rkey]
|
|
if self.match_parmoves_exact(from_diff, to_diff):
|
|
yield from self.doParMove(from_diff, to_diff)
|
|
del self.to_par_move_dict[lkey]
|
|
del self.from_par_move_dict[rkey]
|
|
break
|
|
|
|
# we couldn't find matches. treat type 4 as removal and type 5 as highlight.
|
|
for from_diff in self.from_par_move_dict.values():
|
|
yield from self.doDelete(from_diff)
|
|
|
|
# only we don't know the from index; we assume its already handled.
|
|
for to_diff in self.to_par_move_dict.values():
|
|
offset["from"] = 0
|
|
offset["to"] = None
|
|
diffops = self.doHighlightRange(
|
|
{
|
|
"text": to_diff["text"],
|
|
"highlightRanges": to_diff["highlightRanges"],
|
|
"offset": offset,
|
|
"lineNumber": to_diff["lineNumber"],
|
|
}
|
|
)
|
|
diffops = [
|
|
(type(op)(None, None, op.b1, op.b2), [], bseq)
|
|
for op, _, bseq in diffops
|
|
if isinstance(op, Insert) or isinstance(op, Equal)
|
|
]
|
|
yield from diffops
|
|
|
|
def match_parmoves_exact(self, from_diff, to_diff):
|
|
ops, from_tokens, to_tokens = list(zip(*self.doParMove(from_diff, to_diff)))
|
|
from_text = "".join(chain.from_iterable(from_tokens))
|
|
# we know they match if we apply the highlight ranges and the "from" tokens equal the lhs tokens.
|
|
if from_text == from_diff["text"]:
|
|
print("MATCH FOUND")
|
|
return True
|
|
else:
|
|
print("NO MATCH")
|
|
print(len(from_text))
|
|
print(len(from_diff["text"]))
|
|
return False
|
|
|
|
# mwpersistence expects differences to be represented in order from the
|
|
# result's perspective ("to"), not the previous text. Thus, if a line
|
|
# is moved earlier then its insertion should appear before its deletion.
|
|
# As a rule of thumb, the "to" segments should be non-overlapping and
|
|
# strictly increasing, while the "from" segments should merely be
|
|
# non-overlapping.
|
|
|
|
def doEqual(self, entry):
|
|
equal_segment, offset, lineNumber = (
|
|
entry["text"],
|
|
entry["offset"],
|
|
entry["lineNumber"],
|
|
)
|
|
if isinstance(equal_segment, str):
|
|
equal_bytes = equal_segment.encode()
|
|
elif isinstance(equal_segment, bytes):
|
|
equal_bytes = equal_segment
|
|
else:
|
|
raise ValueError(equal_segment)
|
|
|
|
self.from_linenumber_bytes_map[lineNumber] = offset["from"] + len(equal_bytes)
|
|
self.to_linenumber_bytes_map[lineNumber] = offset["to"] + len(equal_bytes)
|
|
|
|
tokens = self.tokenize(equal_bytes)
|
|
n_tokens = len(tokens)
|
|
yield (
|
|
Equal(
|
|
offset["from"],
|
|
None,
|
|
offset["to"],
|
|
None,
|
|
),
|
|
tokens,
|
|
tokens,
|
|
)
|
|
|
|
def doInsert(self, entry):
|
|
insert_segment, offset, lineNumber = (
|
|
entry["text"],
|
|
entry["offset"],
|
|
entry["lineNumber"],
|
|
)
|
|
if isinstance(insert_segment, str):
|
|
insert_bytes = insert_segment.encode()
|
|
elif isinstance(insert_segment, bytes):
|
|
insert_bytes = insert_segment
|
|
else:
|
|
raise ValueError(insert_segment)
|
|
tokens = self.tokenize(insert_bytes)
|
|
self.to_linenumber_bytes_map[lineNumber] = offset["to"] + len(insert_bytes)
|
|
yield (
|
|
Insert(
|
|
None,
|
|
None,
|
|
offset["to"],
|
|
None,
|
|
),
|
|
[],
|
|
tokens,
|
|
)
|
|
|
|
def doDelete(self, entry):
|
|
delete_segment, offset, lineNumber = (
|
|
entry["text"],
|
|
entry["offset"],
|
|
entry.get("lineNumber", None),
|
|
)
|
|
if isinstance(delete_segment, str):
|
|
delete_bytes = delete_segment.encode()
|
|
elif isinstance(delete_segment, bytes):
|
|
delete_bytes = delete_segment
|
|
else:
|
|
raise ValueError(delete_segment)
|
|
tokens = self.tokenize(delete_bytes)
|
|
if lineNumber is not None:
|
|
self.from_linenumber_bytes_map[lineNumber] = offset["from"] + len(
|
|
delete_bytes
|
|
)
|
|
|
|
yield (
|
|
Delete(offset["from"], None, None, None),
|
|
tokens,
|
|
[],
|
|
)
|
|
|
|
def doHighlightRange(self, entry):
|
|
highlight_text, highlightRanges, offset, lineNumber = (
|
|
entry["text"],
|
|
entry["highlightRanges"],
|
|
entry["offset"],
|
|
entry["lineNumber"],
|
|
)
|
|
|
|
# The text field is an overlapping mix of both the from and to,
|
|
# so we need to handle it highlight-by-highlight.
|
|
# there can be gaps between highlight segments.
|
|
# for instance, if a word is deleted from the middle of a line.
|
|
# we need to track that.
|
|
highlight_bytes = highlight_text.encode()
|
|
highlight_end = 0
|
|
|
|
# it's possible for offset['to'] to be null.
|
|
# we can get it from the line number?
|
|
# this bit is a little hacky as it deals with ideosyncratic wikidiff2 behavior
|
|
if offset["to"] is None:
|
|
# if the line already exists, we insert before it.
|
|
if lineNumber in self.to_linenumber_bytes_map:
|
|
keyidx = self.to_linenumber_bytes_map.bisect_left(lineNumber) - 1
|
|
else:
|
|
keyidx = self.to_linenumber_bytes_map.bisect_right(lineNumber) - 1
|
|
key = None
|
|
if keyidx == -1:
|
|
offset["to"] = 0
|
|
elif len(self.to_linenumber_bytes_map.keys()) > 0:
|
|
key = self.to_linenumber_bytes_map.keys()[keyidx]
|
|
else:
|
|
key = 0
|
|
if key is not None:
|
|
offset["to"] = self.to_linenumber_bytes_map.get(key, 0)
|
|
|
|
highlight_offset = offset
|
|
# note that diffs are token-level, but the indexes are byte-level
|
|
|
|
for highlightRange in highlightRanges:
|
|
highlight_start = highlightRange["start"]
|
|
# equal bytes in between highlights
|
|
if highlight_start > highlight_end:
|
|
equal_bytes = highlight_bytes[highlight_end:highlight_start]
|
|
n_equal_bytes = len(equal_bytes)
|
|
|
|
yield from self.doEqual(
|
|
{
|
|
"text": equal_bytes,
|
|
"offset": highlight_offset,
|
|
"lineNumber": lineNumber,
|
|
}
|
|
)
|
|
highlight_offset["from"] += n_equal_bytes
|
|
highlight_offset["to"] += n_equal_bytes
|
|
|
|
# handle highlighted insert / delete
|
|
highlight_end = highlight_start + highlightRange["length"]
|
|
range_bytes = highlight_bytes[highlight_start:highlight_end]
|
|
n_range_bytes = len(range_bytes)
|
|
|
|
if highlightRange["type"] == 0:
|
|
yield from self.doInsert(
|
|
{
|
|
"text": range_bytes,
|
|
"offset": highlight_offset,
|
|
"lineNumber": lineNumber,
|
|
}
|
|
)
|
|
highlight_offset["to"] += n_range_bytes
|
|
elif highlightRange["type"] == 1:
|
|
yield from self.doDelete(
|
|
{
|
|
"text": range_bytes,
|
|
"offset": highlight_offset,
|
|
"lineNumber": lineNumber,
|
|
}
|
|
)
|
|
highlight_offset["from"] += n_range_bytes
|
|
else:
|
|
raise Exception(entry)
|
|
|
|
# handle the rest of the line which is equal
|
|
if highlight_end < len(highlight_bytes):
|
|
range_bytes = highlight_bytes[highlight_end:]
|
|
yield from self.doEqual(
|
|
{
|
|
"text": range_bytes,
|
|
"offset": highlight_offset,
|
|
"lineNumber": lineNumber,
|
|
}
|
|
)
|
|
|
|
def doParMove(self, from_diff, to_diff):
|
|
from_byte_start = from_diff["offset"]["from"]
|
|
to_byte_start = to_diff["offset"]["to"]
|
|
offset = {"from": from_byte_start, "to": to_byte_start}
|
|
yield from self.doHighlightRange(
|
|
{
|
|
"text": to_diff["text"],
|
|
"highlightRanges": to_diff["highlightRanges"],
|
|
"offset": offset,
|
|
"lineNumber": to_diff["lineNumber"],
|
|
}
|
|
)
|
|
|
|
|
|
class WikiDiffMatcher:
|
|
def __init__(
|
|
self,
|
|
tokenizer: Optional[RegexTokenizer] = None,
|
|
):
|
|
self.tokenizer = tokenizer or TOKENIZER
|
|
|
|
class Processor(DiffEngine.Processor):
|
|
def __init__(self, tokenizer=None):
|
|
self.tokenizer = tokenizer or TOKENIZER
|
|
self.last_tokens = []
|
|
self.previous_text = ""
|
|
self.differ = pywikidiff2.pywikidiff2(
|
|
num_context_lines=1000000,
|
|
max_word_level_diff_complexity=-1,
|
|
moved_paragraph_detection_cutoff=-1,
|
|
words_cache_capacity=10000,
|
|
diff_cache_capacity=10000,
|
|
stats_cache_capacity=10000,
|
|
)
|
|
self.last_diff = None
|
|
|
|
def update(self, last_tokens):
|
|
self.last_tokens = last_tokens
|
|
|
|
def process(self, text, token_class=None):
|
|
# The diff has already been computed, but we need to incrementally
|
|
# retrieve it to recreate the behavior DiffState expects.
|
|
diff = json.loads(self.differ.inline_json_diff(self.previous_text, text))
|
|
self.last_diff = diff
|
|
diffToOperationsMapper = DiffToOperationMap(diff, self.tokenizer)
|
|
|
|
diffops = list(diffToOperationsMapper.to_operations())
|
|
|
|
# this happens when revisions are actually equal.
|
|
if len(diffops) == 0:
|
|
self.last_tokens = self.tokenizer.tokenize(text, token_class=Token)
|
|
ops = [Equal(0, len(self.last_tokens), 0, len(self.last_tokens))]
|
|
return ops, self.last_tokens, self.last_tokens
|
|
|
|
# we get back the byte indices; now we transform to token indices
|
|
|
|
diffops.sort(
|
|
key=lambda t: (t[0].a1 if t[0].a1 is not None else 1e32, t[0].b1)
|
|
)
|
|
aorder_ops = []
|
|
token_offset = 0
|
|
_, aseq, _ = list(zip(*diffops))
|
|
|
|
for op, tokens, _ in diffops:
|
|
a1 = token_offset
|
|
if isinstance(op, Equal) or isinstance(op, Delete):
|
|
token_offset += len(tokens)
|
|
a2 = token_offset
|
|
aorder_ops.append(type(op)(a1, a2, op.b1, op.b1))
|
|
else:
|
|
aorder_ops.append(Insert(a1, a1, op.b1, op.b1))
|
|
|
|
_, aseq, bseq = zip(*diffops)
|
|
diffops = list(zip(aorder_ops, aseq, bseq))
|
|
diffops.sort(
|
|
key=lambda t: (t[0].b1 if t[0].b1 is not None else 1e32, t[0].a1)
|
|
)
|
|
_, _, bseq = list(zip(*diffops))
|
|
border_ops = []
|
|
token_offset = 0
|
|
for op, _, tokens in diffops:
|
|
b1 = token_offset
|
|
if isinstance(op, Equal) or isinstance(op, Insert):
|
|
token_offset += len(tokens)
|
|
b2 = token_offset
|
|
border_ops.append(type(op)(op.a1, op.a2, b1, b2))
|
|
else:
|
|
border_ops.append(type(op)(op.a1, op.a2, b1, b1))
|
|
|
|
self.previous_text = text
|
|
|
|
self.last_tokens = list(chain.from_iterable(aseq))
|
|
tokens = list(chain.from_iterable(bseq))
|
|
return border_ops, self.last_tokens, tokens
|
|
|
|
def processor(self, *args, **kwargs):
|
|
return self.Processor(self.tokenizer)
|
|
|
|
def process(self):
|
|
# DiffState checks for this method even though it is not called.
|
|
raise Exception("Unnecessary implementation")
|