remove server.

This commit is contained in:
Nathan TeBlunthuis
2025-07-07 11:21:16 -07:00
parent 14e819e565
commit cc96bb5f3f
3 changed files with 7 additions and 120 deletions

View File

@@ -4,48 +4,12 @@ from collections import namedtuple
from itertools import chain
from typing import Dict, Generator, List, Optional, Tuple
import requests
from deltas import (Delete, DiffEngine, Equal, Insert, Operation,
RegexTokenizer, Token, tokenizers)
from sortedcontainers import SortedDict
TOKENIZER = tokenizers.wikitext_split
import pywikidiff2
differ = pywikidiff2.pywikidiff2(numContextLines=1000000,
moved_paragraph_detection_cutoff=200000)
def compute_diffs_server(texts, url="http://127.0.0.1:8000"):
response = None
try:
response = requests.post(url, json=texts)
response.raise_for_status()
incremental_diffs = response.json()
except requests.exceptions.ConnectionError as e:
print(
f"Connection Error: Could not connect to the server at {url}. Make sure your local server is running."
)
print(e)
raise e
except requests.exceptions.HTTPError as e:
print(f"HTTP Error: {e}")
if response is not None:
print(f"Response Body: {response.text}")
raise e
except requests.exceptions.JSONDecodeError as e:
# Must come before RequestException as JSONDecodeError is
# a subclass.
print(f"JSON Decode Error: {e}", file=sys.stderr)
if response is not None:
print(f"Response Body: {response.text}", file=sys.stderr)
raise e
except requests.exceptions.RequestException as e:
print(f"An unexpected error occurred: {e}")
raise e
return incremental_diffs
def compute_diffs(texts: list[str]) -> list:
return differ.inline_json_diff_sequence(texts)
import pywikidiff2
class DiffToOperationMap:
def __init__(self, diff, tokenizer):
@@ -402,14 +366,12 @@ class WikiDiffMatcher:
self,
texts: list[str] = None,
tokenizer: Optional[RegexTokenizer] = None,
url: Optional[str] = "http://127.0.0.1:8000",
server=False
):
differ = pywikidiff2.pywikidiff2(numContextLines=1000000,
moved_paragraph_detection_cutoff=200000)
# Pre-compute diffs to reduce traffic overhead.
if server is True:
self.diffs = list(compute_diffs_server(list(texts),url))
else:
self.diffs = list(compute_diffs(list(texts)))
self.diffs = differ.inline_json_diff_sequence(list(texts))
self.tokenizer = tokenizer or TOKENIZER
class Processor(DiffEngine.Processor):