remove server.
This commit is contained in:
@@ -4,48 +4,12 @@ from collections import namedtuple
|
||||
from itertools import chain
|
||||
from typing import Dict, Generator, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
from deltas import (Delete, DiffEngine, Equal, Insert, Operation,
|
||||
RegexTokenizer, Token, tokenizers)
|
||||
from sortedcontainers import SortedDict
|
||||
|
||||
TOKENIZER = tokenizers.wikitext_split
|
||||
import pywikidiff2
|
||||
differ = pywikidiff2.pywikidiff2(numContextLines=1000000,
|
||||
moved_paragraph_detection_cutoff=200000)
|
||||
|
||||
def compute_diffs_server(texts, url="http://127.0.0.1:8000"):
|
||||
response = None
|
||||
try:
|
||||
response = requests.post(url, json=texts)
|
||||
response.raise_for_status()
|
||||
incremental_diffs = response.json()
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
print(
|
||||
f"Connection Error: Could not connect to the server at {url}. Make sure your local server is running."
|
||||
)
|
||||
print(e)
|
||||
raise e
|
||||
except requests.exceptions.HTTPError as e:
|
||||
print(f"HTTP Error: {e}")
|
||||
if response is not None:
|
||||
print(f"Response Body: {response.text}")
|
||||
raise e
|
||||
except requests.exceptions.JSONDecodeError as e:
|
||||
# Must come before RequestException as JSONDecodeError is
|
||||
# a subclass.
|
||||
print(f"JSON Decode Error: {e}", file=sys.stderr)
|
||||
if response is not None:
|
||||
print(f"Response Body: {response.text}", file=sys.stderr)
|
||||
raise e
|
||||
except requests.exceptions.RequestException as e:
|
||||
print(f"An unexpected error occurred: {e}")
|
||||
raise e
|
||||
return incremental_diffs
|
||||
|
||||
|
||||
def compute_diffs(texts: list[str]) -> list:
|
||||
return differ.inline_json_diff_sequence(texts)
|
||||
import pywikidiff2
|
||||
|
||||
class DiffToOperationMap:
|
||||
def __init__(self, diff, tokenizer):
|
||||
@@ -402,14 +366,12 @@ class WikiDiffMatcher:
|
||||
self,
|
||||
texts: list[str] = None,
|
||||
tokenizer: Optional[RegexTokenizer] = None,
|
||||
url: Optional[str] = "http://127.0.0.1:8000",
|
||||
server=False
|
||||
):
|
||||
differ = pywikidiff2.pywikidiff2(numContextLines=1000000,
|
||||
moved_paragraph_detection_cutoff=200000)
|
||||
# Pre-compute diffs to reduce traffic overhead.
|
||||
if server is True:
|
||||
self.diffs = list(compute_diffs_server(list(texts),url))
|
||||
else:
|
||||
self.diffs = list(compute_diffs(list(texts)))
|
||||
self.diffs = differ.inline_json_diff_sequence(list(texts))
|
||||
|
||||
self.tokenizer = tokenizer or TOKENIZER
|
||||
|
||||
class Processor(DiffEngine.Processor):
|
||||
|
||||
Reference in New Issue
Block a user