Prepare wikiq for release: fixes, dependency cleanup, and packaging #2
@@ -32,19 +32,24 @@ from collections import deque, defaultdict
|
|||||||
from hashlib import sha1
|
from hashlib import sha1
|
||||||
from io import TextIOWrapper
|
from io import TextIOWrapper
|
||||||
from itertools import groupby
|
from itertools import groupby
|
||||||
|
from pathlib import Path
|
||||||
from subprocess import PIPE, Popen
|
from subprocess import PIPE, Popen
|
||||||
from typing import IO, Any, Generator, TextIO, Union
|
from typing import IO, Any, Generator, TextIO, Union
|
||||||
|
|
||||||
import mwpersistence
|
import mwpersistence
|
||||||
import mwreverts
|
import mwreverts
|
||||||
import mwxml
|
import mwxml
|
||||||
|
import pyarrow as pa
|
||||||
|
import pyarrow.csv as pacsv
|
||||||
|
from deltas import SegmentMatcher, SequenceMatcher
|
||||||
from deltas.tokenizers import wikitext_split
|
from deltas.tokenizers import wikitext_split
|
||||||
from more_itertools import peekable
|
from more_itertools import peekable
|
||||||
from mwxml import Dump
|
from mwxml import Dump
|
||||||
|
|
||||||
import wikiq.tables as tables
|
import wikiq.tables as tables
|
||||||
|
from wikiq.resume import get_resume_point
|
||||||
from wikiq.tables import RevisionTable
|
from wikiq.tables import RevisionTable
|
||||||
from wikiq.wikitext_parser import WikitextParser
|
from wikiq.wikitext_parser import WikitextParser
|
||||||
from wikiq.resume import get_resume_point
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from importlib.metadata import PackageNotFoundError
|
from importlib.metadata import PackageNotFoundError
|
||||||
@@ -56,11 +61,6 @@ except PackageNotFoundError:
|
|||||||
TO_ENCODE = ("title", "editor")
|
TO_ENCODE = ("title", "editor")
|
||||||
PERSISTENCE_RADIUS = 7
|
PERSISTENCE_RADIUS = 7
|
||||||
DIFF_TIMEOUT_MS = 60000
|
DIFF_TIMEOUT_MS = 60000
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pyarrow as pa
|
|
||||||
import pyarrow.csv as pacsv
|
|
||||||
from deltas import SegmentMatcher, SequenceMatcher
|
|
||||||
|
|
||||||
|
|
||||||
# Some dependencies serve a single feature and are awkward enough to install
|
# Some dependencies serve a single feature and are awkward enough to install
|
||||||
|
|||||||
Reference in New Issue
Block a user