Prepare wikiq for release: fixes, dependency cleanup, and packaging #2

Open
mako wants to merge 33 commits from release_review into jsonl-output
Showing only changes of commit 268c212469 - Show all commits

View File

@@ -32,19 +32,24 @@ from collections import deque, defaultdict
from hashlib import sha1 from hashlib import sha1
from io import TextIOWrapper from io import TextIOWrapper
from itertools import groupby from itertools import groupby
from pathlib import Path
from subprocess import PIPE, Popen from subprocess import PIPE, Popen
from typing import IO, Any, Generator, TextIO, Union from typing import IO, Any, Generator, TextIO, Union
import mwpersistence import mwpersistence
import mwreverts import mwreverts
import mwxml import mwxml
import pyarrow as pa
import pyarrow.csv as pacsv
from deltas import SegmentMatcher, SequenceMatcher
from deltas.tokenizers import wikitext_split from deltas.tokenizers import wikitext_split
from more_itertools import peekable from more_itertools import peekable
from mwxml import Dump from mwxml import Dump
import wikiq.tables as tables import wikiq.tables as tables
from wikiq.resume import get_resume_point
from wikiq.tables import RevisionTable from wikiq.tables import RevisionTable
from wikiq.wikitext_parser import WikitextParser from wikiq.wikitext_parser import WikitextParser
from wikiq.resume import get_resume_point
try: try:
from importlib.metadata import PackageNotFoundError from importlib.metadata import PackageNotFoundError
@@ -56,11 +61,6 @@ except PackageNotFoundError:
TO_ENCODE = ("title", "editor") TO_ENCODE = ("title", "editor")
PERSISTENCE_RADIUS = 7 PERSISTENCE_RADIUS = 7
DIFF_TIMEOUT_MS = 60000 DIFF_TIMEOUT_MS = 60000
from pathlib import Path
import pyarrow as pa
import pyarrow.csv as pacsv
from deltas import SegmentMatcher, SequenceMatcher
# Some dependencies serve a single feature and are awkward enough to install # Some dependencies serve a single feature and are awkward enough to install