Prepare wikiq for release: fixes, dependency cleanup, and packaging #2

Open
mako wants to merge 33 commits from release_review into jsonl-output
Showing only changes of commit f4f0ed72a0 - Show all commits

View File

@@ -21,8 +21,6 @@ def assert_equal_enough(tokens:List[Token], rev):
# the tokens exclude newlines # the tokens exclude newlines
# we allow extra whitespace at the beginning or end # we allow extra whitespace at the beginning or end
token_doc = ''.join(str(t) for t in tokens) token_doc = ''.join(str(t) for t in tokens)
print(token_doc, file = open('token','w'))
print(rev, file = open('rev','w'))
token_doc = re.sub(r'( +)|(\n+)|(\t+)', _replace_whitespace, token_doc).strip() token_doc = re.sub(r'( +)|(\n+)|(\t+)', _replace_whitespace, token_doc).strip()
rev = re.sub(r'( +)|(\n+)|(\t+)', _replace_whitespace, rev).strip() rev = re.sub(r'( +)|(\n+)|(\t+)', _replace_whitespace, rev).strip()
assert token_doc == rev assert token_doc == rev