collective/mediawiki_dump_tools
18
0
Fork 0
Code Issues Pull Requests Activity

Pin to python 3.9

Browse Source 
Since our execution environment requires this

Signed-off-by: Will Beason <willbeason@gmail.com>
This commit is contained in:
Will Beason
2025-06-17 11:37:20 -05:00
parent 84d464ea38
commit 390499dd90
4 changed files with 21 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
wikiq
View File

@@ -14,7 +14,7 @@ from itertools import groupby
from subprocess import Popen, PIPE
from collections import deque
from hashlib import sha1
from typing import Any, IO, TextIO, Generator
from typing import Any, IO, TextIO, Generator, Union
import mwxml
from mwxml import Dump
@@ -100,7 +100,7 @@ class WikiqPage:
self.__revisions: Generator[list[mwxml.Revision]] = self.rev_list()
@staticmethod
def user_text(rev) -> str | None:
def user_text(rev) -> Union[str, None]:
return None if rev.deleted.user else rev.user.text
def rev_list(self):
@@ -203,15 +203,15 @@ class RegexPair(object):
class WikiqParser:
def __init__(self,
input_file: TextIOWrapper | IO[Any] | IO[bytes],
output_file: TextIO | str,
input_file: Union[TextIOWrapper, IO[Any], IO[bytes]],
output_file: Union[TextIO, str],
regex_match_revision: list[str],
regex_match_comment: list[str],
regex_revision_label: list[str],
regex_comment_label: list[str],
collapse_user: bool = False,
persist: int = None,
namespaces: list[int] | None = None,
namespaces: Union[list[int], None] = None,
revert_radius: int = 15,
output_parquet: bool = True,
parquet_buffer_size: int = 2000):
@@ -410,7 +410,7 @@ class WikiqParser:
# Collect the set of pages currently buffered in the table so we can run multi-page functions on them.
row_buffer = table.pop()
is_revert_column: list[bool | None] = []
is_revert_column: list[Union[bool, None]] = []
for r, d in zip(row_buffer['reverteds'], row_buffer['deleted']):
if self.revert_radius == 0 or d:
is_revert_column.append(None)