collective/wikiq
20
0
Fork 0
Code Issues Pull Requests 3 Activity

Prepare wikiq for release: fixes, dependency cleanup, and packaging #2

Open
mako wants to merge 33 commits from release_review into jsonl-output
pull from: release_review
merge into: collective:jsonl-output
Conversation 0 Commits 33 Files Changed 42 +5 -3
2 changed files with 5 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit a26abe0e68 - Show all commits

6
src/wikiq/__init__.py
View File

@@ -122,8 +122,10 @@ def build_table(
"""Build the RevisionTable with appropriate columns based on flags. """Build the RevisionTable with appropriate columns based on flags.
Returns: Returns:
(table, reverts_column) - the table and a reference to the reverts column (table, reverts_column, wikitext_parser) - the table, a reference to
(which process() needs for setting the revert detector). the reverts column (which process() needs for setting the revert
detector), and the wikitext parser (None unless a wikitext-parsing
column was requested).
""" """
reverts_column = tables.RevisionReverts() reverts_column = tables.RevisionReverts()

2
test/Wikiq_Unit_Test.py
View File

@@ -38,7 +38,7 @@ setup()
def read_jsonl_with_schema(filepath: str, **schema_kwargs) -> pd.DataFrame: def read_jsonl_with_schema(filepath: str, **schema_kwargs) -> pd.DataFrame:
"""Read JSONL file using PyArrow with explicit schema from wikiq.""" """Read JSONL file using PyArrow with explicit schema from wikiq."""
table, _ = build_table(**schema_kwargs) table, _, _ = build_table(**schema_kwargs)
schema = build_schema(table, **schema_kwargs) schema = build_schema(table, **schema_kwargs)
pa_table = pj.read_json( pa_table = pj.read_json(
filepath, filepath,