collective/mediawiki_dump_tools
18
0
Fork 0
Code Issues Pull Requests Activity

support partitioning output parquet by namespace.

Browse Source
This commit is contained in:
Nathan TeBlunthuis
2025-07-07 20:58:43 -07:00
parent c9fb94ccc0
commit 76d54ae597
3 changed files with 55 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
test/Wikiq_Unit_Test.py
View File

@@ -199,6 +199,19 @@ def test_collapse_user():
baseline = pd.read_table(tester.baseline_file)
assert_frame_equal(test, baseline, check_like=True)
def test_partition_namespaces():
tester = WikiqTester(SAILORMOON, "collapse-user", in_compression="7z", out_format='parquet', baseline_format='parquet')
try:
tester.call_wikiq("--collapse-user", "--fandom-2020", "--partition-namespaces")
except subprocess.CalledProcessError as exc:
pytest.fail(exc.stderr.decode("utf8"))
test = pd.read_parquet(os.path.join(tester.output,"namespace=10/sailormoon.parquet"))
baseline = pd.read_parquet(tester.baseline_file)
assert_frame_equal(test, baseline, check_like=True)
def test_pwr_wikidiff2():
tester = WikiqTester(SAILORMOON, "persistence_wikidiff2", in_compression="7z")