add test for --resume with --collapse-user
Resume with collapsed revision groups was untested: the resume point is the (articleid, revid) of the last written row, which for collapsed output is the last revision of a group, and nothing verified that the replay reconstructs group boundaries and collapsed_revs counts identically across the resume point. Run sailormoon with --collapse-user, truncate the output at the midpoint, resume, and assert the result is identical to an uninterrupted run. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -684,3 +684,40 @@ def test_resume_revert_detection():
|
|||||||
# Verify revert column matches exactly
|
# Verify revert column matches exactly
|
||||||
assert_series_equal(df_full["revert"], df_resumed["revert"])
|
assert_series_equal(df_full["revert"], df_resumed["revert"])
|
||||||
assert_series_equal(df_full["reverteds"], df_resumed["reverteds"])
|
assert_series_equal(df_full["reverteds"], df_resumed["reverteds"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_resume_collapse_user():
|
||||||
|
"""Test that --resume with --collapse-user matches an uninterrupted run."""
|
||||||
|
import pandas as pd
|
||||||
|
from pandas.testing import assert_frame_equal
|
||||||
|
|
||||||
|
tester_full = WikiqTester(SAILORMOON, "resume_collapse_full", in_compression="7z", out_format="jsonl")
|
||||||
|
|
||||||
|
try:
|
||||||
|
tester_full.call_wikiq("--collapse-user", "--fandom-2020")
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
pytest.fail(exc.stderr.decode("utf8"))
|
||||||
|
|
||||||
|
full_rows = read_jsonl(tester_full.output)
|
||||||
|
|
||||||
|
# Truncate in the middle of a page with several collapsed rows so the
|
||||||
|
# resume point falls between collapsed revision groups
|
||||||
|
middle_idx = len(full_rows) // 2
|
||||||
|
|
||||||
|
tester_partial = WikiqTester(SAILORMOON, "resume_collapse_partial", in_compression="7z", out_format="jsonl")
|
||||||
|
partial_output_path = tester_partial.output
|
||||||
|
|
||||||
|
with open(partial_output_path, 'w') as f:
|
||||||
|
for row in full_rows[:middle_idx + 1]:
|
||||||
|
f.write(json.dumps(row) + "\n")
|
||||||
|
|
||||||
|
try:
|
||||||
|
tester_partial.call_wikiq("--collapse-user", "--fandom-2020", "--resume")
|
||||||
|
except subprocess.CalledProcessError as exc:
|
||||||
|
pytest.fail(exc.stderr.decode("utf8"))
|
||||||
|
|
||||||
|
resumed_rows = read_jsonl(partial_output_path)
|
||||||
|
|
||||||
|
df_full = pd.DataFrame(full_rows)
|
||||||
|
df_resumed = pd.DataFrame(resumed_rows)
|
||||||
|
assert_frame_equal(df_full, df_resumed)
|
||||||
|
|||||||
Reference in New Issue
Block a user