add test for --resume with --collapse-user
Resume with collapsed revision groups was untested: the resume point is
the (articleid, revid) of the last written row, which for collapsed
output is the last revision of a group, and nothing verified that the
replay reconstructs group boundaries and collapsed_revs counts
identically across the resume point. Run sailormoon with
--collapse-user, truncate the output at the midpoint, resume, and
assert the result is identical to an uninterrupted run.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit 6634de51e6)
This commit is contained in:
@@ -806,3 +806,40 @@ def test_resume_revert_detection():
|
||||
# Verify revert column matches exactly
|
||||
assert_series_equal(df_full["revert"], df_resumed["revert"])
|
||||
assert_series_equal(df_full["reverteds"], df_resumed["reverteds"])
|
||||
|
||||
|
||||
def test_resume_collapse_user():
|
||||
"""Test that --resume with --collapse-user matches an uninterrupted run."""
|
||||
import pandas as pd
|
||||
from pandas.testing import assert_frame_equal
|
||||
|
||||
tester_full = WikiqTester(SAILORMOON, "resume_collapse_full", in_compression="7z", out_format="jsonl")
|
||||
|
||||
try:
|
||||
tester_full.call_wikiq("--collapse-user", "--fandom-2020")
|
||||
except subprocess.CalledProcessError as exc:
|
||||
pytest.fail(exc.stderr.decode("utf8"))
|
||||
|
||||
full_rows = read_jsonl(tester_full.output)
|
||||
|
||||
# Truncate in the middle of a page with several collapsed rows so the
|
||||
# resume point falls between collapsed revision groups
|
||||
middle_idx = len(full_rows) // 2
|
||||
|
||||
tester_partial = WikiqTester(SAILORMOON, "resume_collapse_partial", in_compression="7z", out_format="jsonl")
|
||||
partial_output_path = tester_partial.output
|
||||
|
||||
with open(partial_output_path, 'w') as f:
|
||||
for row in full_rows[:middle_idx + 1]:
|
||||
f.write(json.dumps(row) + "\n")
|
||||
|
||||
try:
|
||||
tester_partial.call_wikiq("--collapse-user", "--fandom-2020", "--resume")
|
||||
except subprocess.CalledProcessError as exc:
|
||||
pytest.fail(exc.stderr.decode("utf8"))
|
||||
|
||||
resumed_rows = read_jsonl(partial_output_path)
|
||||
|
||||
df_full = pd.DataFrame(full_rows)
|
||||
df_resumed = pd.DataFrame(resumed_rows)
|
||||
assert_frame_equal(df_full, df_resumed)
|
||||
|
||||
Reference in New Issue
Block a user