don't put checkpoint files inside namespace directories.
This commit is contained in:
@@ -322,7 +322,7 @@ class WikiqParser:
|
||||
"""Open checkpoint file for writing. Keeps file open for performance."""
|
||||
if not self.output_parquet or output_file == sys.stdout.buffer:
|
||||
return
|
||||
checkpoint_path = get_checkpoint_path(output_file)
|
||||
checkpoint_path = get_checkpoint_path(output_file, self.partition_namespaces)
|
||||
Path(checkpoint_path).parent.mkdir(parents=True, exist_ok=True)
|
||||
self.checkpoint_file = open(checkpoint_path, 'w')
|
||||
print(f"Checkpoint file opened: {checkpoint_path}", file=sys.stderr)
|
||||
|
||||
Reference in New Issue
Block a user