parq-cli 0.2.2__tar.gz → 0.2.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. {parq_cli-0.2.2 → parq_cli-0.2.3}/CHANGELOG.md +15 -0
  2. {parq_cli-0.2.2 → parq_cli-0.2.3}/PKG-INFO +1 -1
  3. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/cli.py +100 -102
  4. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_chunk_writers.py +133 -52
  5. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_common.py +44 -7
  6. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_csv.py +3 -1
  7. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_parquet.py +6 -0
  8. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/plain_output.py +4 -1
  9. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/reader.py +210 -105
  10. {parq_cli-0.2.2 → parq_cli-0.2.3}/pyproject.toml +1 -1
  11. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_cli.py +59 -0
  12. parq_cli-0.2.3/tests/test_output_not_input.py +63 -0
  13. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_plain_output.py +18 -0
  14. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_reader.py +220 -1
  15. {parq_cli-0.2.2 → parq_cli-0.2.3}/uv.lock +1 -1
  16. parq_cli-0.2.2/tests/test_output_not_input.py +0 -26
  17. {parq_cli-0.2.2 → parq_cli-0.2.3}/.claude/settings.json +0 -0
  18. {parq_cli-0.2.2 → parq_cli-0.2.3}/.github/workflows/publish.yml +0 -0
  19. {parq_cli-0.2.2 → parq_cli-0.2.3}/.github/workflows/test.yml +0 -0
  20. {parq_cli-0.2.2 → parq_cli-0.2.3}/.gitignore +0 -0
  21. {parq_cli-0.2.2 → parq_cli-0.2.3}/.python-version +0 -0
  22. {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/csv-large.csv +0 -0
  23. {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/csv-small.csv +0 -0
  24. {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/parquet-large.parquet +0 -0
  25. {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/parquet-small.parquet +0 -0
  26. {parq_cli-0.2.2 → parq_cli-0.2.3}/AGENTS.md +0 -0
  27. {parq_cli-0.2.2 → parq_cli-0.2.3}/CLAUDE.md +0 -0
  28. {parq_cli-0.2.2 → parq_cli-0.2.3}/LICENSE +0 -0
  29. {parq_cli-0.2.2 → parq_cli-0.2.3}/README.md +0 -0
  30. {parq_cli-0.2.2 → parq_cli-0.2.3}/README_CN.md +0 -0
  31. {parq_cli-0.2.2 → parq_cli-0.2.3}/data/test.parquet +0 -0
  32. {parq_cli-0.2.2 → parq_cli-0.2.3}/docs/QUICKSTART.md +0 -0
  33. {parq_cli-0.2.2 → parq_cli-0.2.3}/docs/RELEASE.md +0 -0
  34. {parq_cli-0.2.2 → parq_cli-0.2.3}/examples/README.md +0 -0
  35. {parq_cli-0.2.2 → parq_cli-0.2.3}/examples/create_sample_data.py +0 -0
  36. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/__init__.py +0 -0
  37. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/__main__.py +0 -0
  38. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/__init__.py +0 -0
  39. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_xlsx.py +0 -0
  40. {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/output.py +0 -0
  41. {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/README.md +0 -0
  42. {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/bump_version.py +0 -0
  43. {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/check_version.py +0 -0
  44. {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/performance_benchmark.py +0 -0
  45. {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/publish.sh +0 -0
  46. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/__init__.py +0 -0
  47. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/conftest.py +0 -0
  48. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_main_entry.py +0 -0
  49. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_output.py +0 -0
  50. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_package_meta.py +0 -0
  51. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_performance.py +0 -0
  52. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_release_scripts.py +0 -0
  53. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_workflows.py +0 -0
  54. {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_xlsx_defused.py +0 -0
@@ -1,5 +1,20 @@
1
1
  # Changelog
2
2
 
3
+ ## Unreleased
4
+
5
+ ## 0.2.3
6
+
7
+ - `merge --force` and `convert --force` replace an existing output file only after the new file is fully written. A schema mismatch or a write error leaves the previous file in place, and the temporary file is removed, including on Windows.
8
+ - `diff` treats NaN as equal to NaN, including when NaN is the key.
9
+ - `diff` and `merge` close Parquet file handles when they finish.
10
+ - JSON table output summarizes binary values, matching rich and plain output.
11
+ - `split --force` replaces existing part files only after every part is written. A later write error leaves the previous parts in place.
12
+ - Column stats ignore NaN when computing min, max, and mean. Nulls are still counted separately, and NaN stays in the non-null count.
13
+ - An empty, multi-character, or non-ASCII `--delimiter` is rejected with `delimiter must be a single ASCII character` before CSV parsing or writing.
14
+ - `stats --limit`, `stats --top-n`, and `diff --limit` reject negative values instead of slicing from the end. Zero still means an empty result.
15
+ - Selecting a duplicated column name reports that the name cannot be selected, instead of claiming the column does not exist. A unique column in the same file can still be selected.
16
+ - `diff` compares repeated column names by position. A diff that would compare those names by default asks for `--columns` instead of claiming the column does not exist.
17
+
3
18
  ## 0.2.2
4
19
 
5
20
  - `merge --force` and `convert` now refuse an output path that is one of the inputs. Previously `--force` deleted that input before reading it.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: parq-cli
3
- Version: 0.2.2
3
+ Version: 0.2.3
4
4
  Summary: A command-line tool for inspecting tabular files: Parquet, CSV, TSV, and XLSX
5
5
  Project-URL: Homepage, https://github.com/Tendo33/parq-cli
6
6
  Project-URL: Repository, https://github.com/Tendo33/parq-cli
@@ -207,8 +207,8 @@ def meta(
207
207
  _apply_command_options(ctx, output_format, delimiter, sheet)
208
208
 
209
209
  def operation(formatter: Any) -> None:
210
- reader = _get_reader(str(file), _get_state(ctx))
211
- formatter.print_metadata(reader.get_metadata_dict(fast=fast))
210
+ with _get_reader(str(file), _get_state(ctx)) as reader:
211
+ formatter.print_metadata(reader.get_metadata_dict(fast=fast))
212
212
 
213
213
  _run_with_error_handling(
214
214
  ctx, operation, generic_error_prefix="Failed to read file", file_path=file
@@ -227,8 +227,8 @@ def schema(
227
227
  _apply_command_options(ctx, output_format, delimiter, sheet)
228
228
 
229
229
  def operation(formatter: Any) -> None:
230
- reader = _get_reader(str(file), _get_state(ctx))
231
- formatter.print_schema(reader.get_schema_info())
230
+ with _get_reader(str(file), _get_state(ctx)) as reader:
231
+ formatter.print_schema(reader.get_schema_info())
232
232
 
233
233
  _run_with_error_handling(
234
234
  ctx, operation, generic_error_prefix="Failed to read file", file_path=file
@@ -252,9 +252,9 @@ def head(
252
252
  _apply_command_options(ctx, output_format, delimiter, sheet)
253
253
 
254
254
  def operation(formatter: Any) -> None:
255
- reader = _get_reader(str(file), _get_state(ctx))
256
- table = reader.read_head(n, columns=_parse_column_list(columns))
257
- formatter.print_table(table, f"First {n} Rows")
255
+ with _get_reader(str(file), _get_state(ctx)) as reader:
256
+ table = reader.read_head(n, columns=_parse_column_list(columns))
257
+ formatter.print_table(table, f"First {n} Rows")
258
258
 
259
259
  _run_with_error_handling(
260
260
  ctx, operation, generic_error_prefix="Failed to read file", file_path=file
@@ -278,9 +278,9 @@ def tail(
278
278
  _apply_command_options(ctx, output_format, delimiter, sheet)
279
279
 
280
280
  def operation(formatter: Any) -> None:
281
- reader = _get_reader(str(file), _get_state(ctx))
282
- table = reader.read_tail(n, columns=_parse_column_list(columns))
283
- formatter.print_table(table, f"Last {n} Rows")
281
+ with _get_reader(str(file), _get_state(ctx)) as reader:
282
+ table = reader.read_tail(n, columns=_parse_column_list(columns))
283
+ formatter.print_table(table, f"Last {n} Rows")
284
284
 
285
285
  _run_with_error_handling(
286
286
  ctx, operation, generic_error_prefix="Failed to read file", file_path=file
@@ -299,8 +299,8 @@ def count(
299
299
  _apply_command_options(ctx, output_format, delimiter, sheet)
300
300
 
301
301
  def operation(formatter: Any) -> None:
302
- reader = _get_reader(str(file), _get_state(ctx))
303
- formatter.print_count(reader.num_rows)
302
+ with _get_reader(str(file), _get_state(ctx)) as reader:
303
+ formatter.print_count(reader.num_rows)
304
304
 
305
305
  _run_with_error_handling(
306
306
  ctx, operation, generic_error_prefix="Failed to read file", file_path=file
@@ -352,60 +352,59 @@ def split(
352
352
  import time
353
353
 
354
354
  start_time = time.time()
355
- reader = _get_reader(str(file), state)
356
-
357
- if state.output_format == OutputFormat.RICH:
358
- from rich.progress import (
359
- BarColumn,
360
- Progress,
361
- SpinnerColumn,
362
- TextColumn,
363
- TimeRemainingColumn,
364
- )
365
- from parq.output import console
366
-
367
- with Progress(
368
- SpinnerColumn(),
369
- TextColumn("[bold blue]{task.description}"),
370
- BarColumn(),
371
- TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
372
- TimeRemainingColumn(),
373
- console=console,
374
- ) as progress:
375
- task = progress.add_task(f"[cyan]Splitting {file.name}...", total=None)
376
-
377
- def update_progress(current: int, total: int):
378
- if progress.tasks[task].total is None and total:
379
- progress.update(task, total=total)
380
- progress.update(task, completed=current)
381
-
355
+ with _get_reader(str(file), state) as reader:
356
+ if state.output_format == OutputFormat.RICH:
357
+ from rich.progress import (
358
+ BarColumn,
359
+ Progress,
360
+ SpinnerColumn,
361
+ TextColumn,
362
+ TimeRemainingColumn,
363
+ )
364
+ from parq.output import console
365
+
366
+ with Progress(
367
+ SpinnerColumn(),
368
+ TextColumn("[bold blue]{task.description}"),
369
+ BarColumn(),
370
+ TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
371
+ TimeRemainingColumn(),
372
+ console=console,
373
+ ) as progress:
374
+ task = progress.add_task(f"[cyan]Splitting {file.name}...", total=None)
375
+
376
+ def update_progress(current: int, total: int):
377
+ if progress.tasks[task].total is None and total:
378
+ progress.update(task, total=total)
379
+ progress.update(task, completed=current)
380
+
381
+ output_files = reader.split_file(
382
+ output_pattern=name_format,
383
+ file_count=file_count,
384
+ record_count=record_count,
385
+ progress_callback=update_progress,
386
+ force=force,
387
+ )
388
+ else:
382
389
  output_files = reader.split_file(
383
390
  output_pattern=name_format,
384
391
  file_count=file_count,
385
392
  record_count=record_count,
386
- progress_callback=update_progress,
387
393
  force=force,
388
394
  )
389
- else:
390
- output_files = reader.split_file(
391
- output_pattern=name_format,
392
- file_count=file_count,
393
- record_count=record_count,
394
- force=force,
395
- )
396
395
 
397
- elapsed_time = time.time() - start_time
398
- total_rows = (
399
- reader.last_split_total_rows
400
- if reader.last_split_total_rows is not None
401
- else reader.num_rows
402
- )
403
- formatter.print_split_result(
404
- source_file=file,
405
- output_files=output_files,
406
- total_rows=total_rows,
407
- elapsed_time=elapsed_time,
408
- )
396
+ elapsed_time = time.time() - start_time
397
+ total_rows = (
398
+ reader.last_split_total_rows
399
+ if reader.last_split_total_rows is not None
400
+ else reader.num_rows
401
+ )
402
+ formatter.print_split_result(
403
+ source_file=file,
404
+ output_files=output_files,
405
+ total_rows=total_rows,
406
+ elapsed_time=elapsed_time,
407
+ )
409
408
  except typer.Exit:
410
409
  raise
411
410
  except (FileNotFoundError, ValueError, FileExistsError) as e:
@@ -437,10 +436,10 @@ def stats(
437
436
  _apply_command_options(ctx, output_format, delimiter, sheet)
438
437
 
439
438
  def operation(formatter: Any) -> None:
440
- reader = _get_reader(str(file), _get_state(ctx))
441
- formatter.print_stats(
442
- reader.get_stats(columns=_parse_column_list(columns), limit=limit, top_n=top_n)
443
- )
439
+ with _get_reader(str(file), _get_state(ctx)) as reader:
440
+ formatter.print_stats(
441
+ reader.get_stats(columns=_parse_column_list(columns), limit=limit, top_n=top_n)
442
+ )
444
443
 
445
444
  _run_with_error_handling(
446
445
  ctx, operation, generic_error_prefix="Failed to compute stats", file_path=file
@@ -476,52 +475,51 @@ def convert(
476
475
  import time
477
476
 
478
477
  start_time = time.time()
479
- reader = _get_reader(str(source), state)
480
-
481
- if state.output_format == OutputFormat.RICH:
482
- from rich.progress import (
483
- Progress,
484
- SpinnerColumn,
485
- TextColumn,
486
- BarColumn,
487
- TimeRemainingColumn,
488
- )
489
- from parq.output import console
490
-
491
- total_rows_hint = reader.num_rows if reader.input_format == "parquet" else 0
492
-
493
- with Progress(
494
- SpinnerColumn(),
495
- TextColumn("[bold blue]{task.description}"),
496
- BarColumn(),
497
- TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
498
- TimeRemainingColumn(),
499
- console=console,
500
- ) as progress:
501
- task = progress.add_task(
502
- f"[cyan]Converting {source.name}...",
503
- total=total_rows_hint if total_rows_hint > 0 else None,
478
+ with _get_reader(str(source), state) as reader:
479
+ if state.output_format == OutputFormat.RICH:
480
+ from rich.progress import (
481
+ Progress,
482
+ SpinnerColumn,
483
+ TextColumn,
484
+ BarColumn,
485
+ TimeRemainingColumn,
504
486
  )
505
-
506
- def update_progress(current: int, total: int) -> None:
507
- if progress.tasks[task].total is None and total:
508
- progress.update(task, total=total)
509
- progress.update(task, completed=current)
510
-
487
+ from parq.output import console
488
+
489
+ total_rows_hint = reader.num_rows if reader.input_format == "parquet" else 0
490
+
491
+ with Progress(
492
+ SpinnerColumn(),
493
+ TextColumn("[bold blue]{task.description}"),
494
+ BarColumn(),
495
+ TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
496
+ TimeRemainingColumn(),
497
+ console=console,
498
+ ) as progress:
499
+ task = progress.add_task(
500
+ f"[cyan]Converting {source.name}...",
501
+ total=total_rows_hint if total_rows_hint > 0 else None,
502
+ )
503
+
504
+ def update_progress(current: int, total: int) -> None:
505
+ if progress.tasks[task].total is None and total:
506
+ progress.update(task, total=total)
507
+ progress.update(task, completed=current)
508
+
509
+ total_rows = reader.convert_file(
510
+ output,
511
+ columns=_parse_column_list(columns),
512
+ force=force,
513
+ progress_callback=update_progress,
514
+ )
515
+ else:
511
516
  total_rows = reader.convert_file(
512
517
  output,
513
518
  columns=_parse_column_list(columns),
514
519
  force=force,
515
- progress_callback=update_progress,
516
520
  )
517
- else:
518
- total_rows = reader.convert_file(
519
- output,
520
- columns=_parse_column_list(columns),
521
- force=force,
522
- )
523
521
 
524
- formatter.print_convert_result(source, output, total_rows, time.time() - start_time)
522
+ formatter.print_convert_result(source, output, total_rows, time.time() - start_time)
525
523
  except typer.Exit:
526
524
  raise
527
525
  except (FileNotFoundError, ValueError, FileExistsError) as e:
@@ -2,6 +2,9 @@
2
2
  Chunk writers and streaming split/write utilities for all output formats.
3
3
  """
4
4
 
5
+ import gc
6
+ import os
7
+ import tempfile
5
8
  from pathlib import Path
6
9
  from typing import Any, Callable, Iterable, List, Optional
7
10
 
@@ -9,6 +12,7 @@ import pyarrow as pa
9
12
  import pyarrow.csv as pacsv
10
13
  import pyarrow.parquet as pq
11
14
 
15
+ from parq.formats._common import _validate_delimiter
12
16
  from parq.formats._xlsx import _require_openpyxl
13
17
 
14
18
 
@@ -19,24 +23,70 @@ def _coerce_output_value(value: Any) -> Any:
19
23
  return value
20
24
 
21
25
 
22
- def _cleanup_split_files(writer: Any, created_paths: List[Path]) -> None:
23
- """Close a writer (if open) and delete any already-created output files on failure."""
24
- if writer is not None:
26
+ def _release_writer(writer: Any) -> None:
27
+ """Close a writer. PyArrow's own writer is closed and dropped here.
28
+
29
+ On Windows the file stays locked until that object is destroyed, not merely
30
+ until ``close()`` returns.
31
+ """
32
+ if writer is None:
33
+ return
34
+ inner = getattr(writer, "_writer", None)
35
+ if inner is not None:
36
+ writer._writer = None
25
37
  try:
26
- writer.close()
38
+ inner.close()
27
39
  except Exception:
28
40
  pass
41
+ del inner
42
+ return
43
+ try:
44
+ writer.close()
45
+ except Exception:
46
+ pass
47
+
48
+
49
+ def _unlink_output(path: Path) -> None:
50
+ """Delete a staged file. A second try runs after GC, for Windows file locks."""
51
+ try:
52
+ path.unlink()
53
+ return
54
+ except FileNotFoundError:
55
+ return
56
+ except OSError:
57
+ pass
58
+ gc.collect()
59
+ try:
60
+ path.unlink()
61
+ except (FileNotFoundError, OSError):
62
+ pass
63
+
64
+
65
+ def _cleanup_split_files(writers: List[Any], created_paths: List[Path]) -> None:
66
+ """Close writers and delete staged temp files on failure.
67
+
68
+ The caller must put its only reference in ``writers`` and then drop its own
69
+ variable. This function clears that list before deleting the files.
70
+ """
71
+ for index in range(len(writers)):
72
+ _release_writer(writers[index])
73
+ writers[index] = None
74
+ gc.collect()
29
75
  for path in created_paths:
30
- try:
31
- path.unlink()
32
- except (FileNotFoundError, OSError):
33
- pass
76
+ _unlink_output(path)
77
+
78
+
79
+ def _publish_staged_outputs(staged: List[tuple[Path, Path]]) -> None:
80
+ """Move staged split parts onto their final paths after every part is complete."""
81
+ for final_path, temp_path in staged:
82
+ os.replace(temp_path, final_path)
34
83
 
35
84
 
36
85
  class _CsvChunkWriter:
37
86
  """Incremental CSV/TSV writer backed by PyArrow CSVWriter."""
38
87
 
39
88
  def __init__(self, output_path: Path, schema: pa.Schema, delimiter: str = ","):
89
+ _validate_delimiter(delimiter)
40
90
  output_path.parent.mkdir(parents=True, exist_ok=True)
41
91
  self._writer = pacsv.CSVWriter(
42
92
  output_path,
@@ -48,7 +98,10 @@ class _CsvChunkWriter:
48
98
  self._writer.write(batch)
49
99
 
50
100
  def close(self) -> None:
51
- self._writer.close()
101
+ writer = self._writer
102
+ self._writer = None
103
+ if writer is not None:
104
+ writer.close()
52
105
 
53
106
 
54
107
  class _XlsxChunkWriter:
@@ -120,11 +173,8 @@ def _validate_output_pattern(output_pattern: str) -> None:
120
173
  def _open_validated_output_path(output_pattern: str, index: int, force: bool = False) -> Path:
121
174
  """Resolve one output path from a pattern and reject existing targets unless force."""
122
175
  output_path = Path(output_pattern % index)
123
- if output_path.exists():
124
- if force:
125
- output_path.unlink()
126
- else:
127
- raise FileExistsError(f"Output file already exists: {output_path}")
176
+ if output_path.exists() and not force:
177
+ raise FileExistsError(f"Output file already exists: {output_path}")
128
178
  return output_path
129
179
 
130
180
 
@@ -137,19 +187,27 @@ def _split_batches_to_files(
137
187
  progress_callback: Optional[Callable[[int, int], None]] = None,
138
188
  compression: Optional[str] = None,
139
189
  ) -> List[Path]:
140
- """Stream record batches into split output files with cleanup on failure."""
141
- created_output_paths: List[Path] = []
190
+ """Stream record batches into split output files with cleanup on failure.
191
+
192
+ Parts are written beside the destination and moved into place only after every
193
+ part has been closed, so a failed ``--force`` split leaves existing files alone.
194
+ """
195
+ staged_outputs: List[tuple[Path, Path]] = []
196
+ temp_paths: List[Path] = []
142
197
  current_writer: Optional[Any] = None
143
198
 
199
+ def open_staged_writer(final_path: Path) -> Any:
200
+ temp_path = _temporary_output_path(final_path)
201
+ temp_paths.append(temp_path)
202
+ staged_outputs.append((final_path, temp_path))
203
+ return _open_chunk_writer(temp_path, schema, compression=compression)
204
+
144
205
  try:
145
206
  current_file_idx = 0
146
207
  current_file_rows = 0
147
208
  target_rows_for_current_file = chunk_sizes[current_file_idx]
148
209
  rows_processed = 0
149
- current_writer = _open_chunk_writer(
150
- output_files[current_file_idx], schema, compression=compression
151
- )
152
- created_output_paths.append(output_files[current_file_idx])
210
+ current_writer = open_staged_writer(output_files[current_file_idx])
153
211
 
154
212
  for batch in batches:
155
213
  batch_offset = 0
@@ -178,16 +236,16 @@ def _split_batches_to_files(
178
236
  current_file_idx += 1
179
237
  current_file_rows = 0
180
238
  target_rows_for_current_file = chunk_sizes[current_file_idx]
181
- current_writer = _open_chunk_writer(
182
- output_files[current_file_idx], schema, compression=compression
183
- )
184
- created_output_paths.append(output_files[current_file_idx])
239
+ current_writer = open_staged_writer(output_files[current_file_idx])
185
240
 
186
241
  if current_writer is not None:
187
242
  current_writer.close()
188
243
  current_writer = None
244
+ _publish_staged_outputs(staged_outputs)
189
245
  except Exception:
190
- _cleanup_split_files(current_writer, created_output_paths)
246
+ held_writers = [current_writer]
247
+ current_writer = None
248
+ _cleanup_split_files(held_writers, temp_paths)
191
249
  raise
192
250
 
193
251
  return output_files
@@ -207,7 +265,8 @@ def _stream_split_by_record_count(
207
265
  raise ValueError("record_count must be positive")
208
266
 
209
267
  _validate_output_pattern(output_pattern)
210
- created_output_paths: List[Path] = []
268
+ staged_outputs: List[tuple[Path, Path]] = []
269
+ temp_paths: List[Path] = []
211
270
  current_writer: Optional[Any] = None
212
271
  current_rows = 0
213
272
  rows_processed = 0
@@ -221,10 +280,12 @@ def _stream_split_by_record_count(
221
280
  output_path = _open_validated_output_path(
222
281
  output_pattern, current_file_idx, force=force
223
282
  )
283
+ temp_path = _temporary_output_path(output_path)
284
+ temp_paths.append(temp_path)
285
+ staged_outputs.append((output_path, temp_path))
224
286
  current_writer = _open_chunk_writer(
225
- output_path, schema, compression=compression
287
+ temp_path, schema, compression=compression
226
288
  )
227
- created_output_paths.append(output_path)
228
289
  current_rows = 0
229
290
 
230
291
  rows_remaining = record_count - current_rows
@@ -252,13 +313,34 @@ def _stream_split_by_record_count(
252
313
  if current_writer is not None:
253
314
  current_writer.close()
254
315
  current_writer = None
316
+ if rows_processed == 0:
317
+ raise ValueError("Cannot split empty file")
318
+ _publish_staged_outputs(staged_outputs)
255
319
  except Exception:
256
- _cleanup_split_files(current_writer, created_output_paths)
320
+ held_writers = [current_writer]
321
+ current_writer = None
322
+ _cleanup_split_files(held_writers, temp_paths)
257
323
  raise
258
324
 
259
- if rows_processed == 0:
260
- raise ValueError("Cannot split empty file")
261
- return created_output_paths, rows_processed
325
+ return [final_path for final_path, _temp_path in staged_outputs], rows_processed
326
+
327
+
328
+ def _temporary_output_path(output_path: Path) -> Path:
329
+ """Create an empty temp path next to the destination so replace stays on one filesystem.
330
+
331
+ The temp file keeps the destination suffix. Writers choose a format from that suffix.
332
+ """
333
+ output_path.parent.mkdir(parents=True, exist_ok=True)
334
+ fd, tmp_name = tempfile.mkstemp(prefix=f".{output_path.name}.", dir=output_path.parent)
335
+ os.close(fd)
336
+ raw_path = Path(tmp_name)
337
+ if not output_path.suffix:
338
+ raw_path.unlink()
339
+ return raw_path
340
+ temporary_path = raw_path.with_name(raw_path.name + output_path.suffix)
341
+ raw_path.rename(temporary_path)
342
+ temporary_path.unlink()
343
+ return temporary_path
262
344
 
263
345
 
264
346
  def _write_batches_to_output(
@@ -270,31 +352,30 @@ def _write_batches_to_output(
270
352
  progress_callback: Optional[Callable[[int, int], None]] = None,
271
353
  total_rows: int = 0,
272
354
  ) -> int:
273
- """Write a batch iterator to a single output file."""
274
- if output_path.exists():
275
- if force:
276
- output_path.unlink()
277
- else:
278
- raise FileExistsError(f"Output file already exists: {output_path}")
279
-
280
- writer = _open_chunk_writer(output_path, schema, compression=compression)
281
- rows_written = 0
355
+ """Write a batch iterator to a single output file.
356
+
357
+ The destination is replaced only after the temp file is complete, so a failed
358
+ ``--force`` write leaves the previous file in place.
359
+ """
360
+ if output_path.exists() and not force:
361
+ raise FileExistsError(f"Output file already exists: {output_path}")
362
+
363
+ tmp_path = _temporary_output_path(output_path)
364
+ writer: Any = None
282
365
  try:
366
+ writer = _open_chunk_writer(tmp_path, schema, compression=compression)
367
+ rows_written = 0
283
368
  for batch in batches:
284
369
  writer.write_batch(batch)
285
370
  rows_written += len(batch)
286
371
  if progress_callback is not None:
287
372
  progress_callback(rows_written, total_rows)
373
+ writer.close()
374
+ writer = None
375
+ os.replace(tmp_path, output_path)
376
+ return rows_written
288
377
  except Exception:
289
- try:
290
- writer.close()
291
- except Exception:
292
- pass
293
- try:
294
- output_path.unlink()
295
- except (FileNotFoundError, OSError):
296
- pass
378
+ held_writers = [writer]
379
+ writer = None
380
+ _cleanup_split_files(held_writers, [tmp_path])
297
381
  raise
298
-
299
- writer.close()
300
- return rows_written