parq-cli 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parq_cli-0.2.2 → parq_cli-0.2.3}/CHANGELOG.md +15 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/PKG-INFO +1 -1
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/cli.py +100 -102
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_chunk_writers.py +133 -52
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_common.py +44 -7
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_csv.py +3 -1
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_parquet.py +6 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/plain_output.py +4 -1
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/reader.py +210 -105
- {parq_cli-0.2.2 → parq_cli-0.2.3}/pyproject.toml +1 -1
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_cli.py +59 -0
- parq_cli-0.2.3/tests/test_output_not_input.py +63 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_plain_output.py +18 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_reader.py +220 -1
- {parq_cli-0.2.2 → parq_cli-0.2.3}/uv.lock +1 -1
- parq_cli-0.2.2/tests/test_output_not_input.py +0 -26
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.claude/settings.json +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.github/workflows/publish.yml +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.github/workflows/test.yml +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.gitignore +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.python-version +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/csv-large.csv +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/csv-small.csv +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/parquet-large.parquet +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/.tmp-bench/parquet-small.parquet +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/AGENTS.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/CLAUDE.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/LICENSE +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/README.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/README_CN.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/data/test.parquet +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/docs/QUICKSTART.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/docs/RELEASE.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/examples/README.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/examples/create_sample_data.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/__init__.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/__main__.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/__init__.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/formats/_xlsx.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/parq/output.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/README.md +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/bump_version.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/check_version.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/performance_benchmark.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/scripts/publish.sh +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/__init__.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/conftest.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_main_entry.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_output.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_package_meta.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_performance.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_release_scripts.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_workflows.py +0 -0
- {parq_cli-0.2.2 → parq_cli-0.2.3}/tests/test_xlsx_defused.py +0 -0
|
@@ -1,5 +1,20 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
## 0.2.3
|
|
6
|
+
|
|
7
|
+
- `merge --force` and `convert --force` replace an existing output file only after the new file is fully written. A schema mismatch or a write error leaves the previous file in place, and the temporary file is removed, including on Windows.
|
|
8
|
+
- `diff` treats NaN as equal to NaN, including when NaN is the key.
|
|
9
|
+
- `diff` and `merge` close Parquet file handles when they finish.
|
|
10
|
+
- JSON table output summarizes binary values, matching rich and plain output.
|
|
11
|
+
- `split --force` replaces existing part files only after every part is written. A later write error leaves the previous parts in place.
|
|
12
|
+
- Column stats ignore NaN when computing min, max, and mean. Nulls are still counted separately, and NaN stays in the non-null count.
|
|
13
|
+
- An empty, multi-character, or non-ASCII `--delimiter` is rejected with `delimiter must be a single ASCII character` before CSV parsing or writing.
|
|
14
|
+
- `stats --limit`, `stats --top-n`, and `diff --limit` reject negative values instead of slicing from the end. Zero still means an empty result.
|
|
15
|
+
- Selecting a duplicated column name reports that the name cannot be selected, instead of claiming the column does not exist. A unique column in the same file can still be selected.
|
|
16
|
+
- `diff` compares repeated column names by position. A diff that would compare those names by default asks for `--columns` instead of claiming the column does not exist.
|
|
17
|
+
|
|
3
18
|
## 0.2.2
|
|
4
19
|
|
|
5
20
|
- `merge --force` and `convert` now refuse an output path that is one of the inputs. Previously `--force` deleted that input before reading it.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: parq-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: A command-line tool for inspecting tabular files: Parquet, CSV, TSV, and XLSX
|
|
5
5
|
Project-URL: Homepage, https://github.com/Tendo33/parq-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/Tendo33/parq-cli
|
|
@@ -207,8 +207,8 @@ def meta(
|
|
|
207
207
|
_apply_command_options(ctx, output_format, delimiter, sheet)
|
|
208
208
|
|
|
209
209
|
def operation(formatter: Any) -> None:
|
|
210
|
-
|
|
211
|
-
|
|
210
|
+
with _get_reader(str(file), _get_state(ctx)) as reader:
|
|
211
|
+
formatter.print_metadata(reader.get_metadata_dict(fast=fast))
|
|
212
212
|
|
|
213
213
|
_run_with_error_handling(
|
|
214
214
|
ctx, operation, generic_error_prefix="Failed to read file", file_path=file
|
|
@@ -227,8 +227,8 @@ def schema(
|
|
|
227
227
|
_apply_command_options(ctx, output_format, delimiter, sheet)
|
|
228
228
|
|
|
229
229
|
def operation(formatter: Any) -> None:
|
|
230
|
-
|
|
231
|
-
|
|
230
|
+
with _get_reader(str(file), _get_state(ctx)) as reader:
|
|
231
|
+
formatter.print_schema(reader.get_schema_info())
|
|
232
232
|
|
|
233
233
|
_run_with_error_handling(
|
|
234
234
|
ctx, operation, generic_error_prefix="Failed to read file", file_path=file
|
|
@@ -252,9 +252,9 @@ def head(
|
|
|
252
252
|
_apply_command_options(ctx, output_format, delimiter, sheet)
|
|
253
253
|
|
|
254
254
|
def operation(formatter: Any) -> None:
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
255
|
+
with _get_reader(str(file), _get_state(ctx)) as reader:
|
|
256
|
+
table = reader.read_head(n, columns=_parse_column_list(columns))
|
|
257
|
+
formatter.print_table(table, f"First {n} Rows")
|
|
258
258
|
|
|
259
259
|
_run_with_error_handling(
|
|
260
260
|
ctx, operation, generic_error_prefix="Failed to read file", file_path=file
|
|
@@ -278,9 +278,9 @@ def tail(
|
|
|
278
278
|
_apply_command_options(ctx, output_format, delimiter, sheet)
|
|
279
279
|
|
|
280
280
|
def operation(formatter: Any) -> None:
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
281
|
+
with _get_reader(str(file), _get_state(ctx)) as reader:
|
|
282
|
+
table = reader.read_tail(n, columns=_parse_column_list(columns))
|
|
283
|
+
formatter.print_table(table, f"Last {n} Rows")
|
|
284
284
|
|
|
285
285
|
_run_with_error_handling(
|
|
286
286
|
ctx, operation, generic_error_prefix="Failed to read file", file_path=file
|
|
@@ -299,8 +299,8 @@ def count(
|
|
|
299
299
|
_apply_command_options(ctx, output_format, delimiter, sheet)
|
|
300
300
|
|
|
301
301
|
def operation(formatter: Any) -> None:
|
|
302
|
-
|
|
303
|
-
|
|
302
|
+
with _get_reader(str(file), _get_state(ctx)) as reader:
|
|
303
|
+
formatter.print_count(reader.num_rows)
|
|
304
304
|
|
|
305
305
|
_run_with_error_handling(
|
|
306
306
|
ctx, operation, generic_error_prefix="Failed to read file", file_path=file
|
|
@@ -352,60 +352,59 @@ def split(
|
|
|
352
352
|
import time
|
|
353
353
|
|
|
354
354
|
start_time = time.time()
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
progress.update(task,
|
|
380
|
-
|
|
381
|
-
|
|
355
|
+
with _get_reader(str(file), state) as reader:
|
|
356
|
+
if state.output_format == OutputFormat.RICH:
|
|
357
|
+
from rich.progress import (
|
|
358
|
+
BarColumn,
|
|
359
|
+
Progress,
|
|
360
|
+
SpinnerColumn,
|
|
361
|
+
TextColumn,
|
|
362
|
+
TimeRemainingColumn,
|
|
363
|
+
)
|
|
364
|
+
from parq.output import console
|
|
365
|
+
|
|
366
|
+
with Progress(
|
|
367
|
+
SpinnerColumn(),
|
|
368
|
+
TextColumn("[bold blue]{task.description}"),
|
|
369
|
+
BarColumn(),
|
|
370
|
+
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
371
|
+
TimeRemainingColumn(),
|
|
372
|
+
console=console,
|
|
373
|
+
) as progress:
|
|
374
|
+
task = progress.add_task(f"[cyan]Splitting {file.name}...", total=None)
|
|
375
|
+
|
|
376
|
+
def update_progress(current: int, total: int):
|
|
377
|
+
if progress.tasks[task].total is None and total:
|
|
378
|
+
progress.update(task, total=total)
|
|
379
|
+
progress.update(task, completed=current)
|
|
380
|
+
|
|
381
|
+
output_files = reader.split_file(
|
|
382
|
+
output_pattern=name_format,
|
|
383
|
+
file_count=file_count,
|
|
384
|
+
record_count=record_count,
|
|
385
|
+
progress_callback=update_progress,
|
|
386
|
+
force=force,
|
|
387
|
+
)
|
|
388
|
+
else:
|
|
382
389
|
output_files = reader.split_file(
|
|
383
390
|
output_pattern=name_format,
|
|
384
391
|
file_count=file_count,
|
|
385
392
|
record_count=record_count,
|
|
386
|
-
progress_callback=update_progress,
|
|
387
393
|
force=force,
|
|
388
394
|
)
|
|
389
|
-
else:
|
|
390
|
-
output_files = reader.split_file(
|
|
391
|
-
output_pattern=name_format,
|
|
392
|
-
file_count=file_count,
|
|
393
|
-
record_count=record_count,
|
|
394
|
-
force=force,
|
|
395
|
-
)
|
|
396
395
|
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
396
|
+
elapsed_time = time.time() - start_time
|
|
397
|
+
total_rows = (
|
|
398
|
+
reader.last_split_total_rows
|
|
399
|
+
if reader.last_split_total_rows is not None
|
|
400
|
+
else reader.num_rows
|
|
401
|
+
)
|
|
402
|
+
formatter.print_split_result(
|
|
403
|
+
source_file=file,
|
|
404
|
+
output_files=output_files,
|
|
405
|
+
total_rows=total_rows,
|
|
406
|
+
elapsed_time=elapsed_time,
|
|
407
|
+
)
|
|
409
408
|
except typer.Exit:
|
|
410
409
|
raise
|
|
411
410
|
except (FileNotFoundError, ValueError, FileExistsError) as e:
|
|
@@ -437,10 +436,10 @@ def stats(
|
|
|
437
436
|
_apply_command_options(ctx, output_format, delimiter, sheet)
|
|
438
437
|
|
|
439
438
|
def operation(formatter: Any) -> None:
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
439
|
+
with _get_reader(str(file), _get_state(ctx)) as reader:
|
|
440
|
+
formatter.print_stats(
|
|
441
|
+
reader.get_stats(columns=_parse_column_list(columns), limit=limit, top_n=top_n)
|
|
442
|
+
)
|
|
444
443
|
|
|
445
444
|
_run_with_error_handling(
|
|
446
445
|
ctx, operation, generic_error_prefix="Failed to compute stats", file_path=file
|
|
@@ -476,52 +475,51 @@ def convert(
|
|
|
476
475
|
import time
|
|
477
476
|
|
|
478
477
|
start_time = time.time()
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
TimeRemainingColumn,
|
|
488
|
-
)
|
|
489
|
-
from parq.output import console
|
|
490
|
-
|
|
491
|
-
total_rows_hint = reader.num_rows if reader.input_format == "parquet" else 0
|
|
492
|
-
|
|
493
|
-
with Progress(
|
|
494
|
-
SpinnerColumn(),
|
|
495
|
-
TextColumn("[bold blue]{task.description}"),
|
|
496
|
-
BarColumn(),
|
|
497
|
-
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
498
|
-
TimeRemainingColumn(),
|
|
499
|
-
console=console,
|
|
500
|
-
) as progress:
|
|
501
|
-
task = progress.add_task(
|
|
502
|
-
f"[cyan]Converting {source.name}...",
|
|
503
|
-
total=total_rows_hint if total_rows_hint > 0 else None,
|
|
478
|
+
with _get_reader(str(source), state) as reader:
|
|
479
|
+
if state.output_format == OutputFormat.RICH:
|
|
480
|
+
from rich.progress import (
|
|
481
|
+
Progress,
|
|
482
|
+
SpinnerColumn,
|
|
483
|
+
TextColumn,
|
|
484
|
+
BarColumn,
|
|
485
|
+
TimeRemainingColumn,
|
|
504
486
|
)
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
487
|
+
from parq.output import console
|
|
488
|
+
|
|
489
|
+
total_rows_hint = reader.num_rows if reader.input_format == "parquet" else 0
|
|
490
|
+
|
|
491
|
+
with Progress(
|
|
492
|
+
SpinnerColumn(),
|
|
493
|
+
TextColumn("[bold blue]{task.description}"),
|
|
494
|
+
BarColumn(),
|
|
495
|
+
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
|
496
|
+
TimeRemainingColumn(),
|
|
497
|
+
console=console,
|
|
498
|
+
) as progress:
|
|
499
|
+
task = progress.add_task(
|
|
500
|
+
f"[cyan]Converting {source.name}...",
|
|
501
|
+
total=total_rows_hint if total_rows_hint > 0 else None,
|
|
502
|
+
)
|
|
503
|
+
|
|
504
|
+
def update_progress(current: int, total: int) -> None:
|
|
505
|
+
if progress.tasks[task].total is None and total:
|
|
506
|
+
progress.update(task, total=total)
|
|
507
|
+
progress.update(task, completed=current)
|
|
508
|
+
|
|
509
|
+
total_rows = reader.convert_file(
|
|
510
|
+
output,
|
|
511
|
+
columns=_parse_column_list(columns),
|
|
512
|
+
force=force,
|
|
513
|
+
progress_callback=update_progress,
|
|
514
|
+
)
|
|
515
|
+
else:
|
|
511
516
|
total_rows = reader.convert_file(
|
|
512
517
|
output,
|
|
513
518
|
columns=_parse_column_list(columns),
|
|
514
519
|
force=force,
|
|
515
|
-
progress_callback=update_progress,
|
|
516
520
|
)
|
|
517
|
-
else:
|
|
518
|
-
total_rows = reader.convert_file(
|
|
519
|
-
output,
|
|
520
|
-
columns=_parse_column_list(columns),
|
|
521
|
-
force=force,
|
|
522
|
-
)
|
|
523
521
|
|
|
524
|
-
|
|
522
|
+
formatter.print_convert_result(source, output, total_rows, time.time() - start_time)
|
|
525
523
|
except typer.Exit:
|
|
526
524
|
raise
|
|
527
525
|
except (FileNotFoundError, ValueError, FileExistsError) as e:
|
|
@@ -2,6 +2,9 @@
|
|
|
2
2
|
Chunk writers and streaming split/write utilities for all output formats.
|
|
3
3
|
"""
|
|
4
4
|
|
|
5
|
+
import gc
|
|
6
|
+
import os
|
|
7
|
+
import tempfile
|
|
5
8
|
from pathlib import Path
|
|
6
9
|
from typing import Any, Callable, Iterable, List, Optional
|
|
7
10
|
|
|
@@ -9,6 +12,7 @@ import pyarrow as pa
|
|
|
9
12
|
import pyarrow.csv as pacsv
|
|
10
13
|
import pyarrow.parquet as pq
|
|
11
14
|
|
|
15
|
+
from parq.formats._common import _validate_delimiter
|
|
12
16
|
from parq.formats._xlsx import _require_openpyxl
|
|
13
17
|
|
|
14
18
|
|
|
@@ -19,24 +23,70 @@ def _coerce_output_value(value: Any) -> Any:
|
|
|
19
23
|
return value
|
|
20
24
|
|
|
21
25
|
|
|
22
|
-
def
|
|
23
|
-
"""Close a writer
|
|
24
|
-
|
|
26
|
+
def _release_writer(writer: Any) -> None:
|
|
27
|
+
"""Close a writer. PyArrow's own writer is closed and dropped here.
|
|
28
|
+
|
|
29
|
+
On Windows the file stays locked until that object is destroyed, not merely
|
|
30
|
+
until ``close()`` returns.
|
|
31
|
+
"""
|
|
32
|
+
if writer is None:
|
|
33
|
+
return
|
|
34
|
+
inner = getattr(writer, "_writer", None)
|
|
35
|
+
if inner is not None:
|
|
36
|
+
writer._writer = None
|
|
25
37
|
try:
|
|
26
|
-
|
|
38
|
+
inner.close()
|
|
27
39
|
except Exception:
|
|
28
40
|
pass
|
|
41
|
+
del inner
|
|
42
|
+
return
|
|
43
|
+
try:
|
|
44
|
+
writer.close()
|
|
45
|
+
except Exception:
|
|
46
|
+
pass
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _unlink_output(path: Path) -> None:
|
|
50
|
+
"""Delete a staged file. A second try runs after GC, for Windows file locks."""
|
|
51
|
+
try:
|
|
52
|
+
path.unlink()
|
|
53
|
+
return
|
|
54
|
+
except FileNotFoundError:
|
|
55
|
+
return
|
|
56
|
+
except OSError:
|
|
57
|
+
pass
|
|
58
|
+
gc.collect()
|
|
59
|
+
try:
|
|
60
|
+
path.unlink()
|
|
61
|
+
except (FileNotFoundError, OSError):
|
|
62
|
+
pass
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _cleanup_split_files(writers: List[Any], created_paths: List[Path]) -> None:
|
|
66
|
+
"""Close writers and delete staged temp files on failure.
|
|
67
|
+
|
|
68
|
+
The caller must put its only reference in ``writers`` and then drop its own
|
|
69
|
+
variable. This function clears that list before deleting the files.
|
|
70
|
+
"""
|
|
71
|
+
for index in range(len(writers)):
|
|
72
|
+
_release_writer(writers[index])
|
|
73
|
+
writers[index] = None
|
|
74
|
+
gc.collect()
|
|
29
75
|
for path in created_paths:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
76
|
+
_unlink_output(path)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _publish_staged_outputs(staged: List[tuple[Path, Path]]) -> None:
|
|
80
|
+
"""Move staged split parts onto their final paths after every part is complete."""
|
|
81
|
+
for final_path, temp_path in staged:
|
|
82
|
+
os.replace(temp_path, final_path)
|
|
34
83
|
|
|
35
84
|
|
|
36
85
|
class _CsvChunkWriter:
|
|
37
86
|
"""Incremental CSV/TSV writer backed by PyArrow CSVWriter."""
|
|
38
87
|
|
|
39
88
|
def __init__(self, output_path: Path, schema: pa.Schema, delimiter: str = ","):
|
|
89
|
+
_validate_delimiter(delimiter)
|
|
40
90
|
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
41
91
|
self._writer = pacsv.CSVWriter(
|
|
42
92
|
output_path,
|
|
@@ -48,7 +98,10 @@ class _CsvChunkWriter:
|
|
|
48
98
|
self._writer.write(batch)
|
|
49
99
|
|
|
50
100
|
def close(self) -> None:
|
|
51
|
-
self._writer
|
|
101
|
+
writer = self._writer
|
|
102
|
+
self._writer = None
|
|
103
|
+
if writer is not None:
|
|
104
|
+
writer.close()
|
|
52
105
|
|
|
53
106
|
|
|
54
107
|
class _XlsxChunkWriter:
|
|
@@ -120,11 +173,8 @@ def _validate_output_pattern(output_pattern: str) -> None:
|
|
|
120
173
|
def _open_validated_output_path(output_pattern: str, index: int, force: bool = False) -> Path:
|
|
121
174
|
"""Resolve one output path from a pattern and reject existing targets unless force."""
|
|
122
175
|
output_path = Path(output_pattern % index)
|
|
123
|
-
if output_path.exists():
|
|
124
|
-
|
|
125
|
-
output_path.unlink()
|
|
126
|
-
else:
|
|
127
|
-
raise FileExistsError(f"Output file already exists: {output_path}")
|
|
176
|
+
if output_path.exists() and not force:
|
|
177
|
+
raise FileExistsError(f"Output file already exists: {output_path}")
|
|
128
178
|
return output_path
|
|
129
179
|
|
|
130
180
|
|
|
@@ -137,19 +187,27 @@ def _split_batches_to_files(
|
|
|
137
187
|
progress_callback: Optional[Callable[[int, int], None]] = None,
|
|
138
188
|
compression: Optional[str] = None,
|
|
139
189
|
) -> List[Path]:
|
|
140
|
-
"""Stream record batches into split output files with cleanup on failure.
|
|
141
|
-
|
|
190
|
+
"""Stream record batches into split output files with cleanup on failure.
|
|
191
|
+
|
|
192
|
+
Parts are written beside the destination and moved into place only after every
|
|
193
|
+
part has been closed, so a failed ``--force`` split leaves existing files alone.
|
|
194
|
+
"""
|
|
195
|
+
staged_outputs: List[tuple[Path, Path]] = []
|
|
196
|
+
temp_paths: List[Path] = []
|
|
142
197
|
current_writer: Optional[Any] = None
|
|
143
198
|
|
|
199
|
+
def open_staged_writer(final_path: Path) -> Any:
|
|
200
|
+
temp_path = _temporary_output_path(final_path)
|
|
201
|
+
temp_paths.append(temp_path)
|
|
202
|
+
staged_outputs.append((final_path, temp_path))
|
|
203
|
+
return _open_chunk_writer(temp_path, schema, compression=compression)
|
|
204
|
+
|
|
144
205
|
try:
|
|
145
206
|
current_file_idx = 0
|
|
146
207
|
current_file_rows = 0
|
|
147
208
|
target_rows_for_current_file = chunk_sizes[current_file_idx]
|
|
148
209
|
rows_processed = 0
|
|
149
|
-
current_writer =
|
|
150
|
-
output_files[current_file_idx], schema, compression=compression
|
|
151
|
-
)
|
|
152
|
-
created_output_paths.append(output_files[current_file_idx])
|
|
210
|
+
current_writer = open_staged_writer(output_files[current_file_idx])
|
|
153
211
|
|
|
154
212
|
for batch in batches:
|
|
155
213
|
batch_offset = 0
|
|
@@ -178,16 +236,16 @@ def _split_batches_to_files(
|
|
|
178
236
|
current_file_idx += 1
|
|
179
237
|
current_file_rows = 0
|
|
180
238
|
target_rows_for_current_file = chunk_sizes[current_file_idx]
|
|
181
|
-
current_writer =
|
|
182
|
-
output_files[current_file_idx], schema, compression=compression
|
|
183
|
-
)
|
|
184
|
-
created_output_paths.append(output_files[current_file_idx])
|
|
239
|
+
current_writer = open_staged_writer(output_files[current_file_idx])
|
|
185
240
|
|
|
186
241
|
if current_writer is not None:
|
|
187
242
|
current_writer.close()
|
|
188
243
|
current_writer = None
|
|
244
|
+
_publish_staged_outputs(staged_outputs)
|
|
189
245
|
except Exception:
|
|
190
|
-
|
|
246
|
+
held_writers = [current_writer]
|
|
247
|
+
current_writer = None
|
|
248
|
+
_cleanup_split_files(held_writers, temp_paths)
|
|
191
249
|
raise
|
|
192
250
|
|
|
193
251
|
return output_files
|
|
@@ -207,7 +265,8 @@ def _stream_split_by_record_count(
|
|
|
207
265
|
raise ValueError("record_count must be positive")
|
|
208
266
|
|
|
209
267
|
_validate_output_pattern(output_pattern)
|
|
210
|
-
|
|
268
|
+
staged_outputs: List[tuple[Path, Path]] = []
|
|
269
|
+
temp_paths: List[Path] = []
|
|
211
270
|
current_writer: Optional[Any] = None
|
|
212
271
|
current_rows = 0
|
|
213
272
|
rows_processed = 0
|
|
@@ -221,10 +280,12 @@ def _stream_split_by_record_count(
|
|
|
221
280
|
output_path = _open_validated_output_path(
|
|
222
281
|
output_pattern, current_file_idx, force=force
|
|
223
282
|
)
|
|
283
|
+
temp_path = _temporary_output_path(output_path)
|
|
284
|
+
temp_paths.append(temp_path)
|
|
285
|
+
staged_outputs.append((output_path, temp_path))
|
|
224
286
|
current_writer = _open_chunk_writer(
|
|
225
|
-
|
|
287
|
+
temp_path, schema, compression=compression
|
|
226
288
|
)
|
|
227
|
-
created_output_paths.append(output_path)
|
|
228
289
|
current_rows = 0
|
|
229
290
|
|
|
230
291
|
rows_remaining = record_count - current_rows
|
|
@@ -252,13 +313,34 @@ def _stream_split_by_record_count(
|
|
|
252
313
|
if current_writer is not None:
|
|
253
314
|
current_writer.close()
|
|
254
315
|
current_writer = None
|
|
316
|
+
if rows_processed == 0:
|
|
317
|
+
raise ValueError("Cannot split empty file")
|
|
318
|
+
_publish_staged_outputs(staged_outputs)
|
|
255
319
|
except Exception:
|
|
256
|
-
|
|
320
|
+
held_writers = [current_writer]
|
|
321
|
+
current_writer = None
|
|
322
|
+
_cleanup_split_files(held_writers, temp_paths)
|
|
257
323
|
raise
|
|
258
324
|
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
325
|
+
return [final_path for final_path, _temp_path in staged_outputs], rows_processed
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
def _temporary_output_path(output_path: Path) -> Path:
|
|
329
|
+
"""Create an empty temp path next to the destination so replace stays on one filesystem.
|
|
330
|
+
|
|
331
|
+
The temp file keeps the destination suffix. Writers choose a format from that suffix.
|
|
332
|
+
"""
|
|
333
|
+
output_path.parent.mkdir(parents=True, exist_ok=True)
|
|
334
|
+
fd, tmp_name = tempfile.mkstemp(prefix=f".{output_path.name}.", dir=output_path.parent)
|
|
335
|
+
os.close(fd)
|
|
336
|
+
raw_path = Path(tmp_name)
|
|
337
|
+
if not output_path.suffix:
|
|
338
|
+
raw_path.unlink()
|
|
339
|
+
return raw_path
|
|
340
|
+
temporary_path = raw_path.with_name(raw_path.name + output_path.suffix)
|
|
341
|
+
raw_path.rename(temporary_path)
|
|
342
|
+
temporary_path.unlink()
|
|
343
|
+
return temporary_path
|
|
262
344
|
|
|
263
345
|
|
|
264
346
|
def _write_batches_to_output(
|
|
@@ -270,31 +352,30 @@ def _write_batches_to_output(
|
|
|
270
352
|
progress_callback: Optional[Callable[[int, int], None]] = None,
|
|
271
353
|
total_rows: int = 0,
|
|
272
354
|
) -> int:
|
|
273
|
-
"""Write a batch iterator to a single output file.
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
355
|
+
"""Write a batch iterator to a single output file.
|
|
356
|
+
|
|
357
|
+
The destination is replaced only after the temp file is complete, so a failed
|
|
358
|
+
``--force`` write leaves the previous file in place.
|
|
359
|
+
"""
|
|
360
|
+
if output_path.exists() and not force:
|
|
361
|
+
raise FileExistsError(f"Output file already exists: {output_path}")
|
|
362
|
+
|
|
363
|
+
tmp_path = _temporary_output_path(output_path)
|
|
364
|
+
writer: Any = None
|
|
282
365
|
try:
|
|
366
|
+
writer = _open_chunk_writer(tmp_path, schema, compression=compression)
|
|
367
|
+
rows_written = 0
|
|
283
368
|
for batch in batches:
|
|
284
369
|
writer.write_batch(batch)
|
|
285
370
|
rows_written += len(batch)
|
|
286
371
|
if progress_callback is not None:
|
|
287
372
|
progress_callback(rows_written, total_rows)
|
|
373
|
+
writer.close()
|
|
374
|
+
writer = None
|
|
375
|
+
os.replace(tmp_path, output_path)
|
|
376
|
+
return rows_written
|
|
288
377
|
except Exception:
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
pass
|
|
293
|
-
try:
|
|
294
|
-
output_path.unlink()
|
|
295
|
-
except (FileNotFoundError, OSError):
|
|
296
|
-
pass
|
|
378
|
+
held_writers = [writer]
|
|
379
|
+
writer = None
|
|
380
|
+
_cleanup_split_files(held_writers, [tmp_path])
|
|
297
381
|
raise
|
|
298
|
-
|
|
299
|
-
writer.close()
|
|
300
|
-
return rows_written
|