parq-cli 0.2.4__tar.gz → 0.2.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {parq_cli-0.2.4 → parq_cli-0.2.5}/CHANGELOG.md +7 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/PKG-INFO +1 -1
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/reader.py +128 -41
- {parq_cli-0.2.4 → parq_cli-0.2.5}/pyproject.toml +1 -1
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_reader.py +24 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/uv.lock +12 -1
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.claude/settings.json +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.github/workflows/publish.yml +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.github/workflows/test.yml +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.gitignore +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.python-version +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.tmp-bench/csv-large.csv +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.tmp-bench/csv-small.csv +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.tmp-bench/parquet-large.parquet +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/.tmp-bench/parquet-small.parquet +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/AGENTS.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/CLAUDE.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/LICENSE +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/README.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/README_CN.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/data/test.parquet +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/docs/QUICKSTART.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/docs/RELEASE.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/examples/README.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/examples/create_sample_data.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/__init__.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/__main__.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/cli.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/formats/__init__.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/formats/_chunk_writers.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/formats/_common.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/formats/_csv.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/formats/_parquet.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/formats/_xlsx.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/output.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/parq/plain_output.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/scripts/README.md +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/scripts/bump_version.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/scripts/check_version.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/scripts/performance_benchmark.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/scripts/publish.sh +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/__init__.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/conftest.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_cli.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_main_entry.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_output.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_output_not_input.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_package_meta.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_performance.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_plain_output.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_release_scripts.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_workflows.py +0 -0
- {parq_cli-0.2.4 → parq_cli-0.2.5}/tests/test_xlsx_defused.py +0 -0
|
@@ -2,6 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
## 0.2.5
|
|
6
|
+
|
|
7
|
+
- `diff` on inputs large enough to spill samples the smallest differing keys, matching what a smaller input reports. Each partition used to keep its own smallest keys and the merge kept whichever partition came first, so a spilled diff could report keys that were not the smallest in the file. Counts were never affected, only the `only_left` / `only_right` samples in `plain` and `json` output.
|
|
8
|
+
- `diff` partitions into half the row limit per bucket instead of aiming at the limit itself. Hash partitioning is uneven, so most buckets landed just over the threshold and partitioned a second time, rewriting the rows into temporary files two to three times. A 1,000,000-row diff against a 1,000,000-row file now takes about 5.1 s instead of 8.4 s on the same machine.
|
|
9
|
+
- `diff` reads rows as tuples rather than building a dict per row, and compares comparable values with plain tuple equality before falling back to the per-cell NaN-aware comparison. A key part is normalized once instead of twice.
|
|
10
|
+
- A partition still writes at most 32 buckets, so the smaller target only changes inputs below roughly 3.2 million rows: those split into twice as many, half-sized temporary files. At or above that size both limits clamp to 32 buckets and behave identically.
|
|
11
|
+
|
|
5
12
|
## 0.2.4
|
|
6
13
|
|
|
7
14
|
- CSV and TSV reads accept `--encoding` (default `utf8`). The flag works before a subcommand and on it, and the subcommand value wins. An unknown name fails with `Unknown encoding`. Writes stay UTF-8.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: parq-cli
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.5
|
|
4
4
|
Summary: A command-line tool for inspecting tabular files: Parquet, CSV, TSV, and XLSX
|
|
5
5
|
Project-URL: Homepage, https://github.com/Tendo33/parq-cli
|
|
6
6
|
Project-URL: Repository, https://github.com/Tendo33/parq-cli
|
|
@@ -60,6 +60,10 @@ __all__ = [
|
|
|
60
60
|
# Distinct keys kept in one in-memory diff index. Larger inputs are
|
|
61
61
|
# hash-partitioned into temporary Parquet files.
|
|
62
62
|
DIFF_MEMORY_ROW_LIMIT = 100_000
|
|
63
|
+
# Rows aimed for per partition. Hash partitioning is uneven, so aiming at the
|
|
64
|
+
# limit itself leaves most buckets a little over it and every one of them
|
|
65
|
+
# partitions again. Half the limit keeps each bucket clear of the threshold.
|
|
66
|
+
DIFF_PARTITION_TARGET_ROWS = DIFF_MEMORY_ROW_LIMIT // 2
|
|
63
67
|
|
|
64
68
|
|
|
65
69
|
class MultiFormatReader:
|
|
@@ -512,15 +516,22 @@ def _reader_batch_iterator(
|
|
|
512
516
|
yield from reader._iter_input_batches(columns=columns)
|
|
513
517
|
|
|
514
518
|
|
|
515
|
-
def
|
|
519
|
+
def _iter_row_tuples(
|
|
516
520
|
reader: MultiFormatReader,
|
|
517
521
|
columns: List[str],
|
|
518
|
-
) -> Iterator[
|
|
519
|
-
"""Iterate selected rows as
|
|
522
|
+
) -> Iterator[tuple[Any, ...]]:
|
|
523
|
+
"""Iterate selected rows as tuples in ``columns`` order.
|
|
524
|
+
|
|
525
|
+
``zip`` splits the columns into rows in C, which is far cheaper than a
|
|
526
|
+
dictionary comprehension per row. Dict-based callers convert from this.
|
|
527
|
+
"""
|
|
520
528
|
for batch in _reader_batch_iterator(reader, columns=columns):
|
|
521
529
|
batch_dict = batch.to_pydict()
|
|
522
|
-
for
|
|
523
|
-
|
|
530
|
+
column_values = [batch_dict[column] for column in columns]
|
|
531
|
+
if not column_values:
|
|
532
|
+
yield from (() for _ in range(len(batch)))
|
|
533
|
+
continue
|
|
534
|
+
yield from zip(*column_values)
|
|
524
535
|
|
|
525
536
|
|
|
526
537
|
def _build_diff_index(
|
|
@@ -531,14 +542,16 @@ def _build_diff_index(
|
|
|
531
542
|
side_name: str,
|
|
532
543
|
) -> dict[tuple[Any, ...], dict[str, Any]]:
|
|
533
544
|
"""Build a keyed row index for diffing and reject duplicate keys."""
|
|
534
|
-
|
|
535
|
-
for
|
|
536
|
-
|
|
545
|
+
key_positions = [selected_columns.index(column) for column in key_columns]
|
|
546
|
+
comparable_positions = [selected_columns.index(column) for column in comparable_columns]
|
|
547
|
+
index: dict[tuple[Any, ...], tuple[Any, ...]] = {}
|
|
548
|
+
for row in _iter_row_tuples(reader, selected_columns):
|
|
549
|
+
key = _row_key_at(row, key_positions, key_columns)
|
|
537
550
|
if key in index:
|
|
538
551
|
raise ValueError(
|
|
539
552
|
f"Duplicate key found in {side_name} input: {_render_key(key)}"
|
|
540
553
|
)
|
|
541
|
-
index[key] =
|
|
554
|
+
index[key] = tuple(row[position] for position in comparable_positions)
|
|
542
555
|
return index
|
|
543
556
|
|
|
544
557
|
|
|
@@ -609,15 +622,28 @@ _DECIMAL_NAN_KEY = _DecimalNaNKey()
|
|
|
609
622
|
|
|
610
623
|
def _normalize_key_part(part: Any) -> Any:
|
|
611
624
|
"""Map values that should match as one key onto a stable, hashable form."""
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
if
|
|
617
|
-
return
|
|
625
|
+
# Keys are overwhelmingly strings, ints, or None. Testing the exact type
|
|
626
|
+
# first keeps that case to one check instead of a chain of isinstance
|
|
627
|
+
# calls, which cost more than everything else in a large diff.
|
|
628
|
+
kind = type(part)
|
|
629
|
+
if kind is float:
|
|
630
|
+
return _normalize_float_key(part)
|
|
631
|
+
if kind is Decimal:
|
|
632
|
+
return _DECIMAL_NAN_KEY if part.is_nan() else part
|
|
633
|
+
if isinstance(part, float):
|
|
634
|
+
return _normalize_float_key(part)
|
|
635
|
+
if isinstance(part, Decimal):
|
|
636
|
+
return _DECIMAL_NAN_KEY if part.is_nan() else part
|
|
618
637
|
return part
|
|
619
638
|
|
|
620
639
|
|
|
640
|
+
def _normalize_float_key(part: float) -> Any:
|
|
641
|
+
"""Fold NaN onto one sentinel and -0.0 onto 0.0."""
|
|
642
|
+
if part != part:
|
|
643
|
+
return _NAN_KEY
|
|
644
|
+
return 0.0 if part == 0 else part
|
|
645
|
+
|
|
646
|
+
|
|
621
647
|
def _row_key(row: dict[str, Any], key_columns: List[str]) -> tuple[Any, ...]:
|
|
622
648
|
"""Build a hashable key, treating NaN and signed zero as one value."""
|
|
623
649
|
parts = []
|
|
@@ -630,6 +656,22 @@ def _row_key(row: dict[str, Any], key_columns: List[str]) -> tuple[Any, ...]:
|
|
|
630
656
|
return tuple(parts)
|
|
631
657
|
|
|
632
658
|
|
|
659
|
+
def _row_key_at(
|
|
660
|
+
row: tuple[Any, ...],
|
|
661
|
+
key_positions: List[int],
|
|
662
|
+
key_columns: List[str],
|
|
663
|
+
) -> tuple[Any, ...]:
|
|
664
|
+
"""Build a hashable key from a row tuple, given the key positions."""
|
|
665
|
+
parts = []
|
|
666
|
+
for position, column in zip(key_positions, key_columns, strict=True):
|
|
667
|
+
part = _normalize_key_part(row[position])
|
|
668
|
+
if part.__class__ is list or part.__class__ is dict:
|
|
669
|
+
kind = "list" if part.__class__ is list else "struct"
|
|
670
|
+
raise ValueError(f"Key column '{column}' contains a {kind}, which cannot be a key")
|
|
671
|
+
parts.append(part)
|
|
672
|
+
return tuple(parts)
|
|
673
|
+
|
|
674
|
+
|
|
633
675
|
def _key_part_sort_token(part: Any) -> tuple[Any, ...]:
|
|
634
676
|
"""Order one key part without asking Python to compare unrelated types.
|
|
635
677
|
|
|
@@ -710,6 +752,16 @@ def _same_value(left: Any, right: Any) -> bool:
|
|
|
710
752
|
return left == right
|
|
711
753
|
|
|
712
754
|
|
|
755
|
+
def _row_dict(columns: List[str], values: tuple[Any, ...]) -> dict[str, Any]:
|
|
756
|
+
"""Render a comparable-values tuple as the dict the diff payload reports."""
|
|
757
|
+
return dict(zip(columns, values))
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def _same_row_values(left: tuple[Any, ...], right: tuple[Any, ...]) -> bool:
|
|
761
|
+
"""Compare two comparable-value tuples with NaN treated as equal."""
|
|
762
|
+
return all(_same_value(a, b) for a, b in zip(left, right))
|
|
763
|
+
|
|
764
|
+
|
|
713
765
|
def _is_widened_text(value: Any, text: Any) -> bool:
|
|
714
766
|
"""Return whether text is the string XLSX stores when it widens this value.
|
|
715
767
|
|
|
@@ -734,16 +786,13 @@ def _is_widened_text(value: Any, text: Any) -> bool:
|
|
|
734
786
|
return False
|
|
735
787
|
|
|
736
788
|
|
|
737
|
-
def _bucket_key(key: tuple[Any, ...]) -> tuple[Any, ...]:
|
|
738
|
-
"""Normalize parts that compare equal so they land in the same bucket."""
|
|
739
|
-
return tuple(_normalize_key_part(part) for part in key)
|
|
740
|
-
|
|
741
|
-
|
|
742
789
|
def _stable_bucket(key: tuple[Any, ...], num_buckets: int, salt: int) -> int:
|
|
743
790
|
"""Hash a key into a bucket without depending on PYTHONHASHSEED."""
|
|
744
791
|
import zlib
|
|
745
792
|
|
|
746
|
-
|
|
793
|
+
# ``key`` already came from ``_row_key``, so its parts are normalized and
|
|
794
|
+
# normalizing again is pure work.
|
|
795
|
+
payload = repr((salt, key)).encode("utf-8", errors="backslashreplace")
|
|
747
796
|
return zlib.crc32(payload) % num_buckets
|
|
748
797
|
|
|
749
798
|
|
|
@@ -786,21 +835,23 @@ def _diff_rows_in_memory(
|
|
|
786
835
|
indexed_side_name,
|
|
787
836
|
)
|
|
788
837
|
|
|
838
|
+
key_positions = [selected_columns.index(column) for column in key_columns]
|
|
839
|
+
comparable_positions = [selected_columns.index(column) for column in comparable_columns]
|
|
789
840
|
missing_from_indexed: List[tuple[Any, ...]] = []
|
|
790
841
|
missing_from_indexed_count = 0
|
|
791
842
|
changed_rows = []
|
|
792
843
|
changed_count = 0
|
|
793
844
|
seen_streamed_keys: set[tuple[Any, ...]] = set()
|
|
794
845
|
|
|
795
|
-
for row in
|
|
796
|
-
key =
|
|
846
|
+
for row in _iter_row_tuples(streamed_reader, selected_columns):
|
|
847
|
+
key = _row_key_at(row, key_positions, key_columns)
|
|
797
848
|
if key in seen_streamed_keys:
|
|
798
849
|
raise ValueError(
|
|
799
850
|
f"Duplicate key found in {streamed_side_name} input: {_render_key(key)}"
|
|
800
851
|
)
|
|
801
852
|
seen_streamed_keys.add(key)
|
|
802
853
|
|
|
803
|
-
streamed_values =
|
|
854
|
+
streamed_values = tuple(row[position] for position in comparable_positions)
|
|
804
855
|
if key not in indexed_rows:
|
|
805
856
|
missing_from_indexed_count += 1
|
|
806
857
|
if not summary_only and len(missing_from_indexed) < limit:
|
|
@@ -808,17 +859,19 @@ def _diff_rows_in_memory(
|
|
|
808
859
|
continue
|
|
809
860
|
|
|
810
861
|
indexed_values = indexed_rows.pop(key)
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
862
|
+
# Tuple equality settles the common case where every compared cell is
|
|
863
|
+
# identical; the per-cell comparison only runs when something differs,
|
|
864
|
+
# where it still has to treat NaN as equal to NaN.
|
|
865
|
+
if streamed_values != indexed_values and not _same_row_values(
|
|
866
|
+
streamed_values, indexed_values
|
|
814
867
|
):
|
|
815
868
|
changed_count += 1
|
|
816
869
|
if index_left:
|
|
817
|
-
left_values = indexed_values
|
|
818
|
-
right_values = streamed_values
|
|
870
|
+
left_values = _row_dict(comparable_columns, indexed_values)
|
|
871
|
+
right_values = _row_dict(comparable_columns, streamed_values)
|
|
819
872
|
else:
|
|
820
|
-
left_values = streamed_values
|
|
821
|
-
right_values = indexed_values
|
|
873
|
+
left_values = _row_dict(comparable_columns, streamed_values)
|
|
874
|
+
right_values = _row_dict(comparable_columns, indexed_values)
|
|
822
875
|
if not summary_only and len(changed_rows) < limit:
|
|
823
876
|
changed_rows.append(
|
|
824
877
|
{
|
|
@@ -868,18 +921,34 @@ def _spill_reader_to_buckets(
|
|
|
868
921
|
writers: List[Any] = []
|
|
869
922
|
writer_for_bucket: List[Any] = [None] * num_buckets
|
|
870
923
|
try:
|
|
924
|
+
key_positions = [selected_columns.index(column) for column in key_columns]
|
|
871
925
|
for batch in _reader_batch_iterator(reader, columns=selected_columns):
|
|
872
926
|
if len(batch) == 0:
|
|
873
927
|
continue
|
|
874
928
|
batch_dict = batch.to_pydict()
|
|
929
|
+
key_values = [batch_dict[column] for column in selected_columns]
|
|
875
930
|
bucket_indices: List[List[int]] = [[] for _ in range(num_buckets)]
|
|
876
|
-
for
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
)
|
|
881
|
-
|
|
882
|
-
|
|
931
|
+
append = [indices.append for indices in bucket_indices]
|
|
932
|
+
if len(key_positions) == 1:
|
|
933
|
+
position = key_positions[0]
|
|
934
|
+
normalize = _normalize_key_part
|
|
935
|
+
for row_idx, value in enumerate(key_values[position]):
|
|
936
|
+
part = normalize(value)
|
|
937
|
+
if part.__class__ is list or part.__class__ is dict:
|
|
938
|
+
kind = "list" if part.__class__ is list else "struct"
|
|
939
|
+
raise ValueError(
|
|
940
|
+
f"Key column '{key_columns[0]}' contains a {kind}, "
|
|
941
|
+
"which cannot be a key"
|
|
942
|
+
)
|
|
943
|
+
append[_stable_bucket((part,), num_buckets, salt)](row_idx)
|
|
944
|
+
else:
|
|
945
|
+
for row_idx in range(len(batch)):
|
|
946
|
+
key = _row_key_at(
|
|
947
|
+
tuple(values[row_idx] for values in key_values),
|
|
948
|
+
key_positions,
|
|
949
|
+
key_columns,
|
|
950
|
+
)
|
|
951
|
+
append[_stable_bucket(key, num_buckets, salt)](row_idx)
|
|
883
952
|
for bucket, indices in enumerate(bucket_indices):
|
|
884
953
|
if not indices:
|
|
885
954
|
continue
|
|
@@ -918,12 +987,30 @@ def _merge_row_diffs(parts: List[dict], limit: int, summary_only: bool) -> dict:
|
|
|
918
987
|
merged["changed_rows"].extend(part["changed_rows"])
|
|
919
988
|
if summary_only:
|
|
920
989
|
return merged
|
|
921
|
-
|
|
922
|
-
|
|
990
|
+
# Each bucket sampled its own smallest `limit` keys, so the merge kept one
|
|
991
|
+
# bucket's whole sample and dropped the rest. Re-sort the union to keep the
|
|
992
|
+
# smallest keys overall, which is what an unpartitioned diff reports.
|
|
993
|
+
merged["only_left"] = _sort_rendered_keys(merged["only_left"], limit)
|
|
994
|
+
merged["only_right"] = _sort_rendered_keys(merged["only_right"], limit)
|
|
923
995
|
merged["changed_rows"] = merged["changed_rows"][:limit]
|
|
924
996
|
return merged
|
|
925
997
|
|
|
926
998
|
|
|
999
|
+
def _sort_rendered_keys(rows: List[dict[str, Any]], limit: int) -> List[dict[str, Any]]:
|
|
1000
|
+
"""Order ``{"key": ...}`` rows by the diff's key ordering and trim to ``limit``."""
|
|
1001
|
+
if len(rows) <= 1:
|
|
1002
|
+
return rows[:limit]
|
|
1003
|
+
|
|
1004
|
+
def sort_token(row: dict[str, Any]) -> tuple[tuple[Any, ...], ...]:
|
|
1005
|
+
key = row["key"]
|
|
1006
|
+
parts = key if isinstance(key, tuple) else (key,)
|
|
1007
|
+
return tuple(_key_part_sort_token(part) for part in parts)
|
|
1008
|
+
|
|
1009
|
+
if len(rows) > limit:
|
|
1010
|
+
return heapq.nsmallest(limit, rows, key=sort_token)
|
|
1011
|
+
return sorted(rows, key=sort_token)
|
|
1012
|
+
|
|
1013
|
+
|
|
927
1014
|
def _diff_rows_partitioned(
|
|
928
1015
|
left_reader: Optional[MultiFormatReader],
|
|
929
1016
|
right_reader: Optional[MultiFormatReader],
|
|
@@ -940,7 +1027,7 @@ def _diff_rows_partitioned(
|
|
|
940
1027
|
import math
|
|
941
1028
|
import tempfile
|
|
942
1029
|
|
|
943
|
-
num_buckets = min(32, max(2, math.ceil(parent_rows /
|
|
1030
|
+
num_buckets = min(32, max(2, math.ceil(parent_rows / DIFF_PARTITION_TARGET_ROWS)))
|
|
944
1031
|
with tempfile.TemporaryDirectory(prefix="parq-diff-") as tmp_dir:
|
|
945
1032
|
directory = Path(tmp_dir)
|
|
946
1033
|
if left_reader is None:
|
|
@@ -1114,6 +1114,30 @@ class TestMultiFormatReader:
|
|
|
1114
1114
|
assert actual["only_right_count"] == expected["only_right_count"] == 1
|
|
1115
1115
|
assert actual["changed_count"] == expected["changed_count"] == 1
|
|
1116
1116
|
|
|
1117
|
+
def test_diff_partition_samples_smallest_keys_across_buckets(self, tmp_path, monkeypatch):
|
|
1118
|
+
"""Bucketed diff should report the smallest keys, not one bucket's sample.
|
|
1119
|
+
|
|
1120
|
+
Each bucket samples its own smallest keys, so merging the parts without
|
|
1121
|
+
re-sorting would keep whichever bucket came first and drop the rest.
|
|
1122
|
+
"""
|
|
1123
|
+
import parq.reader as reader_mod
|
|
1124
|
+
|
|
1125
|
+
left = tmp_path / "left.csv"
|
|
1126
|
+
right = tmp_path / "right.csv"
|
|
1127
|
+
pacsv.write_csv(
|
|
1128
|
+
pa.table({"id": list(range(20)), "value": ["x"] * 20}),
|
|
1129
|
+
left,
|
|
1130
|
+
)
|
|
1131
|
+
pacsv.write_csv(pa.table({"id": [0], "value": ["x"]}), right)
|
|
1132
|
+
|
|
1133
|
+
expected = diff_files(left, right, ["id"], limit=3)
|
|
1134
|
+
monkeypatch.setattr(reader_mod, "DIFF_MEMORY_ROW_LIMIT", 2)
|
|
1135
|
+
actual = diff_files(left, right, ["id"], limit=3)
|
|
1136
|
+
|
|
1137
|
+
# Key 0 exists on both sides, so the left-only keys start at 1.
|
|
1138
|
+
assert [row["key"] for row in expected["only_left"]] == [1, 2, 3]
|
|
1139
|
+
assert [row["key"] for row in actual["only_left"]] == [1, 2, 3]
|
|
1140
|
+
|
|
1117
1141
|
def test_diff_partition_rejects_duplicate_keys(self, tmp_path, monkeypatch):
|
|
1118
1142
|
"""Duplicate keys should still fail when diff spills into buckets."""
|
|
1119
1143
|
import parq.reader as reader_mod
|
|
@@ -350,6 +350,15 @@ wheels = [
|
|
|
350
350
|
{ url = "https://files.pythonhosted.org/packages/62/15/219075012ab13e8905f3cd572204f4acb4b111df787104346b9bc0cea789/cryptography-50.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:07479a1cb08219ab719147e742e76090c9c773321959bb94946fffdd397a6437", size = 4699593, upload-time = "2026-07-31T14:24:59.951Z" },
|
|
351
351
|
]
|
|
352
352
|
|
|
353
|
+
[[package]]
|
|
354
|
+
name = "defusedxml"
|
|
355
|
+
version = "0.7.1"
|
|
356
|
+
source = { registry = "https://pypi.org/simple" }
|
|
357
|
+
sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" }
|
|
358
|
+
wheels = [
|
|
359
|
+
{ url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" },
|
|
360
|
+
]
|
|
361
|
+
|
|
353
362
|
[[package]]
|
|
354
363
|
name = "docutils"
|
|
355
364
|
version = "0.22.4"
|
|
@@ -572,7 +581,7 @@ wheels = [
|
|
|
572
581
|
|
|
573
582
|
[[package]]
|
|
574
583
|
name = "parq-cli"
|
|
575
|
-
version = "0.2.
|
|
584
|
+
version = "0.2.5"
|
|
576
585
|
source = { editable = "." }
|
|
577
586
|
dependencies = [
|
|
578
587
|
{ name = "pyarrow" },
|
|
@@ -590,6 +599,7 @@ dev = [
|
|
|
590
599
|
{ name = "vulture" },
|
|
591
600
|
]
|
|
592
601
|
xlsx = [
|
|
602
|
+
{ name = "defusedxml" },
|
|
593
603
|
{ name = "openpyxl" },
|
|
594
604
|
{ name = "python-calamine" },
|
|
595
605
|
]
|
|
@@ -597,6 +607,7 @@ xlsx = [
|
|
|
597
607
|
[package.metadata]
|
|
598
608
|
requires-dist = [
|
|
599
609
|
{ name = "build", marker = "extra == 'dev'", specifier = ">=1.2.0" },
|
|
610
|
+
{ name = "defusedxml", marker = "extra == 'xlsx'", specifier = ">=0.7.1" },
|
|
600
611
|
{ name = "openpyxl", marker = "extra == 'xlsx'", specifier = ">=3.1.0" },
|
|
601
612
|
{ name = "pyarrow", specifier = ">=18.0.0" },
|
|
602
613
|
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" },
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|