deepdiff-rs 0.7.1__tar.gz → 0.8.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/Cargo.lock +4 -4
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/Cargo.toml +1 -1
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/PKG-INFO +9 -9
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/README.md +8 -8
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/Cargo.toml +7 -4
- deepdiff_rs-0.8.0/crates/onix-arrow/examples/row_diff_rss.rs +252 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/error.rs +67 -18
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/lib.rs +18 -7
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/row_diff.rs +1776 -207
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/table_diff.rs +34 -31
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/Cargo.toml +1 -1
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/arrow.rs +16 -13
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_table_diff.py +5 -6
- deepdiff_rs-0.8.0/crates/onix-py/tests/test_table_row_diff.py +722 -0
- deepdiff_rs-0.7.1/crates/onix-arrow/examples/row_diff_rss.rs +0 -189
- deepdiff_rs-0.7.1/crates/onix-py/tests/test_table_row_diff.py +0 -368
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/examples/type_stack_cost.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/options.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/schema.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/Cargo.toml +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/examples/stack_frame_cost.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/datetime.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/datetime_tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/array.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/dispatch.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/mod.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/object.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/options.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/scalar.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/set.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/error.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/distance.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/fxhash.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/hash.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/memo.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/mod.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/pairing.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/lcs.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/lcs_tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/lib.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/path.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/report.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/report_tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/test_support.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/unified_diff.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/unified_diff_tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/value.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/value_tests.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/golden.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/ignore_order_memory.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/memory_footprint.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/proptest_diff.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/proptest_ignore_order.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/.python-version +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/benchmarks/bench_bindings.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/convert.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/deepdiff.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/errors.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/fast_path.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/guard.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/lib.rs +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_bindings_memory.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_conversions.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_datetimes.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_depth_guard.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_differential_fuzz.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_golden_parity.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_sets.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_signed_zero.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_smoke.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_suite_hygiene.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_tuples.py +0 -0
- {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/pyproject.toml +0 -0
|
@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
|
629
629
|
|
|
630
630
|
[[package]]
|
|
631
631
|
name = "onix-arrow"
|
|
632
|
-
version = "0.
|
|
632
|
+
version = "0.8.0"
|
|
633
633
|
dependencies = [
|
|
634
634
|
"arrow-array",
|
|
635
635
|
"arrow-buffer",
|
|
@@ -646,7 +646,7 @@ dependencies = [
|
|
|
646
646
|
|
|
647
647
|
[[package]]
|
|
648
648
|
name = "onix-cli"
|
|
649
|
-
version = "0.
|
|
649
|
+
version = "0.8.0"
|
|
650
650
|
dependencies = [
|
|
651
651
|
"onix-core",
|
|
652
652
|
"serde_json",
|
|
@@ -654,7 +654,7 @@ dependencies = [
|
|
|
654
654
|
|
|
655
655
|
[[package]]
|
|
656
656
|
name = "onix-core"
|
|
657
|
-
version = "0.
|
|
657
|
+
version = "0.8.0"
|
|
658
658
|
dependencies = [
|
|
659
659
|
"proptest",
|
|
660
660
|
"serde",
|
|
@@ -665,7 +665,7 @@ dependencies = [
|
|
|
665
665
|
|
|
666
666
|
[[package]]
|
|
667
667
|
name = "onix-py"
|
|
668
|
-
version = "0.
|
|
668
|
+
version = "0.8.0"
|
|
669
669
|
dependencies = [
|
|
670
670
|
"arrow-array",
|
|
671
671
|
"arrow-ipc",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: deepdiff-rs
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.8.0
|
|
4
4
|
Classifier: Programming Language :: Python :: 3
|
|
5
5
|
Classifier: Programming Language :: Rust
|
|
6
6
|
Classifier: License :: OSI Approved :: MIT License
|
|
@@ -118,7 +118,7 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
|
|
|
118
118
|
|
|
119
119
|
`diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
|
|
120
120
|
|
|
121
|
-
It reports the **schema** diff (which columns were added, removed, or changed type)
|
|
121
|
+
It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
|
|
122
122
|
|
|
123
123
|
```python
|
|
124
124
|
import pyarrow as pa
|
|
@@ -137,16 +137,16 @@ right = pa.table({
|
|
|
137
137
|
diff = diff_tables(left, right, key=["id"])
|
|
138
138
|
print(diff.summary())
|
|
139
139
|
print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
|
|
140
|
-
print("
|
|
140
|
+
print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
|
|
141
141
|
```
|
|
142
142
|
|
|
143
143
|
```
|
|
144
|
-
{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0}
|
|
144
|
+
{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
|
|
145
145
|
added ids: [4]
|
|
146
|
-
|
|
146
|
+
cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
|
|
147
147
|
```
|
|
148
148
|
|
|
149
|
-
A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, `1.00` equals `1.0000`, a timestamp compares by its instant across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
|
|
149
|
+
A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
|
|
150
150
|
|
|
151
151
|
Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
|
|
152
152
|
|
|
@@ -168,8 +168,8 @@ Type comparison uses the full logical Arrow type (timestamp unit and timezone, d
|
|
|
168
168
|
- `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
|
|
169
169
|
- In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
|
|
170
170
|
- `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
|
|
171
|
-
- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; and temp disk, because each input is read
|
|
172
|
-
- `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced).
|
|
171
|
+
- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
|
|
172
|
+
- `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
|
|
173
173
|
- Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
|
|
174
174
|
|
|
175
175
|
## Performance
|
|
@@ -226,7 +226,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
|
|
|
226
226
|
- `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict` (with `str` keys), `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, and `datetime.date` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date` comes back as a real `datetime`/`date` — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
|
|
227
227
|
- `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
|
|
228
228
|
- `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
|
|
229
|
-
- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, and `.duplicate_keys()` return Arrow tables
|
|
229
|
+
- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
|
|
230
230
|
|
|
231
231
|
**CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
|
|
232
232
|
|
|
@@ -100,7 +100,7 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
|
|
|
100
100
|
|
|
101
101
|
`diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
|
|
102
102
|
|
|
103
|
-
It reports the **schema** diff (which columns were added, removed, or changed type)
|
|
103
|
+
It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
|
|
104
104
|
|
|
105
105
|
```python
|
|
106
106
|
import pyarrow as pa
|
|
@@ -119,16 +119,16 @@ right = pa.table({
|
|
|
119
119
|
diff = diff_tables(left, right, key=["id"])
|
|
120
120
|
print(diff.summary())
|
|
121
121
|
print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
|
|
122
|
-
print("
|
|
122
|
+
print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
|
|
123
123
|
```
|
|
124
124
|
|
|
125
125
|
```
|
|
126
|
-
{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0}
|
|
126
|
+
{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
|
|
127
127
|
added ids: [4]
|
|
128
|
-
|
|
128
|
+
cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
|
|
129
129
|
```
|
|
130
130
|
|
|
131
|
-
A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, `1.00` equals `1.0000`, a timestamp compares by its instant across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
|
|
131
|
+
A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
|
|
132
132
|
|
|
133
133
|
Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
|
|
134
134
|
|
|
@@ -150,8 +150,8 @@ Type comparison uses the full logical Arrow type (timestamp unit and timezone, d
|
|
|
150
150
|
- `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
|
|
151
151
|
- In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
|
|
152
152
|
- `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
|
|
153
|
-
- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; and temp disk, because each input is read
|
|
154
|
-
- `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced).
|
|
153
|
+
- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
|
|
154
|
+
- `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
|
|
155
155
|
- Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
|
|
156
156
|
|
|
157
157
|
## Performance
|
|
@@ -208,7 +208,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
|
|
|
208
208
|
- `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict` (with `str` keys), `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, and `datetime.date` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date` comes back as a real `datetime`/`date` — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
|
|
209
209
|
- `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
|
|
210
210
|
- `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
|
|
211
|
-
- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, and `.duplicate_keys()` return Arrow tables
|
|
211
|
+
- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
|
|
212
212
|
|
|
213
213
|
**CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
|
|
214
214
|
|
|
@@ -22,10 +22,13 @@ arrow-array = "=59.3.0"
|
|
|
22
22
|
# `i256` for hashing `Decimal256` cells the same way as `Decimal128`.
|
|
23
23
|
arrow-buffer = "=59.3.0"
|
|
24
24
|
# `arrow-cast` decodes dictionary-encoded columns to their value type before
|
|
25
|
-
# hashing,
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
25
|
+
# hashing, casts a right-only duplicate key's columns to the left key types,
|
|
26
|
+
# strips a timestamp's timezone before rendering, and renders each changed
|
|
27
|
+
# cell's value to its canonical string (`display::ArrayFormatter`);
|
|
28
|
+
# `arrow-select` filters the materialized rows, concatenates the output row
|
|
29
|
+
# batches, and `take`s the key columns into the per-cell diff. The row diff
|
|
30
|
+
# re-reads each side through the caller's re-openable `TableInput`, so no IPC
|
|
31
|
+
# spooling lives here — the Python bindings own that.
|
|
29
32
|
arrow-cast = "=59.3.0"
|
|
30
33
|
arrow-select = "=59.3.0"
|
|
31
34
|
# Row identity is a keyed 128-bit SipHash-1-3 (`siphasher`), keyed from 16
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
//! Measures the keyed row diff's peak memory and wall time, to check the memory
|
|
2
|
+
//! bounds the README states.
|
|
3
|
+
//!
|
|
4
|
+
//! Run under the OS's max-RSS reporter:
|
|
5
|
+
//!
|
|
6
|
+
//! ```sh
|
|
7
|
+
//! cargo build -p onix-arrow --release --example row_diff_rss
|
|
8
|
+
//! # linear shape (default): mostly-matching rows, 1% added/removed, ~2% changed
|
|
9
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000
|
|
10
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 10000000
|
|
11
|
+
//! # same shape with no changed rows: the cell pass materializes nothing, the
|
|
12
|
+
//! # pass-one baseline the ~2%-changed run is measured against
|
|
13
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 nochange
|
|
14
|
+
//! # every row changed (narrow int cells): the cell pass at full width
|
|
15
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 allchange
|
|
16
|
+
//! # every row changed with a wide (1 KB) string cell: the rendering worst case
|
|
17
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 100000 wide 1024
|
|
18
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 200000 wide 1024
|
|
19
|
+
//! # duplicate-heavy shape: every key duplicated, wide string key
|
|
20
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 dup 16
|
|
21
|
+
//! /usr/bin/time -l target/release/examples/row_diff_rss 200000 dup 1024
|
|
22
|
+
//! ```
|
|
23
|
+
//!
|
|
24
|
+
//! Each side is generated on the fly, batch by batch, and nothing is retained
|
|
25
|
+
//! between batches, so the process's peak RSS is the diff's own state, not the
|
|
26
|
+
//! table data. The **linear** shape (`id`, `value` int64 columns) exercises the
|
|
27
|
+
//! per-row hash vectors: the left is ids `0..n`, the right `step..n + step` with
|
|
28
|
+
//! `step = n / 100`, so 1% removed, 1% added, ~2% changed — and the cell pass
|
|
29
|
+
//! materializes those ~2% changed rows on both sides. The **nochange** variant
|
|
30
|
+
//! keeps the 1% added/removed but makes every shared row equal, so the cell pass
|
|
31
|
+
//! materializes nothing: the difference in peak RSS between it and the default
|
|
32
|
+
//! run is the cell pass's cost. The **allchange** variant drops the offset and
|
|
33
|
+
//! changes every shared row (no added/removed), so the cell pass materializes
|
|
34
|
+
//! and renders every row. The **wide** shape (`id` int64, `value` a
|
|
35
|
+
//! `value_width`-byte Utf8 that differs between the sides) changes every row too
|
|
36
|
+
//! and renders `value_width` bytes per changed cell — the rendering worst case,
|
|
37
|
+
//! whose peak RSS scales with changed cells times cell width. The **dup** shape
|
|
38
|
+
//! (`key` Utf8 of the given width, `value` int64) makes every key appear twice
|
|
39
|
+
//! on each side, so every distinct
|
|
40
|
+
//! key is a duplicate and the whole `duplicate_keys` report is materialized —
|
|
41
|
+
//! the term that scales with distinct duplicated keys times the key width.
|
|
42
|
+
|
|
43
|
+
use std::sync::Arc;
|
|
44
|
+
|
|
45
|
+
use arrow_array::{ArrayRef, Int64Array, RecordBatch, RecordBatchReader, StringArray};
|
|
46
|
+
use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
|
|
47
|
+
use onix_arrow::{TableDiffError, TableDiffOptions, TableInput, diff_tables};
|
|
48
|
+
|
|
49
|
+
const BATCH: i64 = 65_536;
|
|
50
|
+
|
|
51
|
+
/// The generated table shape.
|
|
52
|
+
#[derive(Clone, Copy)]
|
|
53
|
+
enum Shape {
|
|
54
|
+
/// `(id, value)` int64 columns; `id_offset` shifts the key range and
|
|
55
|
+
/// `change_every` perturbs a fraction of values.
|
|
56
|
+
Linear { id_offset: i64, change_every: i64 },
|
|
57
|
+
/// `(key, value)`; `key` is a `key_width`-byte string and each key value
|
|
58
|
+
/// appears twice, so every key is a duplicate.
|
|
59
|
+
Dup { key_width: usize },
|
|
60
|
+
/// `(id, value)` where `value` is a `value_width`-byte string filled with
|
|
61
|
+
/// `fill`; the two sides share every id but differ in `fill`, so every row
|
|
62
|
+
/// is changed and every changed cell renders `value_width` bytes — the
|
|
63
|
+
/// wide-cell worst case for the per-cell diff's rendering memory.
|
|
64
|
+
Wide { value_width: usize, fill: u8 },
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/// A table generated on demand, retaining nothing between batches.
|
|
68
|
+
struct Generated {
|
|
69
|
+
schema: SchemaRef,
|
|
70
|
+
rows: i64,
|
|
71
|
+
shape: Shape,
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
impl TableInput for Generated {
|
|
75
|
+
fn schema(&self) -> SchemaRef {
|
|
76
|
+
self.schema.clone()
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
|
|
80
|
+
Ok(Box::new(GenReader {
|
|
81
|
+
schema: self.schema.clone(),
|
|
82
|
+
rows: self.rows,
|
|
83
|
+
shape: self.shape,
|
|
84
|
+
next: 0,
|
|
85
|
+
}))
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
struct GenReader {
|
|
90
|
+
schema: SchemaRef,
|
|
91
|
+
rows: i64,
|
|
92
|
+
shape: Shape,
|
|
93
|
+
next: i64,
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
impl Iterator for GenReader {
|
|
97
|
+
type Item = Result<RecordBatch, ArrowError>;
|
|
98
|
+
|
|
99
|
+
fn next(&mut self) -> Option<Self::Item> {
|
|
100
|
+
if self.next >= self.rows {
|
|
101
|
+
return None;
|
|
102
|
+
}
|
|
103
|
+
let end = (self.next + BATCH).min(self.rows);
|
|
104
|
+
let columns: Vec<ArrayRef> = match self.shape {
|
|
105
|
+
Shape::Linear {
|
|
106
|
+
id_offset,
|
|
107
|
+
change_every,
|
|
108
|
+
} => {
|
|
109
|
+
let ids: Int64Array = (self.next..end).map(|i| Some(i + id_offset)).collect();
|
|
110
|
+
// The value is a function of the id, so a shared key holds the
|
|
111
|
+
// same value on both sides except every `change_every`-th id.
|
|
112
|
+
let values: Int64Array = (self.next..end)
|
|
113
|
+
.map(|i| {
|
|
114
|
+
let id = i + id_offset;
|
|
115
|
+
Some(if id % change_every == 0 { id + 1 } else { id })
|
|
116
|
+
})
|
|
117
|
+
.collect();
|
|
118
|
+
vec![Arc::new(ids), Arc::new(values)]
|
|
119
|
+
}
|
|
120
|
+
Shape::Dup { key_width } => {
|
|
121
|
+
// Key value `i / 2`, so each distinct key appears twice.
|
|
122
|
+
let keys: StringArray = (self.next..end)
|
|
123
|
+
.map(|i| Some(format!("{:0>width$}", i / 2, width = key_width)))
|
|
124
|
+
.collect();
|
|
125
|
+
let values: Int64Array = (self.next..end).map(Some).collect();
|
|
126
|
+
vec![Arc::new(keys), Arc::new(values)]
|
|
127
|
+
}
|
|
128
|
+
Shape::Wide { value_width, fill } => {
|
|
129
|
+
let ids: Int64Array = (self.next..end).map(Some).collect();
|
|
130
|
+
let cell = String::from_utf8(vec![fill; value_width]).unwrap();
|
|
131
|
+
let values: StringArray = (self.next..end).map(|_| Some(cell.as_str())).collect();
|
|
132
|
+
vec![Arc::new(ids), Arc::new(values)]
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
self.next = end;
|
|
136
|
+
|
|
137
|
+
Some(RecordBatch::try_new(self.schema.clone(), columns))
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
impl RecordBatchReader for GenReader {
|
|
142
|
+
fn schema(&self) -> SchemaRef {
|
|
143
|
+
self.schema.clone()
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
fn main() {
|
|
148
|
+
let args: Vec<String> = std::env::args().collect();
|
|
149
|
+
let rows: i64 = args
|
|
150
|
+
.get(1)
|
|
151
|
+
.and_then(|a| a.parse().ok())
|
|
152
|
+
.unwrap_or(1_000_000);
|
|
153
|
+
// Mode selects the shape: "dup" (duplicate-heavy), "wide" (all rows changed,
|
|
154
|
+
// wide string value column), "nochange"/"allchange"/default (linear int).
|
|
155
|
+
let mode = args.get(2).map_or("", String::as_str);
|
|
156
|
+
let width: usize = args
|
|
157
|
+
.get(3)
|
|
158
|
+
.and_then(|a| a.parse().ok())
|
|
159
|
+
.unwrap_or(if mode == "wide" { 1024 } else { 16 });
|
|
160
|
+
|
|
161
|
+
let (schema, left_shape, right_shape, key, label) = match mode {
|
|
162
|
+
"dup" => {
|
|
163
|
+
let schema = Arc::new(Schema::new(vec![
|
|
164
|
+
Field::new("key", DataType::Utf8, false),
|
|
165
|
+
Field::new("value", DataType::Int64, false),
|
|
166
|
+
]));
|
|
167
|
+
let shape = Shape::Dup { key_width: width };
|
|
168
|
+
(
|
|
169
|
+
schema,
|
|
170
|
+
shape,
|
|
171
|
+
shape,
|
|
172
|
+
"key",
|
|
173
|
+
format!(" (dup, key_width={width})"),
|
|
174
|
+
)
|
|
175
|
+
}
|
|
176
|
+
"wide" => {
|
|
177
|
+
let schema = Arc::new(Schema::new(vec![
|
|
178
|
+
Field::new("id", DataType::Int64, false),
|
|
179
|
+
Field::new("value", DataType::Utf8, false),
|
|
180
|
+
]));
|
|
181
|
+
(
|
|
182
|
+
schema,
|
|
183
|
+
Shape::Wide {
|
|
184
|
+
value_width: width,
|
|
185
|
+
fill: b'a',
|
|
186
|
+
},
|
|
187
|
+
Shape::Wide {
|
|
188
|
+
value_width: width,
|
|
189
|
+
fill: b'b',
|
|
190
|
+
},
|
|
191
|
+
"id",
|
|
192
|
+
format!(" (wide, value_width={width}, all changed)"),
|
|
193
|
+
)
|
|
194
|
+
}
|
|
195
|
+
_ => {
|
|
196
|
+
let schema = Arc::new(Schema::new(vec![
|
|
197
|
+
Field::new("id", DataType::Int64, false),
|
|
198
|
+
Field::new("value", DataType::Int64, false),
|
|
199
|
+
]));
|
|
200
|
+
let step = (rows / 100).max(1);
|
|
201
|
+
// `nochange`: 1% added/removed but no changed rows (the pass-one
|
|
202
|
+
// baseline). `allchange`: no added/removed, every row changed.
|
|
203
|
+
// Default: 1% added/removed, ~2% changed.
|
|
204
|
+
let (id_offset, change_every, label) = match mode {
|
|
205
|
+
"nochange" => (step, i64::MAX, " (nochange baseline)"),
|
|
206
|
+
"allchange" => (0, 1, " (all changed)"),
|
|
207
|
+
_ => (step, 50, ""),
|
|
208
|
+
};
|
|
209
|
+
(
|
|
210
|
+
schema,
|
|
211
|
+
Shape::Linear {
|
|
212
|
+
id_offset: 0,
|
|
213
|
+
change_every: i64::MAX,
|
|
214
|
+
},
|
|
215
|
+
Shape::Linear {
|
|
216
|
+
id_offset,
|
|
217
|
+
change_every,
|
|
218
|
+
},
|
|
219
|
+
"id",
|
|
220
|
+
label.to_string(),
|
|
221
|
+
)
|
|
222
|
+
}
|
|
223
|
+
};
|
|
224
|
+
|
|
225
|
+
let left = Generated {
|
|
226
|
+
schema: schema.clone(),
|
|
227
|
+
rows,
|
|
228
|
+
shape: left_shape,
|
|
229
|
+
};
|
|
230
|
+
let right = Generated {
|
|
231
|
+
schema,
|
|
232
|
+
rows,
|
|
233
|
+
shape: right_shape,
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
let start = std::time::Instant::now();
|
|
237
|
+
let diff = diff_tables(&left, &right, &TableDiffOptions::new(vec![key.to_string()]))
|
|
238
|
+
.expect("diff succeeds");
|
|
239
|
+
let elapsed = start.elapsed();
|
|
240
|
+
let summary = diff.summary();
|
|
241
|
+
|
|
242
|
+
println!("rows per side: {rows}{label}");
|
|
243
|
+
println!("wall: {:.2}s", elapsed.as_secs_f64());
|
|
244
|
+
println!(
|
|
245
|
+
"rows_added={} rows_removed={} rows_changed={} duplicate_keys={} cells_changed={}",
|
|
246
|
+
summary.rows_added,
|
|
247
|
+
summary.rows_removed,
|
|
248
|
+
summary.rows_changed,
|
|
249
|
+
summary.duplicate_keys,
|
|
250
|
+
summary.cells_changed
|
|
251
|
+
);
|
|
252
|
+
}
|
|
@@ -31,9 +31,8 @@ impl fmt::Display for Side {
|
|
|
31
31
|
|
|
32
32
|
/// Errors that can occur while diffing two tables.
|
|
33
33
|
///
|
|
34
|
-
/// Marked `#[non_exhaustive]`
|
|
35
|
-
///
|
|
36
|
-
/// a wildcard arm.
|
|
34
|
+
/// Marked `#[non_exhaustive]` so future work can add variants without a
|
|
35
|
+
/// breaking change; matching on it must keep a wildcard arm.
|
|
37
36
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
38
37
|
#[non_exhaustive]
|
|
39
38
|
pub enum TableDiffError {
|
|
@@ -95,13 +94,32 @@ pub enum TableDiffError {
|
|
|
95
94
|
/// The underlying error's message.
|
|
96
95
|
message: String,
|
|
97
96
|
},
|
|
98
|
-
/// A
|
|
99
|
-
///
|
|
100
|
-
///
|
|
101
|
-
///
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
97
|
+
/// A cell value could not be rendered to its canonical string for the
|
|
98
|
+
/// per-cell diff — for example a temporal value outside the range the
|
|
99
|
+
/// formatter can format. Reported as a typed error rather than written into
|
|
100
|
+
/// the output as error prose (which a real string cell could not be told
|
|
101
|
+
/// apart from).
|
|
102
|
+
Render {
|
|
103
|
+
/// The column whose cell could not be rendered.
|
|
104
|
+
column: String,
|
|
105
|
+
/// The underlying formatting error's message.
|
|
106
|
+
message: String,
|
|
107
|
+
},
|
|
108
|
+
/// The per-cell diff would need more than `u32::MAX` changed rows on one
|
|
109
|
+
/// side, which its row-index arrays cannot address. Bound the changed-row
|
|
110
|
+
/// count of untrusted input.
|
|
111
|
+
TooManyChangedRows {
|
|
112
|
+
/// The number of changed rows that overflowed.
|
|
113
|
+
rows: usize,
|
|
114
|
+
},
|
|
115
|
+
/// A `value_changed` cell rendered identically on both sides — a broken
|
|
116
|
+
/// invariant, not a caller error. The per-cell diff renders each side in a
|
|
117
|
+
/// common comparison form so a value change always shows two different
|
|
118
|
+
/// strings; this variant guards that guarantee and cannot fire for any real
|
|
119
|
+
/// input.
|
|
120
|
+
EqualRenderings {
|
|
121
|
+
/// The column whose two renderings were equal.
|
|
122
|
+
column: String,
|
|
105
123
|
},
|
|
106
124
|
}
|
|
107
125
|
|
|
@@ -137,10 +155,20 @@ impl fmt::Display for TableDiffError {
|
|
|
137
155
|
needs the key to be the same type on both sides"
|
|
138
156
|
),
|
|
139
157
|
TableDiffError::Read { message } => write!(f, "failed to read table data: {message}"),
|
|
140
|
-
TableDiffError::
|
|
158
|
+
TableDiffError::Render { column, message } => write!(
|
|
141
159
|
f,
|
|
142
|
-
"{
|
|
143
|
-
|
|
160
|
+
"could not render a value of column {column:?} to its canonical string: {message}"
|
|
161
|
+
),
|
|
162
|
+
TableDiffError::TooManyChangedRows { rows } => write!(
|
|
163
|
+
f,
|
|
164
|
+
"the per-cell diff has {rows} changed rows on one side, more than the \
|
|
165
|
+
{} its row-index arrays can address; bound the changed-row count",
|
|
166
|
+
u32::MAX
|
|
167
|
+
),
|
|
168
|
+
TableDiffError::EqualRenderings { column } => write!(
|
|
169
|
+
f,
|
|
170
|
+
"internal invariant: a value change in column {column:?} rendered identically \
|
|
171
|
+
on both sides, which the common-form rendering is designed to prevent"
|
|
144
172
|
),
|
|
145
173
|
}
|
|
146
174
|
}
|
|
@@ -220,13 +248,34 @@ mod tests {
|
|
|
220
248
|
}
|
|
221
249
|
|
|
222
250
|
#[test]
|
|
223
|
-
fn
|
|
224
|
-
let error = TableDiffError::
|
|
225
|
-
|
|
251
|
+
fn render_message_names_the_column() {
|
|
252
|
+
let error = TableDiffError::Render {
|
|
253
|
+
column: "ts".to_string(),
|
|
254
|
+
message: "Cast error".to_string(),
|
|
255
|
+
};
|
|
256
|
+
let message = error.to_string();
|
|
257
|
+
assert!(message.contains("\"ts\""));
|
|
258
|
+
assert!(message.contains("Cast error"));
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
#[test]
|
|
262
|
+
fn equal_renderings_message_reads_as_an_invariant() {
|
|
263
|
+
let error = TableDiffError::EqualRenderings {
|
|
264
|
+
column: "d".to_string(),
|
|
265
|
+
};
|
|
266
|
+
let message = error.to_string();
|
|
267
|
+
assert!(message.contains("\"d\""));
|
|
268
|
+
assert!(message.contains("invariant"));
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
#[test]
|
|
272
|
+
fn too_many_changed_rows_message_names_the_count() {
|
|
273
|
+
let error = TableDiffError::TooManyChangedRows {
|
|
274
|
+
rows: 5_000_000_000,
|
|
226
275
|
};
|
|
227
276
|
let message = error.to_string();
|
|
228
|
-
assert!(message.contains("
|
|
229
|
-
assert!(message.contains("
|
|
277
|
+
assert!(message.contains("5000000000"));
|
|
278
|
+
assert!(message.contains("changed-row count"));
|
|
230
279
|
}
|
|
231
280
|
|
|
232
281
|
#[test]
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
//! Table diffing for onix, over Apache Arrow.
|
|
2
2
|
//!
|
|
3
3
|
//! [`diff_tables`] compares two tables presented as [`TableInput`]s and returns
|
|
4
|
-
//! a [`TableDiff`] carrying
|
|
5
|
-
//! removed, or changed type —
|
|
6
|
-
//!
|
|
7
|
-
//! (`cells_changed`)
|
|
4
|
+
//! a [`TableDiff`] carrying the **schema** diff — which columns were added,
|
|
5
|
+
//! removed, or changed type — the keyed **row** diff — which rows were added,
|
|
6
|
+
//! removed, or changed, and which keys are duplicated — and the per-cell diff
|
|
7
|
+
//! (`cells_changed`), reporting which columns changed in each changed row and
|
|
8
|
+
//! how.
|
|
8
9
|
//!
|
|
9
10
|
//! The two tables are matched on a required, non-empty set of key columns (the
|
|
10
11
|
//! table's primary key), carried in [`TableDiffOptions`]. Every key column must
|
|
@@ -14,9 +15,10 @@
|
|
|
14
15
|
//!
|
|
15
16
|
//! # Inputs
|
|
16
17
|
//!
|
|
17
|
-
//! The row diff reads each side
|
|
18
|
-
//! materialize
|
|
19
|
-
//!
|
|
18
|
+
//! The row diff reads each side more than once (to hash every row, to
|
|
19
|
+
//! materialize the added/removed rows, and to materialize the changed rows for
|
|
20
|
+
//! the per-cell diff), so [`diff_tables`] takes a re-openable [`TableInput`]
|
|
21
|
+
//! rather than a single-use `RecordBatchReader`.
|
|
20
22
|
//! In-memory tables use [`MemoryInput`]; a caller whose data is a one-shot
|
|
21
23
|
//! stream spools it to a temporary Arrow IPC file first and implements
|
|
22
24
|
//! [`TableInput`] over that file (as the Python bindings do).
|
|
@@ -162,6 +164,15 @@ pub const MAX_NESTING_DEPTH: usize = 128;
|
|
|
162
164
|
/// nested key, a run-end-encoded column, or a type combination Arrow cannot
|
|
163
165
|
/// build.
|
|
164
166
|
/// - [`TableDiffError::Read`] if a batch cannot be read from either input.
|
|
167
|
+
/// - [`TableDiffError::Render`] if a changed cell's value cannot be rendered to
|
|
168
|
+
/// its canonical string — for example an out-of-range temporal value the
|
|
169
|
+
/// formatter cannot format.
|
|
170
|
+
/// - [`TableDiffError::TooManyChangedRows`] if one side has more than
|
|
171
|
+
/// `u32::MAX` changed rows, which the per-cell diff's row-index arrays cannot
|
|
172
|
+
/// address.
|
|
173
|
+
/// - [`TableDiffError::EqualRenderings`] never fires for real input: it guards
|
|
174
|
+
/// the internal invariant that a `value_changed` cell always renders two
|
|
175
|
+
/// different strings.
|
|
165
176
|
pub fn diff_tables(
|
|
166
177
|
left: &impl TableInput,
|
|
167
178
|
right: &impl TableInput,
|