deepdiff-rs 0.13.0__tar.gz → 0.14.0__tar.gz

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/Cargo.lock

@@ -629,12 +629,13 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
  "arrow-ipc",
+ "arrow-ord",
  "arrow-schema",
  "arrow-select",
  "getrandom 0.3.4",
@@ -648,7 +649,7 @@ dependencies = [
 
 [[package]]
 name = "onix-cli"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -656,7 +657,7 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "num-bigint",
  "num-traits",
@@ -669,7 +670,7 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.13.0"
+version = "0.14.0"
 dependencies = [
  "arrow-array",
  "arrow-schema",

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.13.0"
+version = "0.14.0"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.13.0
+Version: 0.14.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -136,7 +136,7 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
+It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in [`docs/design/row-diff.md`](docs/design/row-diff.md)'s "Per-cell changes" section. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
 
 ```python
 import pyarrow as pa
@@ -210,7 +210,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, spills each side's changed rows by key-hash partition and renders them across every core, which took its 5 GB-per-side wall from 149.674 to 24.375 s and its peak RSS from about 67 GB to 33.1 GB; and since 0.13.0 the parallel diff decodes the right input once and the left twice instead of each three times, taking the 5 GB pairs from 5.727 to 3.879 s narrow (1.71x DuckDB, 1.93x polars) and from 21.294 to 14.968 s wide (2.90x DuckDB, 5.47x polars) in a rotated, interleaved run, with the wide pair's peak RSS down from 33.2 GB to 28.8 GB; see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, spills each side's changed rows by key-hash partition and renders them across every core, which took its 5 GB-per-side wall from 149.674 to 24.375 s and its peak RSS from about 67 GB to 33.1 GB; and since 0.13.0 the parallel diff decodes the right input once and the left twice instead of each three times, taking the 5 GB pairs from 5.727 to 3.879 s narrow (1.71x DuckDB, 1.93x polars) and from 21.294 to 14.968 s wide (2.90x DuckDB, 5.47x polars) in a rotated, interleaved run, with the wide pair's peak RSS down from 33.2 GB to 28.8 GB; and since 0.14.0 the cell pass compares each same-typed column with one vectorized kernel call instead of hashing every cell, cutting the wide 5 GB pair's wall by 8% and its CPU time by 23% against 0.13.0 measured in the same interleaved run (absolute figures and load in the results' column-wise cell compare section); see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -243,7 +243,7 @@ crates/onix-core   # the diff engine (library, no I/O)
 crates/onix-cli    # the `onix` binary (thin CLI over the core)
 crates/onix-arrow  # Arrow table diffing (schema diff and keyed row diff)
 crates/onix-py     # PyO3 bindings, published as `deepdiff-rs`
-docs/design/       # algorithm/invariant reference pages (list-diff, ignore-order, value-model, depth-budget)
+docs/design/       # algorithm/invariant reference pages (list-diff, ignore-order, value-model, depth-budget, row-diff)
 scripts/           # gen_goldens.py: regenerates tests/golden/ from real DeepDiff
 tests/golden       # DeepDiff-generated expected outputs (the compatibility corpus)
 perf/              # cross-language benchmark harness and RESULTS.md
@@ -267,10 +267,10 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables` row diff: 32 B hash per row per side (left only in parallel): 75 MB at 1M, 660 MB at 10M (single-threaded); 600/618 MB at 8M, 2.7/2.5 GB at 37M (1/18 threads); wall time `N·log N` across workers. Duplicate-key report, 18 threads: 39 MB (16 B keys) to 0.83 GB (1 KB keys) at 200k. Cell pass (both sides' spilled rows × column width, plus 2x `cells_changed`), 18 threads: 0.96/4.62 GB at 200k/1M, 1.56 GB spill-dominated (150k rows, eight 512 B columns, one differing). Right-side duplicate keys, 18 threads: +0.95 GB per 500k repeated keys (2 KB of value columns, 1M rows/side); +85 MB per 1M keys the left lacks (8 B `int64` keys; the entry is 32 B at any key width). Kept rows pin their input batch, per side: the whole batch when over half is kept, its view data otherwise (~2.0 GB, 1 and 18 threads, for 100 removed rows over a 1M-row side with two 1 KB `Utf8View` columns). [RESULTS.md, Fused reads](perf/arrow/RESULTS.md#fused-reads-issue-90-1).
+- `diff_tables` row diff: 32 B hash per row per side (left only in parallel): 75 MB at 1M, 660 MB at 10M (single-threaded); 600/618 MB at 8M, 2.7/2.5 GB at 37M (1/18 threads); wall time `N·log N` across workers. Duplicate-key report, 18 threads: 39 MB (16 B keys) to 0.83 GB (1 KB keys) at 200k. Cell pass (both sides' spilled rows × column width, plus 2x `cells_changed`), 18 threads: 0.96 GB at 200k, about 4.8 GB at 1M (4.77 to 4.91 GB over five runs each of 0.13.0 and 0.14.0), 1.56 GB spill-dominated (150k rows, eight 512 B columns, one differing). The cell pass also copies each column whose two sides share a type once per partition, aligned for comparison (changed rows ÷ partitions × that column's width; byte-view columns are compared as their spilled large-offset type, so they copy in full): +0.3 GB (5.64 → 5.95 GB) for 500k changed rows/side, three equal 1 KB columns (`Utf8View`, `BinaryView`, `Utf8`), only an `Int64` differing, 2 threads, medians of 8 runs of 0.13.1 (identical to 0.13.0 on this path) and 0.14.0. Right-side duplicate keys, 18 threads: +0.95 GB per 500k repeated keys (2 KB of value columns, 1M rows/side); +85 MB per 1M keys the left lacks (8 B `int64` keys; the entry is 32 B at any key width). Kept rows pin their input batch, per side: the whole batch when over half is kept, its view data otherwise (~2.0 GB, 1 and 18 threads, for 100 removed rows over a 1M-row side with two 1 KB `Utf8View` columns). [RESULTS.md, Fused reads](perf/arrow/RESULTS.md#fused-reads-issue-90-1).
 - Temp disk, same call: each input spools to an anonymous file (`tempfile`: unlinked, mode 0600, no name). Changed rows spill to anonymous per-partition files, byte-view columns cast to a large-offset type (i32 caps at ~2 GB) and dictionaries decoded, independent of the partition count (~97 GB at 64 threads undecoded vs ~32 GB, wide pair); in parallel also one row per right key the left holds once that a later batch repeats (1.03 GB per 500k such keys, 2 KB of value columns, 1M rows/side, 2 and 64 threads). All resident until the cell pass ends: ~23.8 GB, wide pair (2/18/64 threads; Linux spill may be a RAM-backed `tmpfs`). Whole-process peaks, wide pair: ~29 GB at 18 threads, 40 GB at 2, 31 GB at 64 (33/46/34 GB on 0.11.2). A full temp filesystem raises `ValueError` naming `TMPDIR`.
 - The size gate peeks up to 50,000 rows or 64 MB decoded per side, holding one whole producer batch per side beyond it: a 49,999-row/side, 8 KB-cell, 100-row-batch pair peaks ~65 MB at 18 threads, 12 MB single-threaded, ~1.6 GB single-threaded (1.2 GB at 18 threads) for one whole-side batch.
-- None of the row-diff memory, temp-disk or size-gate-peek costs above has a built-in cap. Bound: row count; changed fraction; right-side duplicate keys (on the parallel path, one full-width row per repeated key, spilled or held); column widths (key-column width, for duplicate-heavy data; for the cell pass, the total width of all common value columns, since every one spills in full per changed row whether it changed or not, plus the rendered changed cells; any column's width for the size-gate peek, which buffers decoded cells whether or not they change); the producer's batch size; thread count, lowest at 18 threads (measured default), higher at 2 threads (two partitions each holding half the rows), slightly higher at 64: spill-dominated 2.39 GB at 2 threads, 1.96 GB at 64; output-dominated at 200k rows, 1.45 GB at 2, 1.25 GB at 64.
+- None of the row-diff memory, temp-disk or size-gate-peek costs above has a built-in cap. Bound: row count; changed fraction; right-side duplicate keys (on the parallel path, one full-width row per repeated key, spilled or held); column widths (key-column width, for duplicate-heavy data; for the cell pass, the total width of all common value columns, since every one spills in full per changed row whether it changed or not, plus the rendered changed cells; any column's width for the size-gate peek, which buffers decoded cells whether or not they change); the producer's batch size; thread count, lowest at 18 threads (measured default), higher at 2 threads (two partitions each holding half the rows), slightly higher at 64, and fewer partitions also enlarge the cell pass's aligned copy of each same-typed compared column (changed rows per partition × the widest such column): spill-dominated 2.39 GB at 2 threads, 1.96 GB at 64; output-dominated at 200k rows, 1.45 GB at 2, 1.25 GB at 64.
 - Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per batch from `ROW_DIFF_BATCH`), macOS on an Apple M-series laptop, 2026-09-24, same method as [Performance](#performance); the single-threaded hash-term figures carry run-to-run slack up to 2.7-3.8 GB at 37M rows/side, the other cell-pass and thread-count figures are medians over 3 runs, and the two 150k spill/output shapes over 5. See [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md) for the size-gate peek table and the decode-vs-undecoded partition figures, and [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs) for the implementation.
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the first row-diff bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/README.md

@@ -118,7 +118,7 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
+It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in [`docs/design/row-diff.md`](docs/design/row-diff.md)'s "Per-cell changes" section. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
 
 ```python
 import pyarrow as pa
@@ -192,7 +192,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, spills each side's changed rows by key-hash partition and renders them across every core, which took its 5 GB-per-side wall from 149.674 to 24.375 s and its peak RSS from about 67 GB to 33.1 GB; and since 0.13.0 the parallel diff decodes the right input once and the left twice instead of each three times, taking the 5 GB pairs from 5.727 to 3.879 s narrow (1.71x DuckDB, 1.93x polars) and from 21.294 to 14.968 s wide (2.90x DuckDB, 5.47x polars) in a rotated, interleaved run, with the wide pair's peak RSS down from 33.2 GB to 28.8 GB; see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, spills each side's changed rows by key-hash partition and renders them across every core, which took its 5 GB-per-side wall from 149.674 to 24.375 s and its peak RSS from about 67 GB to 33.1 GB; and since 0.13.0 the parallel diff decodes the right input once and the left twice instead of each three times, taking the 5 GB pairs from 5.727 to 3.879 s narrow (1.71x DuckDB, 1.93x polars) and from 21.294 to 14.968 s wide (2.90x DuckDB, 5.47x polars) in a rotated, interleaved run, with the wide pair's peak RSS down from 33.2 GB to 28.8 GB; and since 0.14.0 the cell pass compares each same-typed column with one vectorized kernel call instead of hashing every cell, cutting the wide 5 GB pair's wall by 8% and its CPU time by 23% against 0.13.0 measured in the same interleaved run (absolute figures and load in the results' column-wise cell compare section); see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -225,7 +225,7 @@ crates/onix-core   # the diff engine (library, no I/O)
 crates/onix-cli    # the `onix` binary (thin CLI over the core)
 crates/onix-arrow  # Arrow table diffing (schema diff and keyed row diff)
 crates/onix-py     # PyO3 bindings, published as `deepdiff-rs`
-docs/design/       # algorithm/invariant reference pages (list-diff, ignore-order, value-model, depth-budget)
+docs/design/       # algorithm/invariant reference pages (list-diff, ignore-order, value-model, depth-budget, row-diff)
 scripts/           # gen_goldens.py: regenerates tests/golden/ from real DeepDiff
 tests/golden       # DeepDiff-generated expected outputs (the compatibility corpus)
 perf/              # cross-language benchmark harness and RESULTS.md
@@ -249,10 +249,10 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables` row diff: 32 B hash per row per side (left only in parallel): 75 MB at 1M, 660 MB at 10M (single-threaded); 600/618 MB at 8M, 2.7/2.5 GB at 37M (1/18 threads); wall time `N·log N` across workers. Duplicate-key report, 18 threads: 39 MB (16 B keys) to 0.83 GB (1 KB keys) at 200k. Cell pass (both sides' spilled rows × column width, plus 2x `cells_changed`), 18 threads: 0.96/4.62 GB at 200k/1M, 1.56 GB spill-dominated (150k rows, eight 512 B columns, one differing). Right-side duplicate keys, 18 threads: +0.95 GB per 500k repeated keys (2 KB of value columns, 1M rows/side); +85 MB per 1M keys the left lacks (8 B `int64` keys; the entry is 32 B at any key width). Kept rows pin their input batch, per side: the whole batch when over half is kept, its view data otherwise (~2.0 GB, 1 and 18 threads, for 100 removed rows over a 1M-row side with two 1 KB `Utf8View` columns). [RESULTS.md, Fused reads](perf/arrow/RESULTS.md#fused-reads-issue-90-1).
+- `diff_tables` row diff: 32 B hash per row per side (left only in parallel): 75 MB at 1M, 660 MB at 10M (single-threaded); 600/618 MB at 8M, 2.7/2.5 GB at 37M (1/18 threads); wall time `N·log N` across workers. Duplicate-key report, 18 threads: 39 MB (16 B keys) to 0.83 GB (1 KB keys) at 200k. Cell pass (both sides' spilled rows × column width, plus 2x `cells_changed`), 18 threads: 0.96 GB at 200k, about 4.8 GB at 1M (4.77 to 4.91 GB over five runs each of 0.13.0 and 0.14.0), 1.56 GB spill-dominated (150k rows, eight 512 B columns, one differing). The cell pass also copies each column whose two sides share a type once per partition, aligned for comparison (changed rows ÷ partitions × that column's width; byte-view columns are compared as their spilled large-offset type, so they copy in full): +0.3 GB (5.64 → 5.95 GB) for 500k changed rows/side, three equal 1 KB columns (`Utf8View`, `BinaryView`, `Utf8`), only an `Int64` differing, 2 threads, medians of 8 runs of 0.13.1 (identical to 0.13.0 on this path) and 0.14.0. Right-side duplicate keys, 18 threads: +0.95 GB per 500k repeated keys (2 KB of value columns, 1M rows/side); +85 MB per 1M keys the left lacks (8 B `int64` keys; the entry is 32 B at any key width). Kept rows pin their input batch, per side: the whole batch when over half is kept, its view data otherwise (~2.0 GB, 1 and 18 threads, for 100 removed rows over a 1M-row side with two 1 KB `Utf8View` columns). [RESULTS.md, Fused reads](perf/arrow/RESULTS.md#fused-reads-issue-90-1).
 - Temp disk, same call: each input spools to an anonymous file (`tempfile`: unlinked, mode 0600, no name). Changed rows spill to anonymous per-partition files, byte-view columns cast to a large-offset type (i32 caps at ~2 GB) and dictionaries decoded, independent of the partition count (~97 GB at 64 threads undecoded vs ~32 GB, wide pair); in parallel also one row per right key the left holds once that a later batch repeats (1.03 GB per 500k such keys, 2 KB of value columns, 1M rows/side, 2 and 64 threads). All resident until the cell pass ends: ~23.8 GB, wide pair (2/18/64 threads; Linux spill may be a RAM-backed `tmpfs`). Whole-process peaks, wide pair: ~29 GB at 18 threads, 40 GB at 2, 31 GB at 64 (33/46/34 GB on 0.11.2). A full temp filesystem raises `ValueError` naming `TMPDIR`.
 - The size gate peeks up to 50,000 rows or 64 MB decoded per side, holding one whole producer batch per side beyond it: a 49,999-row/side, 8 KB-cell, 100-row-batch pair peaks ~65 MB at 18 threads, 12 MB single-threaded, ~1.6 GB single-threaded (1.2 GB at 18 threads) for one whole-side batch.
-- None of the row-diff memory, temp-disk or size-gate-peek costs above has a built-in cap. Bound: row count; changed fraction; right-side duplicate keys (on the parallel path, one full-width row per repeated key, spilled or held); column widths (key-column width, for duplicate-heavy data; for the cell pass, the total width of all common value columns, since every one spills in full per changed row whether it changed or not, plus the rendered changed cells; any column's width for the size-gate peek, which buffers decoded cells whether or not they change); the producer's batch size; thread count, lowest at 18 threads (measured default), higher at 2 threads (two partitions each holding half the rows), slightly higher at 64: spill-dominated 2.39 GB at 2 threads, 1.96 GB at 64; output-dominated at 200k rows, 1.45 GB at 2, 1.25 GB at 64.
+- None of the row-diff memory, temp-disk or size-gate-peek costs above has a built-in cap. Bound: row count; changed fraction; right-side duplicate keys (on the parallel path, one full-width row per repeated key, spilled or held); column widths (key-column width, for duplicate-heavy data; for the cell pass, the total width of all common value columns, since every one spills in full per changed row whether it changed or not, plus the rendered changed cells; any column's width for the size-gate peek, which buffers decoded cells whether or not they change); the producer's batch size; thread count, lowest at 18 threads (measured default), higher at 2 threads (two partitions each holding half the rows), slightly higher at 64, and fewer partitions also enlarge the cell pass's aligned copy of each same-typed compared column (changed rows per partition × the widest such column): spill-dominated 2.39 GB at 2 threads, 1.96 GB at 64; output-dominated at 200k rows, 1.45 GB at 2, 1.25 GB at 64.
 - Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per batch from `ROW_DIFF_BATCH`), macOS on an Apple M-series laptop, 2026-09-24, same method as [Performance](#performance); the single-threaded hash-term figures carry run-to-run slack up to 2.7-3.8 GB at 37M rows/side, the other cell-pass and thread-count figures are medians over 3 runs, and the two 150k spill/output shapes over 5. See [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md) for the size-gate peek table and the decode-vs-undecoded partition figures, and [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs) for the implementation.
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the first row-diff bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/crates/onix-arrow/Cargo.toml

@@ -45,6 +45,9 @@ arrow-select = "=59.3.0"
 # file so only one is resident at once. Pinned to the same 59.3.0 as the other
 # arrow crates.
 arrow-ipc = "=59.3.0"
+# `arrow-ord`'s null-aware `distinct` kernel finds a partition's changed cells one
+# column at a time in the parallel cell pass.
+arrow-ord = "=59.3.0"
 tempfile = "3"
 # Row identity is a keyed 128-bit SipHash-1-3 (`siphasher`), keyed from 16
 # bytes of OS randomness per diff (`getrandom`), so the row-matching table on

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/crates/onix-arrow/examples/row_diff_profile.rs

@@ -1,6 +1,5 @@
-//! Per-pass wall-time and peak-RSS profile of one keyed row diff, the committed
-//! harness every row-diff performance change posts a before/after table from.
-//! Builds only with the `profile` feature, which the release wheel never enables.
+//! Per-pass wall-time and peak-RSS profile of one keyed row diff. Builds only
+//! with the `profile` feature, which the release wheel never enables.
 //!
 //! Each invocation runs a discarded warm-up diff, a timed uninstrumented diff
 //! (the `uninstrumented wall` line), and an instrumented diff whose passes make

{deepdiff_rs-0.13.0 → deepdiff_rs-0.14.0}/crates/onix-arrow/src/lib.rs

@@ -9,8 +9,9 @@
 //! tables use [`MemoryInput`]; a one-shot stream spools to a temporary file
 //! and implements [`TableInput`] over it, as the Python bindings do.
 //!
-//! See `src/row_diff.rs` for the row-matching and value-comparison rules, and
-//! `src/schema.rs` for the column type-normalization rules.
+//! See `src/row_diff.rs` for the row-matching passes, `docs/design/row-diff.md`
+//! for the algorithm, hashing, and value-comparison rules, and `src/schema.rs`
+//! for the column type-normalization rules.
 //!
 //! # Example
 //!
@@ -77,8 +78,9 @@ pub use schema::{ChangeKind, SchemaChange, diff_schemas};
 pub use table_diff::{TableDiff, TableDiffSummary};
 
 /// The maximum column-type nesting depth [`diff_tables`] will compare; deeper is refused
-/// with [`TableDiffError::MaxDepthExceeded`], turning a native-stack overflow (recursive
-/// comparison, `Display`, `Clone`/`Drop`) into an error before it can run.
+/// with [`TableDiffError::MaxDepthExceeded`], bounding the native-stack recursion in
+/// comparison, `Display`, `Clone`, and the drop of values onix builds from accepted
+/// input — not a caller's own drop of a `DataType` it built past this depth.
 /// Per-level cost is measured by `crates/onix-arrow/examples/type_stack_cost.rs`.
 pub const MAX_NESTING_DEPTH: usize = 128;