deepdiff-rs 0.10.0__tar.gz → 0.11.1__tar.gz

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/Cargo.lock

@@ -629,11 +629,12 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.10.0"
+version = "0.11.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
+ "arrow-ipc",
  "arrow-schema",
  "arrow-select",
  "getrandom 0.3.4",
@@ -642,11 +643,12 @@ dependencies = [
  "serde",
  "serde_json",
  "siphasher",
+ "tempfile",
 ]
 
 [[package]]
 name = "onix-cli"
-version = "0.10.0"
+version = "0.11.1"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -654,8 +656,10 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.10.0"
+version = "0.11.1"
 dependencies = [
+ "num-bigint",
+ "num-traits",
  "proptest",
  "serde",
  "serde_json",
@@ -665,17 +669,16 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.10.0"
+version = "0.11.1"
 dependencies = [
  "arrow-array",
- "arrow-ipc",
  "arrow-schema",
+ "num-bigint",
  "onix-arrow",
  "onix-core",
  "pyo3",
  "pyo3-arrow",
  "serde_json",
- "tempfile",
 ]
 
 [[package]]

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.10.0"
+version = "0.11.1"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.10.0
+Version: 0.11.1
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -195,7 +195,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on a seeded 5 GB parquet pair and its 1M-row subset — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' rather than the several-fold gap the earlier single-threaded row diff showed; see the linked results for the figures and the before/after; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, now spills each side's changed rows by key-hash partition and renders them across every core, cutting its 5 GB-per-side wall time 6.14x (149.674 -> 24.375 s; about 3.8x DuckDB and 6.2x polars) and its peak RSS 2.02x (about 67 GB -> 33.1 GB), see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -236,7 +236,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. An `int` of any magnitude converts and keeps its exact value (an arbitrary-precision integer beyond `i64`/`u64` is compared, ordered, hashed and rendered by its full digits), while a custom object raises `TypeError` naming the exact path it was found at. Under `ignore_order`, comparing an integer beyond `f64::MAX` (about `2**1024`) reports the change deterministically where real DeepDiff raises `OverflowError` (its distance function calls `float()` on it unguarded), a documented crash-class divergence like the datetime one below (see [`tests/golden/README.md`](tests/golden/README.md)). Reading JSON *text* (`diff_json` and the CLI, not the Python-object path), an integer beyond `i64`/`u64` parses as the nearest `float`, so two documents whose integers differ only past that range compare **equal** and diff to `{}` — a limitation of the JSON number reader, not the value model, tracked in [#92](https://github.com/ksco92/onix/issues/92). A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, or `datetime.timedelta` (including `namedtuple` and pandas' `Timestamp`) converts and compares as its base type but carries its own class name into a `type_changes` entry, matching DeepDiff's `type(obj).__name__` reporting, with three exceptions: a calendar-type (`datetime`/`date`/`time`/`timedelta`) subclass held as a `set`/`frozenset` member compares as its base with no `type_changes`, matching DeepDiff, since set membership has no pairwise comparison to report one against; a `tuple`/`frozenset` subclass, including a `namedtuple`, is not accepted as a `set`/`frozenset` member at all and raises `TypeError` where DeepDiff accepts and compares it by value (a documented divergence); and `namedtuple` diffs positionally rather than by field (also documented). A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. See [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
@@ -250,7 +250,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which spills each side's changed value rows — every common value column of every changed row, changed or not — to anonymous per-key-hash-partition Arrow IPC files and compares and renders one partition at a time across the workers. Its resident cost is the spilled changed value rows (the changed-row count times the total width of the common value columns, both sides — resident where written temp pages count, e.g. macOS or a RAM-backed `tmpfs`) plus about twice the `cells_changed` output (its one out-of-place reorder) plus the per-row hash vectors; the spill term dominates for wide rows with few changed cells, the output term for many changed cells. Measured (`row_diff_rss`, 18 threads): a 1 KB `string` cell changed on every row (output-dominated) peaks at 0.92 GB at 100k rows/side, 1.31 GB at 200k, and 4.81 GB at 1M (the single-threaded path, holding both sides, is 8.30 GB); eight 512 B value columns with only one differing (spill-dominated: 150k changed cells, ~160 MB output) peaks at about 2.09 GB at 150k rows/side, versus 0.84 GB with one such column (medians over 5 runs); these are at 18 threads, the low point of the range -- the same two shapes peak higher at the ends (spill-dominated 3.03 GB at 2 threads and 2.09 GB at 64; output-dominated, 200k, 1.98 GB at 2 and 1.49 GB at 64) as fewer partitions enlarge the resident chunk and the 64-way framing adds a little; two narrow `int64` columns with every row changed peak at 0.38 GB at 1M; the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 33 GB whole-process at 16.875M rows/side at 18 threads, down from about 67 GB before the streaming cell pass. The two Arrow types whose `take` retains data beyond the selected rows are decoded before the spill -- byte-view columns (`Utf8View`/`BinaryView`) cast to their large i64-offset non-view type (`LargeUtf8`/`LargeBinary`, since one batch's retained view buffers can exceed the i32 offset type's ~2 GB ceiling) and dictionaries (what polars and DuckDB emit for strings) decoded to their value type -- so the spill stays compact and independent of the partition count; without the decode it would grow with the partition count (about 97 GB at 64 threads for the wide pair vs about 34 GB with it), see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), and the cell pass additionally spills both sides' changed value rows to anonymous per-partition files (byte-view columns cast to their non-view type and dictionaries decoded to their value type first, so the spill is compact and does not grow with the thread count); the input spools and the partition spill are resident together, and for the full wide pair the whole process (input spool + spill + working set) peaks at about 33 GB at 18 threads, 45 GB at 2, and 34 GB at 64 (on Linux the spill may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data; for the cell pass, the total width of the common value columns — every one is spilled in full for each changed row, changed or not — plus the rendered changed cells; and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), the producer's batch size, and the thread count (the cell pass's peak is lowest at the default and higher at 2 threads, where 2 partitions hold half the rows, and slightly higher at 64, the partition cap), since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/README.md

@@ -177,7 +177,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on a seeded 5 GB parquet pair and its 1M-row subset — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' rather than the several-fold gap the earlier single-threaded row diff showed; see the linked results for the figures and the before/after; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, now spills each side's changed rows by key-hash partition and renders them across every core, cutting its 5 GB-per-side wall time 6.14x (149.674 -> 24.375 s; about 3.8x DuckDB and 6.2x polars) and its peak RSS 2.02x (about 67 GB -> 33.1 GB), see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -218,7 +218,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. An `int` of any magnitude converts and keeps its exact value (an arbitrary-precision integer beyond `i64`/`u64` is compared, ordered, hashed and rendered by its full digits), while a custom object raises `TypeError` naming the exact path it was found at. Under `ignore_order`, comparing an integer beyond `f64::MAX` (about `2**1024`) reports the change deterministically where real DeepDiff raises `OverflowError` (its distance function calls `float()` on it unguarded), a documented crash-class divergence like the datetime one below (see [`tests/golden/README.md`](tests/golden/README.md)). Reading JSON *text* (`diff_json` and the CLI, not the Python-object path), an integer beyond `i64`/`u64` parses as the nearest `float`, so two documents whose integers differ only past that range compare **equal** and diff to `{}` — a limitation of the JSON number reader, not the value model, tracked in [#92](https://github.com/ksco92/onix/issues/92). A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, or `datetime.timedelta` (including `namedtuple` and pandas' `Timestamp`) converts and compares as its base type but carries its own class name into a `type_changes` entry, matching DeepDiff's `type(obj).__name__` reporting, with three exceptions: a calendar-type (`datetime`/`date`/`time`/`timedelta`) subclass held as a `set`/`frozenset` member compares as its base with no `type_changes`, matching DeepDiff, since set membership has no pairwise comparison to report one against; a `tuple`/`frozenset` subclass, including a `namedtuple`, is not accepted as a `set`/`frozenset` member at all and raises `TypeError` where DeepDiff accepts and compares it by value (a documented divergence); and `namedtuple` diffs positionally rather than by field (also documented). A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. See [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
@@ -232,7 +232,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which spills each side's changed value rows — every common value column of every changed row, changed or not — to anonymous per-key-hash-partition Arrow IPC files and compares and renders one partition at a time across the workers. Its resident cost is the spilled changed value rows (the changed-row count times the total width of the common value columns, both sides — resident where written temp pages count, e.g. macOS or a RAM-backed `tmpfs`) plus about twice the `cells_changed` output (its one out-of-place reorder) plus the per-row hash vectors; the spill term dominates for wide rows with few changed cells, the output term for many changed cells. Measured (`row_diff_rss`, 18 threads): a 1 KB `string` cell changed on every row (output-dominated) peaks at 0.92 GB at 100k rows/side, 1.31 GB at 200k, and 4.81 GB at 1M (the single-threaded path, holding both sides, is 8.30 GB); eight 512 B value columns with only one differing (spill-dominated: 150k changed cells, ~160 MB output) peaks at about 2.09 GB at 150k rows/side, versus 0.84 GB with one such column (medians over 5 runs); these are at 18 threads, the low point of the range -- the same two shapes peak higher at the ends (spill-dominated 3.03 GB at 2 threads and 2.09 GB at 64; output-dominated, 200k, 1.98 GB at 2 and 1.49 GB at 64) as fewer partitions enlarge the resident chunk and the 64-way framing adds a little; two narrow `int64` columns with every row changed peak at 0.38 GB at 1M; the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 33 GB whole-process at 16.875M rows/side at 18 threads, down from about 67 GB before the streaming cell pass. The two Arrow types whose `take` retains data beyond the selected rows are decoded before the spill -- byte-view columns (`Utf8View`/`BinaryView`) cast to their large i64-offset non-view type (`LargeUtf8`/`LargeBinary`, since one batch's retained view buffers can exceed the i32 offset type's ~2 GB ceiling) and dictionaries (what polars and DuckDB emit for strings) decoded to their value type -- so the spill stays compact and independent of the partition count; without the decode it would grow with the partition count (about 97 GB at 64 threads for the wide pair vs about 34 GB with it), see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), and the cell pass additionally spills both sides' changed value rows to anonymous per-partition files (byte-view columns cast to their non-view type and dictionaries decoded to their value type first, so the spill is compact and does not grow with the thread count); the input spools and the partition spill are resident together, and for the full wide pair the whole process (input spool + spill + working set) peaks at about 33 GB at 18 threads, 45 GB at 2, and 34 GB at 64 (on Linux the spill may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data; for the cell pass, the total width of the common value columns — every one is spilled in full for each changed row, changed or not — plus the rendered changed cells; and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), the producer's batch size, and the thread count (the cell pass's peak is lowest at the default and higher at 2 threads, where 2 partitions hold half the rows, and slightly higher at 64, the partition cap), since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/crates/onix-arrow/Cargo.toml

@@ -27,10 +27,19 @@ arrow-buffer = "=59.3.0"
 # cell's value to its canonical string (`display::ArrayFormatter`);
 # `arrow-select` filters the materialized rows, concatenates the output row
 # batches, and `take`s the key columns into the per-cell diff. The row diff
-# re-reads each side through the caller's re-openable `TableInput`, so no IPC
-# spooling lives here — the Python bindings own that.
+# re-reads each *input* side through the caller's re-openable `TableInput`. The
+# `spool` module owns the anonymous temporary Arrow IPC primitive used both by
+# the Python bindings' input spool and by the parallel cell pass, which spills
+# its changed-row key-hash partitions so it holds one partition in memory at a
+# time rather than both sides' full changed rows.
 arrow-cast = "=59.3.0"
 arrow-select = "=59.3.0"
+# The `spool` module writes to an anonymous `tempfile::tempfile()` as an Arrow
+# IPC stream and re-reads it; the cell pass spills one key-hash partition per
+# file so only one is resident at once. Pinned to the same 59.3.0 as the other
+# arrow crates.
+arrow-ipc = "=59.3.0"
+tempfile = "3"
 # Row identity is a keyed 128-bit SipHash-1-3 (`siphasher`), keyed from 16
 # bytes of OS randomness per diff (`getrandom`), so the row-matching table on
 # this default path cannot be forced into collisions by chosen input.

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/crates/onix-arrow/examples/row_diff_rss.rs

@@ -16,6 +16,8 @@
 //! # every row changed with a wide (1 KB) string cell: the rendering worst case
 //! /usr/bin/time -l target/release/examples/row_diff_rss 100000 wide 1024
 //! /usr/bin/time -l target/release/examples/row_diff_rss 200000 wide 1024
+//! # wide rows, few changed cells: id + 8 512-byte columns, only one differing
+//! ROW_DIFF_THREADS=18 /usr/bin/time -l target/release/examples/row_diff_rss 150000 manycols 8 512
 //! # duplicate-heavy shape: every key duplicated, wide string key
 //! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 dup 16
 //! /usr/bin/time -l target/release/examples/row_diff_rss 200000 dup 1024
@@ -76,6 +78,15 @@ enum Shape {
     /// is changed and every changed cell renders `value_width` bytes — the
     /// wide-cell worst case for the per-cell diff's rendering memory.
     Wide { value_width: usize, fill: u8 },
+    /// `(id, value0..value{ncols})` each a `width`-byte string; only `value0`
+    /// differs between the sides (`first_fill`), so every row is changed but only
+    /// one cell per row — the wide-rows-few-changed-cells case, where the spill
+    /// holds every common value column of every changed row though few change.
+    ManyCols {
+        ncols: usize,
+        width: usize,
+        first_fill: u8,
+    },
 }
 
 /// A table generated on demand, retaining nothing between batches.
@@ -148,6 +159,25 @@ impl Iterator for GenReader {
                 let values: StringArray = (self.next..end).map(|_| Some(cell.as_str())).collect();
                 vec![Arc::new(ids), Arc::new(values)]
             }
+            Shape::ManyCols {
+                ncols,
+                width,
+                first_fill,
+            } => {
+                let ids: Int64Array = (self.next..end).map(Some).collect();
+                let mut columns: Vec<ArrayRef> = Vec::with_capacity(ncols + 1);
+                columns.push(Arc::new(ids));
+                for c in 0..ncols {
+                    // Only value0 differs between the sides; the rest are equal,
+                    // so every row is changed but only one cell per row.
+                    let fill = if c == 0 { first_fill } else { b'a' };
+                    let cell = String::from_utf8(vec![fill; width]).unwrap();
+                    let values: StringArray =
+                        (self.next..end).map(|_| Some(cell.as_str())).collect();
+                    columns.push(Arc::new(values));
+                }
+                columns
+            }
         };
         self.next = end;
 
@@ -178,21 +208,15 @@ fn options_from_env(key: &str) -> TableDiffOptions {
     options
 }
 
-fn main() {
-    let args: Vec<String> = std::env::args().collect();
-    let rows: i64 = args
-        .get(1)
-        .and_then(|a| a.parse().ok())
-        .unwrap_or(1_000_000);
-    // Mode selects the shape: "dup" (duplicate-heavy), "wide" (all rows changed,
-    // wide string value column), "nochange"/"allchange"/default (linear int).
-    let mode = args.get(2).map_or("", String::as_str);
-    let width: usize = args
-        .get(3)
-        .and_then(|a| a.parse().ok())
-        .unwrap_or(if mode == "wide" { 1024 } else { 16 });
-
-    let (schema, left_shape, right_shape, key, label) = match mode {
+/// Builds the (schema, left shape, right shape, key column, label) for a mode.
+#[allow(clippy::too_many_lines)]
+fn build_case(
+    mode: &str,
+    rows: i64,
+    width: usize,
+    args: &[String],
+) -> (SchemaRef, Shape, Shape, &'static str, String) {
+    match mode {
         "dup" => {
             let schema = Arc::new(Schema::new(vec![
                 Field::new("key", DataType::Utf8, false),
@@ -229,6 +253,30 @@ fn main() {
                 format!(" ({mode}, value_width={width})"),
             )
         }
+        "manycols" => {
+            let ncols: usize = args.get(3).and_then(|a| a.parse().ok()).unwrap_or(8);
+            let cell_width: usize = args.get(4).and_then(|a| a.parse().ok()).unwrap_or(512);
+            let mut fields = vec![Field::new("id", DataType::Int64, false)];
+            for c in 0..ncols {
+                fields.push(Field::new(format!("value{c}"), DataType::Utf8, false));
+            }
+            let schema = Arc::new(Schema::new(fields));
+            (
+                schema,
+                Shape::ManyCols {
+                    ncols,
+                    width: cell_width,
+                    first_fill: b'a',
+                },
+                Shape::ManyCols {
+                    ncols,
+                    width: cell_width,
+                    first_fill: b'b',
+                },
+                "id",
+                format!(" (manycols, ncols={ncols}, width={cell_width})"),
+            )
+        }
         _ => {
             let schema = Arc::new(Schema::new(vec![
                 Field::new("id", DataType::Int64, false),
@@ -257,7 +305,25 @@ fn main() {
                 label.to_string(),
             )
         }
-    };
+    }
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    let rows: i64 = args
+        .get(1)
+        .and_then(|a| a.parse().ok())
+        .unwrap_or(1_000_000);
+    // Mode selects the shape: "dup" (duplicate-heavy), "wide" (all rows changed,
+    // wide string value column), "manycols" (wide rows, one differing column),
+    // "nochange"/"allchange"/default (linear int).
+    let mode = args.get(2).map_or("", String::as_str);
+    let width: usize = args
+        .get(3)
+        .and_then(|a| a.parse().ok())
+        .unwrap_or(if mode == "wide" { 1024 } else { 16 });
+
+    let (schema, left_shape, right_shape, key, label) = build_case(mode, rows, width, &args);
 
     let batch = batch_rows();
     let left = Generated {

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.1}/crates/onix-arrow/src/lib.rs

@@ -106,6 +106,7 @@ mod json_rows;
 mod options;
 mod row_diff;
 mod schema;
+pub mod spool;
 mod table_diff;
 
 pub use error::{Side, TableDiffError};