deepdiff-rs 0.9.3__tar.gz → 0.11.0__tar.gz

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/Cargo.lock

@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.9.3"
+version = "0.11.0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -646,7 +646,7 @@ dependencies = [
 
 [[package]]
 name = "onix-cli"
-version = "0.9.3"
+version = "0.11.0"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -654,8 +654,10 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.9.3"
+version = "0.11.0"
 dependencies = [
+ "num-bigint",
+ "num-traits",
  "proptest",
  "serde",
  "serde_json",
@@ -665,11 +667,12 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.9.3"
+version = "0.11.0"
 dependencies = [
  "arrow-array",
  "arrow-ipc",
  "arrow-schema",
+ "num-bigint",
  "onix-arrow",
  "onix-core",
  "pyo3",

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.9.3"
+version = "0.11.0"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.9.3
+Version: 0.11.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -195,7 +195,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on a seeded 5 GB parquet pair and its 1M-row subset — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): onix's single-threaded row diff trails both parallelized baselines at both sizes; see the linked results for the figures; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared, still trails by a much larger multiple, see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -204,7 +204,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions, including which types a `dict` key may be); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date`/`time`/`timedelta` comes back as a real one of those — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
+- `diff_tables(left, right, key=[...], threads=None) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `threads` sets the row diff's worker count — `None` uses the machine's available parallelism, `1` runs single-threaded, and a value below 1 or above 1024 raises `ValueError` (the diff spawns one worker per thread); the result is byte-identical at any value. `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -236,7 +236,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. An `int` of any magnitude converts and keeps its exact value (an arbitrary-precision integer beyond `i64`/`u64` is compared, ordered, hashed and rendered by its full digits), while a custom object raises `TypeError` naming the exact path it was found at. Under `ignore_order`, comparing an integer beyond `f64::MAX` (about `2**1024`) reports the change deterministically where real DeepDiff raises `OverflowError` (its distance function calls `float()` on it unguarded), a documented crash-class divergence like the datetime one below (see [`tests/golden/README.md`](tests/golden/README.md)). Reading JSON *text* (`diff_json` and the CLI, not the Python-object path), an integer beyond `i64`/`u64` parses as the nearest `float`, so two documents whose integers differ only past that range compare **equal** and diff to `{}` — a limitation of the JSON number reader, not the value model, tracked in [#92](https://github.com/ksco92/onix/issues/92). A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, or `datetime.timedelta` (including `namedtuple` and pandas' `Timestamp`) converts and compares as its base type but carries its own class name into a `type_changes` entry, matching DeepDiff's `type(obj).__name__` reporting, with three exceptions: a calendar-type (`datetime`/`date`/`time`/`timedelta`) subclass held as a `set`/`frozenset` member compares as its base with no `type_changes`, matching DeepDiff, since set membership has no pairwise comparison to report one against; a `tuple`/`frozenset` subclass, including a `namedtuple`, is not accepted as a `set`/`frozenset` member at all and raises `TypeError` where DeepDiff accepts and compares it by value (a documented divergence); and `namedtuple` diffs positionally rather than by field (also documented). A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. See [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
@@ -250,7 +250,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M); the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 67 GB at 16.875M rows/side, see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/README.md

@@ -177,7 +177,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on a seeded 5 GB parquet pair and its 1M-row subset — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): onix's single-threaded row diff trails both parallelized baselines at both sizes; see the linked results for the figures; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared, still trails by a much larger multiple, see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -186,7 +186,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions, including which types a `dict` key may be); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date`/`time`/`timedelta` comes back as a real one of those — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
+- `diff_tables(left, right, key=[...], threads=None) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `threads` sets the row diff's worker count — `None` uses the machine's available parallelism, `1` runs single-threaded, and a value below 1 or above 1024 raises `ValueError` (the diff spawns one worker per thread); the result is byte-identical at any value. `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -218,7 +218,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. An `int` of any magnitude converts and keeps its exact value (an arbitrary-precision integer beyond `i64`/`u64` is compared, ordered, hashed and rendered by its full digits), while a custom object raises `TypeError` naming the exact path it was found at. Under `ignore_order`, comparing an integer beyond `f64::MAX` (about `2**1024`) reports the change deterministically where real DeepDiff raises `OverflowError` (its distance function calls `float()` on it unguarded), a documented crash-class divergence like the datetime one below (see [`tests/golden/README.md`](tests/golden/README.md)). Reading JSON *text* (`diff_json` and the CLI, not the Python-object path), an integer beyond `i64`/`u64` parses as the nearest `float`, so two documents whose integers differ only past that range compare **equal** and diff to `{}` — a limitation of the JSON number reader, not the value model, tracked in [#92](https://github.com/ksco92/onix/issues/92). A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, or `datetime.timedelta` (including `namedtuple` and pandas' `Timestamp`) converts and compares as its base type but carries its own class name into a `type_changes` entry, matching DeepDiff's `type(obj).__name__` reporting, with three exceptions: a calendar-type (`datetime`/`date`/`time`/`timedelta`) subclass held as a `set`/`frozenset` member compares as its base with no `type_changes`, matching DeepDiff, since set membership has no pairwise comparison to report one against; a `tuple`/`frozenset` subclass, including a `namedtuple`, is not accepted as a `set`/`frozenset` member at all and raises `TypeError` where DeepDiff accepts and compares it by value (a documented divergence); and `namedtuple` diffs positionally rather than by field (also documented). A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. See [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
@@ -232,7 +232,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M); the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 67 GB at 16.875M rows/side, see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/crates/onix-arrow/examples/row_diff_rss.rs

@@ -19,6 +19,9 @@
 //! # duplicate-heavy shape: every key duplicated, wide string key
 //! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 dup 16
 //! /usr/bin/time -l target/release/examples/row_diff_rss 200000 dup 1024
+//! # size-gate peek: identical wide-cell sides (zero changes); ROW_DIFF_BATCH
+//! # sets the producer's batch size, ROW_DIFF_THREADS the worker count
+//! ROW_DIFF_BATCH=100 ROW_DIFF_THREADS=18 /usr/bin/time -l target/release/examples/row_diff_rss 49999 widesame 8192
 //! ```
 //!
 //! Each side is generated on the fly, batch by batch, and nothing is retained
@@ -46,8 +49,19 @@ use arrow_array::{ArrayRef, Int64Array, RecordBatch, RecordBatchReader, StringAr
 use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
 use onix_arrow::{TableDiffError, TableDiffOptions, TableInput, diff_tables};
 
+/// The default rows per generated batch; override with `ROW_DIFF_BATCH` to
+/// simulate a streamed input of many small batches.
 const BATCH: i64 = 65_536;
 
+/// The rows-per-batch used by the generator, from `ROW_DIFF_BATCH` or [`BATCH`].
+fn batch_rows() -> i64 {
+    std::env::var("ROW_DIFF_BATCH")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .filter(|&n| n > 0)
+        .unwrap_or(BATCH)
+}
+
 /// The generated table shape.
 #[derive(Clone, Copy)]
 enum Shape {
@@ -69,6 +83,7 @@ struct Generated {
     schema: SchemaRef,
     rows: i64,
     shape: Shape,
+    batch: i64,
 }
 
 impl TableInput for Generated {
@@ -81,6 +96,7 @@ impl TableInput for Generated {
             schema: self.schema.clone(),
             rows: self.rows,
             shape: self.shape,
+            batch: self.batch,
             next: 0,
         }))
     }
@@ -90,6 +106,7 @@ struct GenReader {
     schema: SchemaRef,
     rows: i64,
     shape: Shape,
+    batch: i64,
     next: i64,
 }
 
@@ -100,7 +117,7 @@ impl Iterator for GenReader {
         if self.next >= self.rows {
             return None;
         }
-        let end = (self.next + BATCH).min(self.rows);
+        let end = (self.next + self.batch).min(self.rows);
         let columns: Vec<ArrayRef> = match self.shape {
             Shape::Linear {
                 id_offset,
@@ -144,6 +161,23 @@ impl RecordBatchReader for GenReader {
     }
 }
 
+/// Options for the diff, honoring a `ROW_DIFF_THREADS` override so the parallel
+/// path's peak RSS can be compared against the single-threaded baseline; unset
+/// uses the default (available parallelism).
+fn options_from_env(key: &str) -> TableDiffOptions {
+    let mut options = TableDiffOptions::new(vec![key.to_string()]);
+    if let Some(threads) = std::env::var("ROW_DIFF_THREADS")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .and_then(std::num::NonZeroUsize::new)
+    {
+        options = options
+            .with_threads(threads)
+            .expect("ROW_DIFF_THREADS within MAX_THREADS");
+    }
+    options
+}
+
 fn main() {
     let args: Vec<String> = std::env::args().collect();
     let rows: i64 = args
@@ -173,11 +207,14 @@ fn main() {
                 format!(" (dup, key_width={width})"),
             )
         }
-        "wide" => {
+        "wide" | "widesame" => {
             let schema = Arc::new(Schema::new(vec![
                 Field::new("id", DataType::Int64, false),
                 Field::new("value", DataType::Utf8, false),
             ]));
+            // `widesame` fills both sides identically (zero changes), isolating
+            // the size gate's peek buffer; `wide` differs, changing every row.
+            let right_fill = if mode == "widesame" { b'a' } else { b'b' };
             (
                 schema,
                 Shape::Wide {
@@ -186,10 +223,10 @@ fn main() {
                 },
                 Shape::Wide {
                     value_width: width,
-                    fill: b'b',
+                    fill: right_fill,
                 },
                 "id",
-                format!(" (wide, value_width={width}, all changed)"),
+                format!(" ({mode}, value_width={width})"),
             )
         }
         _ => {
@@ -222,24 +259,28 @@ fn main() {
         }
     };
 
+    let batch = batch_rows();
     let left = Generated {
         schema: schema.clone(),
         rows,
         shape: left_shape,
+        batch,
     };
     let right = Generated {
         schema,
         rows,
         shape: right_shape,
+        batch,
     };
 
+    let options = options_from_env(key);
     let start = std::time::Instant::now();
-    let diff = diff_tables(&left, &right, &TableDiffOptions::new(vec![key.to_string()]))
-        .expect("diff succeeds");
+    let diff = diff_tables(&left, &right, &options).expect("diff succeeds");
     let elapsed = start.elapsed();
     let summary = diff.summary();
 
     println!("rows per side: {rows}{label}");
+    println!("threads: {}", options.threads());
     println!("wall: {:.2}s", elapsed.as_secs_f64());
     println!(
         "rows_added={} rows_removed={} rows_changed={} duplicate_keys={} cells_changed={}",

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/crates/onix-arrow/src/error.rs

@@ -140,6 +140,24 @@ pub enum TableDiffError {
         /// The underlying `serde_json` error's message.
         message: String,
     },
+    /// A worker thread in the parallel row diff panicked. The panic is caught
+    /// and surfaced as this typed error rather than allowed to abort the
+    /// process, so a bug or an unexpected failure inside one worker fails the
+    /// diff recoverably.
+    WorkerPanicked {
+        /// The panic payload, when it was a string.
+        message: String,
+    },
+    /// The requested worker-thread count exceeds [`crate::MAX_THREADS`]. The
+    /// row diff spawns one worker per requested thread, so an unbounded count
+    /// is refused before any thread is spawned rather than risking thread
+    /// exhaustion.
+    ThreadCountTooLarge {
+        /// The requested thread count.
+        threads: usize,
+        /// The maximum accepted, [`crate::MAX_THREADS`].
+        max: usize,
+    },
 }
 
 impl fmt::Display for TableDiffError {
@@ -198,6 +216,14 @@ impl fmt::Display for TableDiffError {
             TableDiffError::Json { message } => {
                 write!(f, "failed to serialize the table diff to JSON: {message}")
             }
+            TableDiffError::WorkerPanicked { message } => {
+                write!(f, "a parallel row-diff worker thread panicked: {message}")
+            }
+            TableDiffError::ThreadCountTooLarge { threads, max } => write!(
+                f,
+                "the requested thread count {threads} exceeds the maximum of {max}; \
+                 pass a smaller `threads` value"
+            ),
         }
     }
 }
@@ -265,6 +291,16 @@ mod tests {
         assert!(message.contains("RunEndEncoded"));
     }
 
+    #[test]
+    fn worker_panicked_message_carries_the_payload() {
+        let error = TableDiffError::WorkerPanicked {
+            message: "kaboom".to_string(),
+        };
+        let message = error.to_string();
+        assert!(message.contains("worker thread panicked"));
+        assert!(message.contains("kaboom"));
+    }
+
     #[test]
     fn key_type_mismatch_message_names_the_column() {
         let error = TableDiffError::KeyTypeMismatch {
@@ -275,6 +311,17 @@ mod tests {
         assert!(message.contains("same type on both sides"));
     }
 
+    #[test]
+    fn thread_count_too_large_message_names_count_and_max() {
+        let error = TableDiffError::ThreadCountTooLarge {
+            threads: 5000,
+            max: 1024,
+        };
+        let message = error.to_string();
+        assert!(message.contains("5000"));
+        assert!(message.contains("1024"));
+    }
+
     #[test]
     fn render_message_names_the_column() {
         let error = TableDiffError::Render {

{deepdiff_rs-0.9.3 → deepdiff_rs-0.11.0}/crates/onix-arrow/src/lib.rs

@@ -144,6 +144,10 @@ pub use table_diff::{TableDiff, TableDiffSummary};
 /// backstop.
 pub const MAX_NESTING_DEPTH: usize = 128;
 
+/// The maximum worker-thread count for the row diff. A larger `threads` is
+/// refused with [`TableDiffError::ThreadCountTooLarge`] before any thread spawns.
+pub const MAX_THREADS: usize = 1024;
+
 /// Diffs two tables presented as re-openable [`TableInput`]s.
 ///
 /// See the [crate-level docs](crate) for the type-comparison rules, the
@@ -224,7 +228,14 @@ pub fn diff_tables(
         }
     }
 
-    let rows = row_diff::diff_rows(left, right, &left_schema, &right_schema, options.key())?;
+    let rows = row_diff::diff_rows(
+        left,
+        right,
+        &left_schema,
+        &right_schema,
+        options.key(),
+        options.threads(),
+    )?;
 
     Ok(TableDiff::new(changes, rows))
 }

deepdiff_rs-0.11.0/crates/onix-arrow/src/options.rs

@@ -0,0 +1,135 @@
+//! Options controlling a table diff.
+
+use std::num::NonZeroUsize;
+
+use crate::error::TableDiffError;
+
+/// Options for [`crate::diff_tables`].
+///
+/// The key columns are the table's primary key: rows are matched across the
+/// two inputs by their values (in the later row-diff versions), and the key
+/// must be non-empty. Later versions add more fields (value-comparison
+/// tolerances and the like); construct this through [`TableDiffOptions::new`]
+/// rather than a struct literal so those additions stay backward compatible.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TableDiffOptions {
+    /// The key columns, in the order given. Required and non-empty; an empty
+    /// key makes [`crate::diff_tables`] return
+    /// [`crate::TableDiffError::EmptyKey`].
+    key: Vec<String>,
+    /// Worker threads the row diff hashes and classifies rows with. Defaults to
+    /// the machine's available parallelism; `1` runs the single-threaded path.
+    /// The result is byte-identical at any value.
+    threads: NonZeroUsize,
+}
+
+impl TableDiffOptions {
+    /// Creates options keyed on `key`, with [`threads`](Self::threads)
+    /// defaulting to the machine's available parallelism.
+    ///
+    /// No validation happens here — an empty `key` is reported by
+    /// [`crate::diff_tables`] so every misuse surfaces through one error
+    /// channel.
+    #[must_use]
+    pub fn new(key: Vec<String>) -> Self {
+        Self {
+            key,
+            threads: default_threads(),
+        }
+    }
+
+    /// Sets the number of worker threads the row diff uses. `1` selects the
+    /// single-threaded path; higher values partition the hash and classify
+    /// work across that many threads. The diff's output is byte-identical at
+    /// any value.
+    ///
+    /// # Errors
+    ///
+    /// [`TableDiffError::ThreadCountTooLarge`] if `threads` exceeds
+    /// [`crate::MAX_THREADS`] — the row diff spawns one worker per thread, so
+    /// the count is bounded here, before any thread or buffer is allocated.
+    pub fn with_threads(mut self, threads: NonZeroUsize) -> Result<Self, TableDiffError> {
+        if threads.get() > crate::MAX_THREADS {
+            return Err(TableDiffError::ThreadCountTooLarge {
+                threads: threads.get(),
+                max: crate::MAX_THREADS,
+            });
+        }
+        self.threads = threads;
+        Ok(self)
+    }
+
+    /// The key columns, in the order supplied.
+    #[must_use]
+    pub fn key(&self) -> &[String] {
+        &self.key
+    }
+
+    /// The number of worker threads the row diff uses.
+    #[must_use]
+    pub fn threads(&self) -> NonZeroUsize {
+        self.threads
+    }
+}
+
+/// The machine's available parallelism, or `1` when it cannot be queried.
+fn default_threads() -> NonZeroUsize {
+    std::thread::available_parallelism().unwrap_or(NonZeroUsize::MIN)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{TableDiffOptions, default_threads};
+    use std::num::NonZeroUsize;
+
+    #[test]
+    fn new_preserves_key_order() {
+        let options = TableDiffOptions::new(vec!["b".to_string(), "a".to_string()]);
+        assert_eq!(options.key(), &["b".to_string(), "a".to_string()]);
+    }
+
+    #[test]
+    fn empty_key_is_accepted_by_the_constructor() {
+        let options = TableDiffOptions::new(Vec::new());
+        assert!(options.key().is_empty());
+    }
+
+    #[test]
+    fn threads_defaults_to_available_parallelism() {
+        let options = TableDiffOptions::new(vec!["id".to_string()]);
+        assert_eq!(options.threads(), default_threads());
+    }
+
+    #[test]
+    fn default_threads_is_the_machine_parallelism() {
+        let expected = std::thread::available_parallelism().map_or(1, NonZeroUsize::get);
+        assert_eq!(default_threads().get(), expected);
+    }
+
+    #[test]
+    fn with_threads_overrides_the_default() {
+        let options = TableDiffOptions::new(vec!["id".to_string()])
+            .with_threads(NonZeroUsize::new(4).unwrap())
+            .unwrap();
+        assert_eq!(options.threads().get(), 4);
+    }
+
+    #[test]
+    fn with_threads_at_the_ceiling_is_accepted() {
+        let options = TableDiffOptions::new(vec!["id".to_string()])
+            .with_threads(NonZeroUsize::new(crate::MAX_THREADS).unwrap())
+            .unwrap();
+        assert_eq!(options.threads().get(), crate::MAX_THREADS);
+    }
+
+    #[test]
+    fn with_threads_above_the_ceiling_errors() {
+        let result = TableDiffOptions::new(vec!["id".to_string()])
+            .with_threads(NonZeroUsize::new(crate::MAX_THREADS + 1).unwrap());
+        assert!(matches!(
+            result,
+            Err(crate::error::TableDiffError::ThreadCountTooLarge { threads, max })
+                if threads == crate::MAX_THREADS + 1 && max == crate::MAX_THREADS
+        ));
+    }
+}