deepdiff-rs 0.11.0__tar.gz → 0.11.2__tar.gz

{deepdiff_rs-0.11.0 → deepdiff_rs-0.11.2}/Cargo.lock

@@ -629,11 +629,12 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.11.0"
+version = "0.11.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
  "arrow-cast",
+ "arrow-ipc",
  "arrow-schema",
  "arrow-select",
  "getrandom 0.3.4",
@@ -642,11 +643,12 @@ dependencies = [
  "serde",
  "serde_json",
  "siphasher",
+ "tempfile",
 ]
 
 [[package]]
 name = "onix-cli"
-version = "0.11.0"
+version = "0.11.2"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -654,7 +656,7 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.11.0"
+version = "0.11.2"
 dependencies = [
  "num-bigint",
  "num-traits",
@@ -667,10 +669,9 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.11.0"
+version = "0.11.2"
 dependencies = [
  "arrow-array",
- "arrow-ipc",
  "arrow-schema",
  "num-bigint",
  "onix-arrow",
@@ -678,7 +679,6 @@ dependencies = [
  "pyo3",
  "pyo3-arrow",
  "serde_json",
- "tempfile",
 ]
 
 [[package]]

{deepdiff_rs-0.11.0 → deepdiff_rs-0.11.2}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.11.0"
+version = "0.11.2"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.11.0 → deepdiff_rs-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.11.0
+Version: 0.11.2
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -124,15 +124,10 @@ It reports the **schema** diff (which columns were added, removed, or changed ty
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
-left = pa.table({
-    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
-    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
-})
-right = pa.table({
-    "id": pa.array([2, 3, 4], pa.int64()),
-    "amount": pa.array([20, 31, 40], pa.int64()),
-    "note": pa.array(["a", "b", "c"], pa.string()),
-})
+# 9 is a duplicate key
+left = pa.table({"id": pa.array([1, 2, 3, 9, 9], pa.int64()), "amount": pa.array([10, 20, 30, 90, 91], pa.int32())})
+right = pa.table({"id": pa.array([2, 3, 4], pa.int64()), "amount": pa.array([20, 31, 40], pa.int64()),
+                   "note": pa.array(["a", "b", "c"], pa.string())})
 
 diff = diff_tables(left, right, key=["id"])
 print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
@@ -144,11 +139,13 @@ print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate k
 cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
-A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+A key appearing more than once on either side is reported in `duplicate_keys` (`left_count`/`right_count`) and excluded from added/removed/changed; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is skipped rather than compared. The exact rules are on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings sharing a logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes; nullability is ignored but reported in each record. The full rules are on `normalized_type`/`map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs). Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` needs no pyarrow, and `.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but needs pyarrow installed regardless.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra; `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
+`pyarrow` is optional: `pip install deepdiff-rs[arrow]`. It's needed only for `to_pyarrow()` and passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. An object implementing neither Arrow protocol raises `TypeError`; `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed; see [Known limitations](#known-limitations) for its row cap.
+
+Beyond a join-based diff, `diff_tables` reports duplicate and null keys rather than silently multiplying or dropping them, distinguishes `type_changed` from `value_changed` per cell, renders every value by one documented rule set (Python's, with `Duration` the one exception -- an ISO 8601 string, not Python's own `str()`), and produces byte-identical output at any thread count with memory proportional to row count from a streamed input. What that costs over a hand-rolled join is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)'s polars-backed ceiling section.
 
 ## Performance
 
@@ -195,7 +192,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared, still trails by a much larger multiple, see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, now spills each side's changed rows by key-hash partition and renders them across every core, cutting its 5 GB-per-side wall time 6.14x (149.674 -> 24.375 s; about 3.8x DuckDB and 6.2x polars) and its peak RSS 2.02x (about 67 GB -> 33.1 GB), see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -250,7 +247,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M); the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 67 GB at 16.875M rows/side, see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which spills each side's changed value rows — every common value column of every changed row, changed or not — to anonymous per-key-hash-partition Arrow IPC files and compares and renders one partition at a time across the workers. Its resident cost is the spilled changed value rows (the changed-row count times the total width of the common value columns, both sides — resident where written temp pages count, e.g. macOS or a RAM-backed `tmpfs`) plus about twice the `cells_changed` output (its one out-of-place reorder) plus the per-row hash vectors; the spill term dominates for wide rows with few changed cells, the output term for many changed cells. Measured (`row_diff_rss`, 18 threads): a 1 KB `string` cell changed on every row (output-dominated) peaks at 0.92 GB at 100k rows/side, 1.31 GB at 200k, and 4.81 GB at 1M (the single-threaded path, holding both sides, is 8.30 GB); eight 512 B value columns with only one differing (spill-dominated: 150k changed cells, ~160 MB output) peaks at about 2.09 GB at 150k rows/side, versus 0.84 GB with one such column (medians over 5 runs); these are at 18 threads, the low point of the range -- the same two shapes peak higher at the ends (spill-dominated 3.03 GB at 2 threads and 2.09 GB at 64; output-dominated, 200k, 1.98 GB at 2 and 1.49 GB at 64) as fewer partitions enlarge the resident chunk and the 64-way framing adds a little; two narrow `int64` columns with every row changed peak at 0.38 GB at 1M; the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 33 GB whole-process at 16.875M rows/side at 18 threads, down from about 67 GB before the streaming cell pass. The two Arrow types whose `take` retains data beyond the selected rows are decoded before the spill -- byte-view columns (`Utf8View`/`BinaryView`) cast to their large i64-offset non-view type (`LargeUtf8`/`LargeBinary`, since one batch's retained view buffers can exceed the i32 offset type's ~2 GB ceiling) and dictionaries (what polars and DuckDB emit for strings) decoded to their value type -- so the spill stays compact and independent of the partition count; without the decode it would grow with the partition count (about 97 GB at 64 threads for the wide pair vs about 34 GB with it), see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), and the cell pass additionally spills both sides' changed value rows to anonymous per-partition files (byte-view columns cast to their non-view type and dictionaries decoded to their value type first, so the spill is compact and does not grow with the thread count); the input spools and the partition spill are resident together, and for the full wide pair the whole process (input spool + spill + working set) peaks at about 33 GB at 18 threads, 45 GB at 2, and 34 GB at 64 (on Linux the spill may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data; for the cell pass, the total width of the common value columns — every one is spilled in full for each changed row, changed or not — plus the rendered changed cells; and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), the producer's batch size, and the thread count (the cell pass's peak is lowest at the default and higher at 2 threads, where 2 partitions hold half the rows, and slightly higher at 64, the partition cap), since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.11.0 → deepdiff_rs-0.11.2}/README.md

@@ -106,15 +106,10 @@ It reports the **schema** diff (which columns were added, removed, or changed ty
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
-left = pa.table({
-    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
-    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
-})
-right = pa.table({
-    "id": pa.array([2, 3, 4], pa.int64()),
-    "amount": pa.array([20, 31, 40], pa.int64()),
-    "note": pa.array(["a", "b", "c"], pa.string()),
-})
+# 9 is a duplicate key
+left = pa.table({"id": pa.array([1, 2, 3, 9, 9], pa.int64()), "amount": pa.array([10, 20, 30, 90, 91], pa.int32())})
+right = pa.table({"id": pa.array([2, 3, 4], pa.int64()), "amount": pa.array([20, 31, 40], pa.int64()),
+                   "note": pa.array(["a", "b", "c"], pa.string())})
 
 diff = diff_tables(left, right, key=["id"])
 print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
@@ -126,11 +121,13 @@ print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate k
 cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
-A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+A key appearing more than once on either side is reported in `duplicate_keys` (`left_count`/`right_count`) and excluded from added/removed/changed; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is skipped rather than compared. The exact rules are on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings sharing a logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes; nullability is ignored but reported in each record. The full rules are on `normalized_type`/`map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs). Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` needs no pyarrow, and `.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but needs pyarrow installed regardless.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra; `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
+`pyarrow` is optional: `pip install deepdiff-rs[arrow]`. It's needed only for `to_pyarrow()` and passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. An object implementing neither Arrow protocol raises `TypeError`; `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed; see [Known limitations](#known-limitations) for its row cap.
+
+Beyond a join-based diff, `diff_tables` reports duplicate and null keys rather than silently multiplying or dropping them, distinguishes `type_changed` from `value_changed` per cell, renders every value by one documented rule set (Python's, with `Duration` the one exception -- an ISO 8601 string, not Python's own `str()`), and produces byte-identical output at any thread count with memory proportional to row count from a streamed input. What that costs over a hand-rolled join is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)'s polars-backed ceiling section.
 
 ## Performance
 
@@ -177,7 +174,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared, still trails by a much larger multiple, see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared and rendered, now spills each side's changed rows by key-hash partition and renders them across every core, cutting its 5 GB-per-side wall time 6.14x (149.674 -> 24.375 s; about 3.8x DuckDB and 6.2x polars) and its peak RSS 2.02x (about 67 GB -> 33.1 GB), see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -232,7 +229,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M); the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 67 GB at 16.875M rows/side, see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which spills each side's changed value rows — every common value column of every changed row, changed or not — to anonymous per-key-hash-partition Arrow IPC files and compares and renders one partition at a time across the workers. Its resident cost is the spilled changed value rows (the changed-row count times the total width of the common value columns, both sides — resident where written temp pages count, e.g. macOS or a RAM-backed `tmpfs`) plus about twice the `cells_changed` output (its one out-of-place reorder) plus the per-row hash vectors; the spill term dominates for wide rows with few changed cells, the output term for many changed cells. Measured (`row_diff_rss`, 18 threads): a 1 KB `string` cell changed on every row (output-dominated) peaks at 0.92 GB at 100k rows/side, 1.31 GB at 200k, and 4.81 GB at 1M (the single-threaded path, holding both sides, is 8.30 GB); eight 512 B value columns with only one differing (spill-dominated: 150k changed cells, ~160 MB output) peaks at about 2.09 GB at 150k rows/side, versus 0.84 GB with one such column (medians over 5 runs); these are at 18 threads, the low point of the range -- the same two shapes peak higher at the ends (spill-dominated 3.03 GB at 2 threads and 2.09 GB at 64; output-dominated, 200k, 1.98 GB at 2 and 1.49 GB at 64) as fewer partitions enlarge the resident chunk and the 64-way framing adds a little; two narrow `int64` columns with every row changed peak at 0.38 GB at 1M; the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 33 GB whole-process at 16.875M rows/side at 18 threads, down from about 67 GB before the streaming cell pass. The two Arrow types whose `take` retains data beyond the selected rows are decoded before the spill -- byte-view columns (`Utf8View`/`BinaryView`) cast to their large i64-offset non-view type (`LargeUtf8`/`LargeBinary`, since one batch's retained view buffers can exceed the i32 offset type's ~2 GB ceiling) and dictionaries (what polars and DuckDB emit for strings) decoded to their value type -- so the spill stays compact and independent of the partition count; without the decode it would grow with the partition count (about 97 GB at 64 threads for the wide pair vs about 34 GB with it), see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), and the cell pass additionally spills both sides' changed value rows to anonymous per-partition files (byte-view columns cast to their non-view type and dictionaries decoded to their value type first, so the spill is compact and does not grow with the thread count); the input spools and the partition spill are resident together, and for the full wide pair the whole process (input spool + spill + working set) peaks at about 33 GB at 18 threads, 45 GB at 2, and 34 GB at 64 (on Linux the spill may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data; for the cell pass, the total width of the common value columns — every one is spilled in full for each changed row, changed or not — plus the rendered changed cells; and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), the producer's batch size, and the thread count (the cell pass's peak is lowest at the default and higher at 2 threads, where 2 partitions hold half the rows, and slightly higher at 64, the partition cap), since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.11.0 → deepdiff_rs-0.11.2}/crates/onix-arrow/Cargo.toml

@@ -9,6 +9,12 @@ description = "Arrow table diffing (schema and keyed rows) built on the onix dif
 [lints]
 workspace = true
 
+[features]
+# Compiles in the per-pass wall-time and peak-RSS instrumentation the
+# `row_diff_profile` example reads. Off by default and never enabled by the
+# release wheel, so the instrumentation is absent from the shipped `.so`.
+profile = []
+
 [dependencies]
 # Pinned to an exact Arrow version so this crate and the `pyo3-arrow` bridge
 # in `onix-py` (which requires `arrow` ^59) resolve to the identical Arrow
@@ -27,10 +33,19 @@ arrow-buffer = "=59.3.0"
 # cell's value to its canonical string (`display::ArrayFormatter`);
 # `arrow-select` filters the materialized rows, concatenates the output row
 # batches, and `take`s the key columns into the per-cell diff. The row diff
-# re-reads each side through the caller's re-openable `TableInput`, so no IPC
-# spooling lives here — the Python bindings own that.
+# re-reads each *input* side through the caller's re-openable `TableInput`. The
+# `spool` module owns the anonymous temporary Arrow IPC primitive used both by
+# the Python bindings' input spool and by the parallel cell pass, which spills
+# its changed-row key-hash partitions so it holds one partition in memory at a
+# time rather than both sides' full changed rows.
 arrow-cast = "=59.3.0"
 arrow-select = "=59.3.0"
+# The `spool` module writes to an anonymous `tempfile::tempfile()` as an Arrow
+# IPC stream and re-reads it; the cell pass spills one key-hash partition per
+# file so only one is resident at once. Pinned to the same 59.3.0 as the other
+# arrow crates.
+arrow-ipc = "=59.3.0"
+tempfile = "3"
 # Row identity is a keyed 128-bit SipHash-1-3 (`siphasher`), keyed from 16
 # bytes of OS randomness per diff (`getrandom`), so the row-matching table on
 # this default path cannot be forced into collisions by chosen input.
@@ -43,3 +58,13 @@ serde_json = "1"
 proptest = "1"
 # `half::f16` builds a Float16 test column; the same version arrow-array uses.
 half = "2"
+
+# The profiling harness reads the per-pass instrumentation, so it only builds
+# with the `profile` feature (and never as part of the default build or wheel).
+[[example]]
+name = "row_diff_profile"
+required-features = ["profile"]
+
+[[test]]
+name = "profile_passes"
+required-features = ["profile"]

deepdiff_rs-0.11.2/crates/onix-arrow/examples/row_diff_profile.rs

@@ -0,0 +1,316 @@
+//! Per-pass wall-time and peak-RSS profile of one keyed row diff, the committed
+//! harness every row-diff performance change posts a before/after table from.
+//! Builds only with the `profile` feature, which the release wheel never enables.
+//!
+//! Each invocation runs a discarded warm-up diff, a timed uninstrumented diff
+//! (the `uninstrumented wall` line), and an instrumented diff whose passes make
+//! up the table. The closure line compares the passes' sum with the instrumented
+//! wall minus the profiler's own boundary `ps` reads; `share` is each row's wall
+//! over that net wall. Peak RSS is the process's, third diff in the process.
+//!
+//! - **file**: reads each side from an uncompressed Arrow IPC file, re-opened
+//!   and re-decoded by every pass. Convert a parquet fixture once with
+//!   `python -c "import pyarrow.parquet as p, pyarrow.feather as f;
+//!   f.write_feather(p.read_table('a.parquet'), 'a.arrow', compression='uncompressed')"`.
+//! - **generated**: spools both sides of a deterministic proxy shape to anonymous
+//!   Arrow IPC files (the `spool write` line times the IPC writer alone).
+//!
+//! ```sh
+//! cargo build -p onix-arrow --release --features profile --example row_diff_profile
+//! target/release/examples/row_diff_profile file a.arrow b.arrow --key id --threads 18
+//! target/release/examples/row_diff_profile 1000000 linear 18
+//! target/release/examples/row_diff_profile 1000000 manycols 18 34 64
+//! ```
+//!
+//! Generated args: `[rows [shape [threads [shape params...]]]]`, key `id`,
+//! `threads` defaulting to `ROW_DIFF_THREADS` or available parallelism. Shapes:
+//!
+//! - `linear`: `id`/`value` int64; 1% of keys added, 1% removed, ~2% changed.
+//! - `allchange`: `id`/`value` int64, every row changed.
+//! - `wide [width=512]`: `id` plus one `width`-byte string, every row changed.
+//! - `manycols [ncols=34 [width=64]]`: `id` plus `ncols` `width`-byte strings,
+//!   only the first differing, so the spill carries every value column.
+
+use std::fs::File;
+use std::io::BufReader;
+use std::num::NonZeroUsize;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use arrow_array::RecordBatchReader;
+use arrow_ipc::reader::FileReader;
+use arrow_schema::SchemaRef;
+use onix_arrow::{TableDiffError, TableDiffOptions, TableInput, diff_tables, profile, spool};
+
+#[path = "shared/gen_shapes.rs"]
+mod gen_shapes;
+use gen_shapes::{Case, Generated, batch_rows};
+
+/// A table read from an Arrow IPC file, re-opened on every `open`.
+struct FileInput {
+    path: PathBuf,
+    schema: SchemaRef,
+}
+
+impl FileInput {
+    fn load(path: &str) -> FileInput {
+        let reader = open_ipc(path).unwrap_or_else(|e| panic!("open {path}: {e}"));
+        FileInput {
+            path: PathBuf::from(path),
+            schema: reader.schema(),
+        }
+    }
+}
+
+fn open_ipc(path: &str) -> Result<FileReader<BufReader<File>>, TableDiffError> {
+    let file = File::open(path).map_err(|e| TableDiffError::Read {
+        message: format!("open {path}: {e}"),
+    })?;
+    FileReader::try_new(BufReader::new(file), None).map_err(|e| TableDiffError::Read {
+        message: format!("read Arrow IPC {path}: {e}"),
+    })
+}
+
+impl TableInput for FileInput {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+    fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
+        let path = self.path.to_string_lossy();
+        Ok(Box::new(open_ipc(&path)?))
+    }
+}
+
+/// A side spooled to an anonymous Arrow IPC file, re-read on every `open` — the
+/// same re-openable spool the Python bindings hand the row diff.
+struct Spooled {
+    file: File,
+    schema: SchemaRef,
+}
+
+impl TableInput for Spooled {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+    fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
+        Ok(Box::new(spool::reopen(&self.file)?))
+    }
+}
+
+/// Spools one generated side, returning it and the time spent in the IPC
+/// writer (generation excluded).
+fn spool_side(side: &Generated) -> (Spooled, Duration) {
+    let (file, mut writer) = spool::open(&side.schema).expect("open spool");
+    let mut writing = Duration::ZERO;
+    for batch in side.open().expect("open generator") {
+        let batch = batch.expect("generate batch");
+        let start = Instant::now();
+        writer.write(&batch).expect("spool write");
+        writing += start.elapsed();
+    }
+    let start = Instant::now();
+    writer.finish().expect("spool finish");
+    writing += start.elapsed();
+    let spooled = Spooled {
+        file,
+        schema: side.schema.clone(),
+    };
+    (spooled, writing)
+}
+
+/// Runs one discarded warm-up diff, one timed uninstrumented diff, and one
+/// instrumented diff, then prints both walls, the closure of the instrumented
+/// wall over its passes, and the per-pass table.
+fn run(left: &impl TableInput, right: &impl TableInput, options: &TableDiffOptions, label: &str) {
+    let _ = diff_tables(left, right, options).expect("diff succeeds");
+    let start = Instant::now();
+    let diff = diff_tables(left, right, options).expect("diff succeeds");
+    let uninstrumented = start.elapsed().as_secs_f64();
+    let summary = diff.summary();
+
+    let session = profile::begin();
+    let start = Instant::now();
+    let _ = diff_tables(left, right, options).expect("diff succeeds");
+    let instrumented = start.elapsed().as_secs_f64();
+    let report = session.finish();
+
+    let boundary: f64 = report
+        .iter()
+        .filter(|row| row.label == profile::BOUNDARY_LABEL)
+        .map(|row| row.wall_secs)
+        .sum();
+    let passes: f64 = report
+        .iter()
+        .filter(|row| row.peak_rss_mib.is_some())
+        .map(|row| row.wall_secs)
+        .sum();
+    let closed = instrumented - boundary;
+
+    println!("{label}");
+    println!("threads: {}", options.threads());
+    println!("uninstrumented wall: {uninstrumented:.3} s");
+    println!("instrumented wall: {instrumented:.3} s");
+    println!("boundary ps reads: {boundary:.3} s");
+    println!(
+        "passes sum: {passes:.3} s of {closed:.3} s (instrumented minus boundary); residual {:.3} s ({:.1}%)",
+        closed - passes,
+        100.0 * (closed - passes) / closed
+    );
+    println!(
+        "rows_added={} rows_removed={} rows_changed={} duplicate_keys={} cells_changed={}",
+        summary.rows_added,
+        summary.rows_removed,
+        summary.rows_changed,
+        summary.duplicate_keys,
+        summary.cells_changed
+    );
+    println!();
+    println!(
+        "{:<44} {:>10} {:>8} {:>14}",
+        "pass", "wall (s)", "share", "peak RSS (MB)"
+    );
+    for row in report
+        .iter()
+        .filter(|row| row.label != profile::BOUNDARY_LABEL)
+    {
+        let share = 100.0 * row.wall_secs / closed;
+        match row.peak_rss_mib {
+            Some(rss) => println!(
+                "{:<44} {:>10.3} {:>7.1}% {:>14.1}",
+                row.label, row.wall_secs, share, rss
+            ),
+            None => println!(
+                "{:<44} {:>10.3} {:>7.1}% {:>14}",
+                format!("  {}", row.label),
+                row.wall_secs,
+                share,
+                "-"
+            ),
+        }
+    }
+}
+
+const USAGE: &str =
+    "usage: row_diff_profile file <left.arrow> <right.arrow> [--key col[,col...]] [--threads N]
+       row_diff_profile [rows [linear|allchange|wide|manycols [threads [shape params...]]]]";
+
+fn usage_error(message: &str) -> ! {
+    eprintln!("{message}\n{USAGE}");
+    std::process::exit(2);
+}
+
+/// Parses a positive integer argument, exiting with a usage error otherwise.
+fn positive(what: &str, value: &str) -> NonZeroUsize {
+    value.parse().unwrap_or_else(|_| {
+        usage_error(&format!("{what} must be a positive integer, got {value:?}"))
+    })
+}
+
+fn options_for(keys: Vec<String>, threads: Option<NonZeroUsize>) -> TableDiffOptions {
+    let threads = threads.or_else(|| {
+        std::env::var("ROW_DIFF_THREADS")
+            .ok()
+            .map(|v| positive("ROW_DIFF_THREADS", &v))
+    });
+    let options = TableDiffOptions::new(keys);
+    match threads {
+        Some(t) => options
+            .with_threads(t)
+            .unwrap_or_else(|e| usage_error(&e.to_string())),
+        None => options,
+    }
+}
+
+fn file_mode(args: &[String]) {
+    let [left_path, right_path, flags @ ..] = args else {
+        usage_error("file mode needs <left.arrow> <right.arrow>");
+    };
+    let mut keys = Vec::new();
+    let mut threads = None;
+    let mut flags = flags.iter();
+    while let Some(flag) = flags.next() {
+        let Some(value) = flags.next() else {
+            usage_error(&format!("{flag} needs a value"));
+        };
+        match flag.as_str() {
+            "--key" => keys.extend(value.split(',').map(str::to_string)),
+            "--threads" => threads = Some(positive("--threads", value)),
+            _ => usage_error(&format!("unknown flag {flag:?}")),
+        }
+    }
+    if keys.is_empty() {
+        keys.push("id".to_string());
+    }
+    let left = FileInput::load(left_path);
+    let right = FileInput::load(right_path);
+    run(
+        &left,
+        &right,
+        &options_for(keys, threads),
+        &format!("file: {left_path} vs {right_path}"),
+    );
+}
+
+fn generated_mode(args: &[String]) {
+    let rows = args.first().map_or(1_000_000, |a| {
+        i64::try_from(positive("rows", a).get()).unwrap_or_else(|_| usage_error("rows too large"))
+    });
+    let shape = args.get(1).map_or("linear", String::as_str);
+    let threads = args.get(2).map(|a| positive("threads", a));
+    let params: Vec<usize> = args
+        .iter()
+        .skip(3)
+        .map(|a| positive("shape parameter", a).get())
+        .collect();
+    let param = |i: usize, default: usize| params.get(i).copied().unwrap_or(default);
+    let (case, arity) = match shape {
+        "linear" => (Case::Linear, 0),
+        "allchange" => (Case::AllChange, 0),
+        "wide" => (Case::Wide(param(0, 512)), 1),
+        "manycols" => (
+            Case::ManyCols {
+                ncols: param(0, 34),
+                width: param(1, 64),
+            },
+            2,
+        ),
+        _ => usage_error(&format!("unknown shape {shape:?}")),
+    };
+    if params.len() > arity {
+        usage_error(&format!("shape {shape} takes at most {arity} parameter(s)"));
+    }
+
+    let (schema, left_shape, right_shape, key) = case.build(rows);
+    let options = options_for(vec![key.to_string()], threads);
+    let batch = batch_rows();
+    let (left, left_write) = spool_side(&Generated {
+        schema: schema.clone(),
+        rows,
+        shape: left_shape,
+        batch,
+    });
+    let (right, right_write) = spool_side(&Generated {
+        schema,
+        rows,
+        shape: right_shape,
+        batch,
+    });
+    println!(
+        "spool write (both sides, before the diff): {:.3} s",
+        (left_write + right_write).as_secs_f64()
+    );
+    run(
+        &left,
+        &right,
+        &options,
+        &format!("rows per side: {rows} (shape={shape})"),
+    );
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    match args.split_first() {
+        Some((mode, rest)) if mode == "file" => file_mode(rest),
+        _ => generated_mode(&args),
+    }
+}