deepdiff-rs 0.11.1__tar.gz → 0.11.2__tar.gz

{deepdiff_rs-0.11.1 → deepdiff_rs-0.11.2}/Cargo.lock

@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.11.1"
+version = "0.11.2"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -648,7 +648,7 @@ dependencies = [
 
 [[package]]
 name = "onix-cli"
-version = "0.11.1"
+version = "0.11.2"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -656,7 +656,7 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.11.1"
+version = "0.11.2"
 dependencies = [
  "num-bigint",
  "num-traits",
@@ -669,7 +669,7 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.11.1"
+version = "0.11.2"
 dependencies = [
  "arrow-array",
  "arrow-schema",

{deepdiff_rs-0.11.1 → deepdiff_rs-0.11.2}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.11.1"
+version = "0.11.2"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.11.1 → deepdiff_rs-0.11.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.11.1
+Version: 0.11.2
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -124,15 +124,10 @@ It reports the **schema** diff (which columns were added, removed, or changed ty
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
-left = pa.table({
-    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
-    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
-})
-right = pa.table({
-    "id": pa.array([2, 3, 4], pa.int64()),
-    "amount": pa.array([20, 31, 40], pa.int64()),
-    "note": pa.array(["a", "b", "c"], pa.string()),
-})
+# 9 is a duplicate key
+left = pa.table({"id": pa.array([1, 2, 3, 9, 9], pa.int64()), "amount": pa.array([10, 20, 30, 90, 91], pa.int32())})
+right = pa.table({"id": pa.array([2, 3, 4], pa.int64()), "amount": pa.array([20, 31, 40], pa.int64()),
+                   "note": pa.array(["a", "b", "c"], pa.string())})
 
 diff = diff_tables(left, right, key=["id"])
 print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
@@ -144,11 +139,13 @@ print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate k
 cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
-A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+A key appearing more than once on either side is reported in `duplicate_keys` (`left_count`/`right_count`) and excluded from added/removed/changed; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is skipped rather than compared. The exact rules are on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings sharing a logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes; nullability is ignored but reported in each record. The full rules are on `normalized_type`/`map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs). Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` needs no pyarrow, and `.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but needs pyarrow installed regardless.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra; `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
+`pyarrow` is optional: `pip install deepdiff-rs[arrow]`. It's needed only for `to_pyarrow()` and passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. An object implementing neither Arrow protocol raises `TypeError`; `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed; see [Known limitations](#known-limitations) for its row cap.
+
+Beyond a join-based diff, `diff_tables` reports duplicate and null keys rather than silently multiplying or dropping them, distinguishes `type_changed` from `value_changed` per cell, renders every value by one documented rule set (Python's, with `Duration` the one exception -- an ISO 8601 string, not Python's own `str()`), and produces byte-identical output at any thread count with memory proportional to row count from a streamed input. What that costs over a hand-rolled join is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)'s polars-backed ceiling section.
 
 ## Performance
 

{deepdiff_rs-0.11.1 → deepdiff_rs-0.11.2}/README.md

@@ -106,15 +106,10 @@ It reports the **schema** diff (which columns were added, removed, or changed ty
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
-left = pa.table({
-    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
-    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
-})
-right = pa.table({
-    "id": pa.array([2, 3, 4], pa.int64()),
-    "amount": pa.array([20, 31, 40], pa.int64()),
-    "note": pa.array(["a", "b", "c"], pa.string()),
-})
+# 9 is a duplicate key
+left = pa.table({"id": pa.array([1, 2, 3, 9, 9], pa.int64()), "amount": pa.array([10, 20, 30, 90, 91], pa.int32())})
+right = pa.table({"id": pa.array([2, 3, 4], pa.int64()), "amount": pa.array([20, 31, 40], pa.int64()),
+                   "note": pa.array(["a", "b", "c"], pa.string())})
 
 diff = diff_tables(left, right, key=["id"])
 print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
@@ -126,11 +121,13 @@ print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate k
 cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
-A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+A key appearing more than once on either side is reported in `duplicate_keys` (`left_count`/`right_count`) and excluded from added/removed/changed; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is skipped rather than compared. The exact rules are on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings sharing a logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes; nullability is ignored but reported in each record. The full rules are on `normalized_type`/`map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs). Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` needs no pyarrow, and `.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but needs pyarrow installed regardless.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra; `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
+`pyarrow` is optional: `pip install deepdiff-rs[arrow]`. It's needed only for `to_pyarrow()` and passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. An object implementing neither Arrow protocol raises `TypeError`; `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed; see [Known limitations](#known-limitations) for its row cap.
+
+Beyond a join-based diff, `diff_tables` reports duplicate and null keys rather than silently multiplying or dropping them, distinguishes `type_changed` from `value_changed` per cell, renders every value by one documented rule set (Python's, with `Duration` the one exception -- an ISO 8601 string, not Python's own `str()`), and produces byte-identical output at any thread count with memory proportional to row count from a streamed input. What that costs over a hand-rolled join is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)'s polars-backed ceiling section.
 
 ## Performance
 

{deepdiff_rs-0.11.1 → deepdiff_rs-0.11.2}/crates/onix-arrow/Cargo.toml

@@ -9,6 +9,12 @@ description = "Arrow table diffing (schema and keyed rows) built on the onix dif
 [lints]
 workspace = true
 
+[features]
+# Compiles in the per-pass wall-time and peak-RSS instrumentation the
+# `row_diff_profile` example reads. Off by default and never enabled by the
+# release wheel, so the instrumentation is absent from the shipped `.so`.
+profile = []
+
 [dependencies]
 # Pinned to an exact Arrow version so this crate and the `pyo3-arrow` bridge
 # in `onix-py` (which requires `arrow` ^59) resolve to the identical Arrow
@@ -52,3 +58,13 @@ serde_json = "1"
 proptest = "1"
 # `half::f16` builds a Float16 test column; the same version arrow-array uses.
 half = "2"
+
+# The profiling harness reads the per-pass instrumentation, so it only builds
+# with the `profile` feature (and never as part of the default build or wheel).
+[[example]]
+name = "row_diff_profile"
+required-features = ["profile"]
+
+[[test]]
+name = "profile_passes"
+required-features = ["profile"]

deepdiff_rs-0.11.2/crates/onix-arrow/examples/row_diff_profile.rs

@@ -0,0 +1,316 @@
+//! Per-pass wall-time and peak-RSS profile of one keyed row diff, the committed
+//! harness every row-diff performance change posts a before/after table from.
+//! Builds only with the `profile` feature, which the release wheel never enables.
+//!
+//! Each invocation runs a discarded warm-up diff, a timed uninstrumented diff
+//! (the `uninstrumented wall` line), and an instrumented diff whose passes make
+//! up the table. The closure line compares the passes' sum with the instrumented
+//! wall minus the profiler's own boundary `ps` reads; `share` is each row's wall
+//! over that net wall. Peak RSS is the process's, third diff in the process.
+//!
+//! - **file**: reads each side from an uncompressed Arrow IPC file, re-opened
+//!   and re-decoded by every pass. Convert a parquet fixture once with
+//!   `python -c "import pyarrow.parquet as p, pyarrow.feather as f;
+//!   f.write_feather(p.read_table('a.parquet'), 'a.arrow', compression='uncompressed')"`.
+//! - **generated**: spools both sides of a deterministic proxy shape to anonymous
+//!   Arrow IPC files (the `spool write` line times the IPC writer alone).
+//!
+//! ```sh
+//! cargo build -p onix-arrow --release --features profile --example row_diff_profile
+//! target/release/examples/row_diff_profile file a.arrow b.arrow --key id --threads 18
+//! target/release/examples/row_diff_profile 1000000 linear 18
+//! target/release/examples/row_diff_profile 1000000 manycols 18 34 64
+//! ```
+//!
+//! Generated args: `[rows [shape [threads [shape params...]]]]`, key `id`,
+//! `threads` defaulting to `ROW_DIFF_THREADS` or available parallelism. Shapes:
+//!
+//! - `linear`: `id`/`value` int64; 1% of keys added, 1% removed, ~2% changed.
+//! - `allchange`: `id`/`value` int64, every row changed.
+//! - `wide [width=512]`: `id` plus one `width`-byte string, every row changed.
+//! - `manycols [ncols=34 [width=64]]`: `id` plus `ncols` `width`-byte strings,
+//!   only the first differing, so the spill carries every value column.
+
+use std::fs::File;
+use std::io::BufReader;
+use std::num::NonZeroUsize;
+use std::path::PathBuf;
+use std::time::{Duration, Instant};
+
+use arrow_array::RecordBatchReader;
+use arrow_ipc::reader::FileReader;
+use arrow_schema::SchemaRef;
+use onix_arrow::{TableDiffError, TableDiffOptions, TableInput, diff_tables, profile, spool};
+
+#[path = "shared/gen_shapes.rs"]
+mod gen_shapes;
+use gen_shapes::{Case, Generated, batch_rows};
+
+/// A table read from an Arrow IPC file, re-opened on every `open`.
+struct FileInput {
+    path: PathBuf,
+    schema: SchemaRef,
+}
+
+impl FileInput {
+    fn load(path: &str) -> FileInput {
+        let reader = open_ipc(path).unwrap_or_else(|e| panic!("open {path}: {e}"));
+        FileInput {
+            path: PathBuf::from(path),
+            schema: reader.schema(),
+        }
+    }
+}
+
+fn open_ipc(path: &str) -> Result<FileReader<BufReader<File>>, TableDiffError> {
+    let file = File::open(path).map_err(|e| TableDiffError::Read {
+        message: format!("open {path}: {e}"),
+    })?;
+    FileReader::try_new(BufReader::new(file), None).map_err(|e| TableDiffError::Read {
+        message: format!("read Arrow IPC {path}: {e}"),
+    })
+}
+
+impl TableInput for FileInput {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+    fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
+        let path = self.path.to_string_lossy();
+        Ok(Box::new(open_ipc(&path)?))
+    }
+}
+
+/// A side spooled to an anonymous Arrow IPC file, re-read on every `open` — the
+/// same re-openable spool the Python bindings hand the row diff.
+struct Spooled {
+    file: File,
+    schema: SchemaRef,
+}
+
+impl TableInput for Spooled {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+    fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
+        Ok(Box::new(spool::reopen(&self.file)?))
+    }
+}
+
+/// Spools one generated side, returning it and the time spent in the IPC
+/// writer (generation excluded).
+fn spool_side(side: &Generated) -> (Spooled, Duration) {
+    let (file, mut writer) = spool::open(&side.schema).expect("open spool");
+    let mut writing = Duration::ZERO;
+    for batch in side.open().expect("open generator") {
+        let batch = batch.expect("generate batch");
+        let start = Instant::now();
+        writer.write(&batch).expect("spool write");
+        writing += start.elapsed();
+    }
+    let start = Instant::now();
+    writer.finish().expect("spool finish");
+    writing += start.elapsed();
+    let spooled = Spooled {
+        file,
+        schema: side.schema.clone(),
+    };
+    (spooled, writing)
+}
+
+/// Runs one discarded warm-up diff, one timed uninstrumented diff, and one
+/// instrumented diff, then prints both walls, the closure of the instrumented
+/// wall over its passes, and the per-pass table.
+fn run(left: &impl TableInput, right: &impl TableInput, options: &TableDiffOptions, label: &str) {
+    let _ = diff_tables(left, right, options).expect("diff succeeds");
+    let start = Instant::now();
+    let diff = diff_tables(left, right, options).expect("diff succeeds");
+    let uninstrumented = start.elapsed().as_secs_f64();
+    let summary = diff.summary();
+
+    let session = profile::begin();
+    let start = Instant::now();
+    let _ = diff_tables(left, right, options).expect("diff succeeds");
+    let instrumented = start.elapsed().as_secs_f64();
+    let report = session.finish();
+
+    let boundary: f64 = report
+        .iter()
+        .filter(|row| row.label == profile::BOUNDARY_LABEL)
+        .map(|row| row.wall_secs)
+        .sum();
+    let passes: f64 = report
+        .iter()
+        .filter(|row| row.peak_rss_mib.is_some())
+        .map(|row| row.wall_secs)
+        .sum();
+    let closed = instrumented - boundary;
+
+    println!("{label}");
+    println!("threads: {}", options.threads());
+    println!("uninstrumented wall: {uninstrumented:.3} s");
+    println!("instrumented wall: {instrumented:.3} s");
+    println!("boundary ps reads: {boundary:.3} s");
+    println!(
+        "passes sum: {passes:.3} s of {closed:.3} s (instrumented minus boundary); residual {:.3} s ({:.1}%)",
+        closed - passes,
+        100.0 * (closed - passes) / closed
+    );
+    println!(
+        "rows_added={} rows_removed={} rows_changed={} duplicate_keys={} cells_changed={}",
+        summary.rows_added,
+        summary.rows_removed,
+        summary.rows_changed,
+        summary.duplicate_keys,
+        summary.cells_changed
+    );
+    println!();
+    println!(
+        "{:<44} {:>10} {:>8} {:>14}",
+        "pass", "wall (s)", "share", "peak RSS (MB)"
+    );
+    for row in report
+        .iter()
+        .filter(|row| row.label != profile::BOUNDARY_LABEL)
+    {
+        let share = 100.0 * row.wall_secs / closed;
+        match row.peak_rss_mib {
+            Some(rss) => println!(
+                "{:<44} {:>10.3} {:>7.1}% {:>14.1}",
+                row.label, row.wall_secs, share, rss
+            ),
+            None => println!(
+                "{:<44} {:>10.3} {:>7.1}% {:>14}",
+                format!("  {}", row.label),
+                row.wall_secs,
+                share,
+                "-"
+            ),
+        }
+    }
+}
+
+const USAGE: &str =
+    "usage: row_diff_profile file <left.arrow> <right.arrow> [--key col[,col...]] [--threads N]
+       row_diff_profile [rows [linear|allchange|wide|manycols [threads [shape params...]]]]";
+
+fn usage_error(message: &str) -> ! {
+    eprintln!("{message}\n{USAGE}");
+    std::process::exit(2);
+}
+
+/// Parses a positive integer argument, exiting with a usage error otherwise.
+fn positive(what: &str, value: &str) -> NonZeroUsize {
+    value.parse().unwrap_or_else(|_| {
+        usage_error(&format!("{what} must be a positive integer, got {value:?}"))
+    })
+}
+
+fn options_for(keys: Vec<String>, threads: Option<NonZeroUsize>) -> TableDiffOptions {
+    let threads = threads.or_else(|| {
+        std::env::var("ROW_DIFF_THREADS")
+            .ok()
+            .map(|v| positive("ROW_DIFF_THREADS", &v))
+    });
+    let options = TableDiffOptions::new(keys);
+    match threads {
+        Some(t) => options
+            .with_threads(t)
+            .unwrap_or_else(|e| usage_error(&e.to_string())),
+        None => options,
+    }
+}
+
+fn file_mode(args: &[String]) {
+    let [left_path, right_path, flags @ ..] = args else {
+        usage_error("file mode needs <left.arrow> <right.arrow>");
+    };
+    let mut keys = Vec::new();
+    let mut threads = None;
+    let mut flags = flags.iter();
+    while let Some(flag) = flags.next() {
+        let Some(value) = flags.next() else {
+            usage_error(&format!("{flag} needs a value"));
+        };
+        match flag.as_str() {
+            "--key" => keys.extend(value.split(',').map(str::to_string)),
+            "--threads" => threads = Some(positive("--threads", value)),
+            _ => usage_error(&format!("unknown flag {flag:?}")),
+        }
+    }
+    if keys.is_empty() {
+        keys.push("id".to_string());
+    }
+    let left = FileInput::load(left_path);
+    let right = FileInput::load(right_path);
+    run(
+        &left,
+        &right,
+        &options_for(keys, threads),
+        &format!("file: {left_path} vs {right_path}"),
+    );
+}
+
+fn generated_mode(args: &[String]) {
+    let rows = args.first().map_or(1_000_000, |a| {
+        i64::try_from(positive("rows", a).get()).unwrap_or_else(|_| usage_error("rows too large"))
+    });
+    let shape = args.get(1).map_or("linear", String::as_str);
+    let threads = args.get(2).map(|a| positive("threads", a));
+    let params: Vec<usize> = args
+        .iter()
+        .skip(3)
+        .map(|a| positive("shape parameter", a).get())
+        .collect();
+    let param = |i: usize, default: usize| params.get(i).copied().unwrap_or(default);
+    let (case, arity) = match shape {
+        "linear" => (Case::Linear, 0),
+        "allchange" => (Case::AllChange, 0),
+        "wide" => (Case::Wide(param(0, 512)), 1),
+        "manycols" => (
+            Case::ManyCols {
+                ncols: param(0, 34),
+                width: param(1, 64),
+            },
+            2,
+        ),
+        _ => usage_error(&format!("unknown shape {shape:?}")),
+    };
+    if params.len() > arity {
+        usage_error(&format!("shape {shape} takes at most {arity} parameter(s)"));
+    }
+
+    let (schema, left_shape, right_shape, key) = case.build(rows);
+    let options = options_for(vec![key.to_string()], threads);
+    let batch = batch_rows();
+    let (left, left_write) = spool_side(&Generated {
+        schema: schema.clone(),
+        rows,
+        shape: left_shape,
+        batch,
+    });
+    let (right, right_write) = spool_side(&Generated {
+        schema,
+        rows,
+        shape: right_shape,
+        batch,
+    });
+    println!(
+        "spool write (both sides, before the diff): {:.3} s",
+        (left_write + right_write).as_secs_f64()
+    );
+    run(
+        &left,
+        &right,
+        &options,
+        &format!("rows per side: {rows} (shape={shape})"),
+    );
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    match args.split_first() {
+        Some((mode, rest)) if mode == "file" => file_mode(rest),
+        _ => generated_mode(&args),
+    }
+}

deepdiff_rs-0.11.2/crates/onix-arrow/examples/row_diff_rss.rs

@@ -0,0 +1,141 @@
+//! Measures the keyed row diff's peak memory and wall time, to check the memory
+//! bounds the README states.
+//!
+//! Run under the OS's max-RSS reporter:
+//!
+//! ```sh
+//! cargo build -p onix-arrow --release --example row_diff_rss
+//! # linear shape (default): mostly-matching rows, 1% added/removed, ~2% changed
+//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000
+//! /usr/bin/time -l target/release/examples/row_diff_rss 10000000
+//! # same shape with no changed rows: the cell pass materializes nothing, the
+//! # pass-one baseline the ~2%-changed run is measured against
+//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 nochange
+//! # every row changed (narrow int cells): the cell pass at full width
+//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 allchange
+//! # every row changed with a wide (1 KB) string cell: the rendering worst case
+//! /usr/bin/time -l target/release/examples/row_diff_rss 100000 wide 1024
+//! /usr/bin/time -l target/release/examples/row_diff_rss 200000 wide 1024
+//! # wide rows, few changed cells: id + 8 512-byte columns, only one differing
+//! ROW_DIFF_THREADS=18 /usr/bin/time -l target/release/examples/row_diff_rss 150000 manycols 8 512
+//! # duplicate-heavy shape: every key duplicated, wide string key
+//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 dup 16
+//! /usr/bin/time -l target/release/examples/row_diff_rss 200000 dup 1024
+//! # size-gate peek: identical wide-cell sides (zero changes); ROW_DIFF_BATCH
+//! # sets the producer's batch size, ROW_DIFF_THREADS the worker count
+//! ROW_DIFF_BATCH=100 ROW_DIFF_THREADS=18 /usr/bin/time -l target/release/examples/row_diff_rss 49999 widesame 8192
+//! ```
+//!
+//! Each side is generated on the fly, batch by batch, and nothing is retained
+//! between batches, so the process's peak RSS is the diff's own state, not the
+//! table data. The **linear** shape (`id`, `value` int64 columns) exercises the
+//! per-row hash vectors: the left is ids `0..n`, the right `step..n + step` with
+//! `step = n / 100`, so 1% removed, 1% added, ~2% changed — and the cell pass
+//! materializes those ~2% changed rows on both sides. The **nochange** variant
+//! keeps the 1% added/removed but makes every shared row equal, so the cell pass
+//! materializes nothing: the difference in peak RSS between it and the default
+//! run is the cell pass's cost. The **allchange** variant drops the offset and
+//! changes every shared row (no added/removed), so the cell pass materializes
+//! and renders every row. The **wide** shape (`id` int64, `value` a
+//! `value_width`-byte Utf8 that differs between the sides) changes every row too
+//! and renders `value_width` bytes per changed cell — the rendering worst case,
+//! whose peak RSS scales with changed cells times cell width. The **dup** shape
+//! (`key` Utf8 of the given width, `value` int64) makes every key appear twice
+//! on each side, so every distinct
+//! key is a duplicate and the whole `duplicate_keys` report is materialized —
+//! the term that scales with distinct duplicated keys times the key width.
+
+use onix_arrow::{TableDiffOptions, diff_tables};
+
+#[path = "shared/gen_shapes.rs"]
+mod gen_shapes;
+use gen_shapes::{Case, Generated, batch_rows};
+
+/// Options for the diff, honoring a `ROW_DIFF_THREADS` override so the parallel
+/// path's peak RSS can be compared against the single-threaded baseline; unset
+/// uses the default (available parallelism).
+fn options_from_env(key: &str) -> TableDiffOptions {
+    let mut options = TableDiffOptions::new(vec![key.to_string()]);
+    if let Some(threads) = std::env::var("ROW_DIFF_THREADS")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .and_then(std::num::NonZeroUsize::new)
+    {
+        options = options
+            .with_threads(threads)
+            .expect("ROW_DIFF_THREADS within MAX_THREADS");
+    }
+    options
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    let rows: i64 = args
+        .get(1)
+        .and_then(|a| a.parse().ok())
+        .unwrap_or(1_000_000);
+    let mode = args.get(2).map_or("", String::as_str);
+    let width: usize = args
+        .get(3)
+        .and_then(|a| a.parse().ok())
+        .unwrap_or(if mode == "wide" { 1024 } else { 16 });
+
+    let (case, label) = match mode {
+        "" | "linear" => (Case::Linear, String::new()),
+        "nochange" => (Case::NoChange, " (nochange baseline)".to_string()),
+        "allchange" => (Case::AllChange, " (all changed)".to_string()),
+        "wide" => (Case::Wide(width), format!(" (wide, value_width={width})")),
+        "widesame" => (
+            Case::WideSame(width),
+            format!(" (widesame, value_width={width})"),
+        ),
+        "manycols" => {
+            let ncols: usize = args.get(3).and_then(|a| a.parse().ok()).unwrap_or(8);
+            let width: usize = args.get(4).and_then(|a| a.parse().ok()).unwrap_or(512);
+            (
+                Case::ManyCols { ncols, width },
+                format!(" (manycols, ncols={ncols}, width={width})"),
+            )
+        }
+        "dup" => (Case::Dup(width), format!(" (dup, key_width={width})")),
+        other => {
+            eprintln!(
+                "unknown mode {other:?}; expected linear (the default), nochange, allchange, wide, widesame, manycols or dup"
+            );
+            std::process::exit(2);
+        }
+    };
+    let (schema, left_shape, right_shape, key) = case.build(rows);
+
+    let batch = batch_rows();
+    let left = Generated {
+        schema: schema.clone(),
+        rows,
+        shape: left_shape,
+        batch,
+    };
+    let right = Generated {
+        schema,
+        rows,
+        shape: right_shape,
+        batch,
+    };
+
+    let options = options_from_env(key);
+    let start = std::time::Instant::now();
+    let diff = diff_tables(&left, &right, &options).expect("diff succeeds");
+    let elapsed = start.elapsed();
+    let summary = diff.summary();
+
+    println!("rows per side: {rows}{label}");
+    println!("threads: {}", options.threads());
+    println!("wall: {:.2}s", elapsed.as_secs_f64());
+    println!(
+        "rows_added={} rows_removed={} rows_changed={} duplicate_keys={} cells_changed={}",
+        summary.rows_added,
+        summary.rows_removed,
+        summary.rows_changed,
+        summary.duplicate_keys,
+        summary.cells_changed
+    );
+}