deepdiff-rs 0.6.2__tar.gz → 0.7.1__tar.gz

{deepdiff_rs-0.6.2 → deepdiff_rs-0.7.1}/Cargo.lock

@@ -92,6 +92,20 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-ipc"
+version = "59.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599"
+dependencies = [
+ "arrow-array",
+ "arrow-buffer",
+ "arrow-data",
+ "arrow-schema",
+ "arrow-select",
+ "flatbuffers",
+]
+
 [[package]]
 name = "arrow-ord"
 version = "59.3.0"
@@ -291,6 +305,16 @@ version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d"
 
+[[package]]
+name = "flatbuffers"
+version = "25.12.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3"
+dependencies = [
+ "bitflags",
+ "rustc_version",
+]
+
 [[package]]
 name = "fnv"
 version = "1.0.7"
@@ -605,17 +629,24 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.6.2"
+version = "0.7.1"
 dependencies = [
  "arrow-array",
+ "arrow-buffer",
+ "arrow-cast",
  "arrow-schema",
+ "arrow-select",
+ "getrandom 0.3.4",
+ "half",
+ "proptest",
  "serde",
  "serde_json",
+ "siphasher",
 ]
 
 [[package]]
 name = "onix-cli"
-version = "0.6.2"
+version = "0.7.1"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -623,25 +654,28 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.6.2"
+version = "0.7.1"
 dependencies = [
  "proptest",
  "serde",
  "serde_json",
  "stats_alloc",
+ "unicode-general-category",
 ]
 
 [[package]]
 name = "onix-py"
-version = "0.6.2"
+version = "0.7.1"
 dependencies = [
  "arrow-array",
+ "arrow-ipc",
  "arrow-schema",
  "onix-arrow",
  "onix-core",
  "pyo3",
  "pyo3-arrow",
  "serde_json",
+ "tempfile",
 ]
 
 [[package]]
@@ -884,6 +918,15 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustix"
 version = "1.1.4"
@@ -921,6 +964,12 @@ version = "1.0.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+
 [[package]]
 name = "serde"
 version = "1.0.229"
@@ -1064,6 +1113,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
 
+[[package]]
+name = "unicode-general-category"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.24"

{deepdiff_rs-0.6.2 → deepdiff_rs-0.7.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.6.2"
+version = "0.7.1"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.6.2 → deepdiff_rs-0.7.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.6.2
+Version: 0.7.1
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -118,34 +118,36 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-This version reports the **schema** diff: which columns were added, removed, or changed type. Row-level results (which rows were added or removed and which cells changed) arrive in a later version; the `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` members exist now but raise `NotImplementedError`.
+It reports the **schema** diff (which columns were added, removed, or changed type) and the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated). Rows are matched by the key columns; `rows_added`, `rows_removed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome. The per-cell diff (`cells_changed`) arrives in a later version and raises `NotImplementedError` until then.
 
 ```python
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
 left = pa.table({
-    "id": pa.array([1, 2], pa.int64()),
-    "amount": pa.array([1, 2], pa.int32()),
+    "id": pa.array([1, 2, 3], pa.int64()),
+    "amount": pa.array([10, 20, 30], pa.int32()),
 })
 right = pa.table({
-    "id": pa.array([1, 2], pa.int64()),
-    "amount": pa.array([1, 2], pa.int64()),
-    "note": pa.array(["a", "b"], pa.string()),
+    "id": pa.array([2, 3, 4], pa.int64()),
+    "amount": pa.array([20, 31, 40], pa.int64()),
+    "note": pa.array(["a", "b", "c"], pa.string()),
 })
 
 diff = diff_tables(left, right, key=["id"])
 print(diff.summary())
-for change in diff.schema:
-    print(change["column"], change["change"], change["left_type"], change["right_type"])
+print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
+print("removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
 ```
 
 ```
-{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1}
-amount type_changed Int32 Int64
-note added None Utf8
+{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0}
+added ids: [4]
+removed ids: [1]
 ```
 
+A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, `1.00` equals `1.0000`, a timestamp compares by its instant across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+
 Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
 
 `pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra.
@@ -158,15 +160,16 @@ Type comparison uses the full logical Arrow type (timestamp unit and timezone, d
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime`, `time` and `timedelta` are not supported. The normalized-versus-raw split is documented in [`tests/golden/README.md`](tests/golden/README.md).
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
 - A `str` containing a lone (unpaired) surrogate code point (e.g. `'\udc80'`, legal in Python but not encodable as UTF-8) raises `ValueError` naming the exact path on either side, before the two values are ever compared — including a pair DeepDiff would call equal and report as no change, since DeepDiff's scalar equality is plain Python `==` and never hits the encoding problem; DeepDiff does report a plain change for a *differing* pair, and crashes with an unhandled `UnicodeEncodeError` if such a string is ever hashed (a `set`/`frozenset` member). See [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
-- A `str` inside a `tuple` or `frozenset` set item is rendered with Python's `repr()`, which escapes every non-printable character; onix escapes those below `U+0100` (the complete set in that range) and passes higher non-printable code points through literally, since escaping them would mean carrying a Unicode category table. Exact for all of ASCII and all printable text. See [`crates/onix-core/src/path.rs`](crates/onix-core/src/path.rs).
+- A `str` inside a `tuple` or `frozenset` set item is escaped exactly as Python's `repr()` escapes it, against Unicode 16.0.0; on a Python older than 3.14 (an older `unicodedata` table), a code point assigned to Unicode after that Python's own version is escaped by DeepDiff and rendered literally by onix. See [`tests/golden/README.md`](tests/golden/README.md)'s "Pinned versions" section.
 - Adversarially deep input raises `MaxDepthError` instead of crashing: the default `max_depth` is 512 and the hard ceiling is `MAX_DEPTH_CEILING` (20,000). See [`crates/onix-py/src/guard.rs`](crates/onix-py/src/guard.rs).
 - `ignore_order` pairing is `O(N^2)` in unpaired elements per side and carries a polynomial cost in both time and memory with input depth; it has no `max_passes`/`max_diffs` cutoff, so bound the size and depth of untrusted input yourself. See [`crates/onix-core/src/ignore_order/mod.rs`](crates/onix-core/src/ignore_order/mod.rs).
 - A `values_changed` between two multi-line strings runs a `difflib`-style `O(N*M)` line diff on the default path with no opt-out, worst when changes are spread evenly through the text (about 35 s for a heavily edited 1 MB string and growing quadratically, so a few megabytes is minutes), so bound the size of untrusted strings yourself. See [`crates/onix-core/src/unified_diff.rs`](crates/onix-core/src/unified_diff.rs).
 - Ordered sequences (`list` or `tuple`) of scalars (null, bool, number, string, datetime, date) run a `difflib`-style `O(N*M)` matcher on the default path with its popular-element (autojunk) purge disabled for `DeepDiff` parity, worst for sequences of a few repeated values with dense edits (about 620 s for two 8,000-element lists of two repeated values with every other element changed, growing faster than quadratically), so bound the size of untrusted sequences yourself. See [`crates/onix-core/src/lcs.rs`](crates/onix-core/src/lcs.rs).
-- In `diff_tables`, a column name containing an embedded NUL byte (`\0`) arrives truncated at the NUL through the Arrow C Data Interface, and the report shows the truncated name; such names are rare in practice.
-- In `diff_tables`, a list of structs named exactly `key`/`value` with a nullable key and a real map are not distinguished, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type); see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs).
+- `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; and temp disk, because each input is read twice and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: bound the row count and, for duplicate-heavy data, the key width of untrusted input. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced).
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
 
 ## Performance
@@ -218,12 +221,12 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 
 ## Reference
 
-**Python API.** The public surface is `DeepDiff`, `diff_json`, `MaxDepthError`, and `MAX_DEPTH_CEILING`.
+**Python API.** The public surface is `DeepDiff`, `diff_json`, `diff_tables` (returning a `TableDiff`), `MaxDepthError`, and `MAX_DEPTH_CEILING`.
 
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict` (with `str` keys), `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, and `datetime.date` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date` comes back as a real `datetime`/`date` — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the change counts, `.to_json()` both as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` raise `NotImplementedError` until a later version.
+- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, and `.duplicate_keys()` return Arrow tables, and `.cells_changed()` raises `NotImplementedError` until a later version.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -245,7 +248,7 @@ Exit codes:
 ```
 crates/onix-core   # the diff engine (library, no I/O)
 crates/onix-cli    # the `onix` binary (thin CLI over the core)
-crates/onix-arrow  # Arrow table diffing (schema diff; rows in a later version)
+crates/onix-arrow  # Arrow table diffing (schema diff and keyed row diff)
 crates/onix-py     # PyO3 bindings, published as `deepdiff-rs`
 scripts/           # gen_goldens.py: regenerates tests/golden/ from real DeepDiff
 tests/golden       # DeepDiff-generated expected outputs (the compatibility corpus)

{deepdiff_rs-0.6.2 → deepdiff_rs-0.7.1}/README.md

@@ -100,34 +100,36 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-This version reports the **schema** diff: which columns were added, removed, or changed type. Row-level results (which rows were added or removed and which cells changed) arrive in a later version; the `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` members exist now but raise `NotImplementedError`.
+It reports the **schema** diff (which columns were added, removed, or changed type) and the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated). Rows are matched by the key columns; `rows_added`, `rows_removed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome. The per-cell diff (`cells_changed`) arrives in a later version and raises `NotImplementedError` until then.
 
 ```python
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
 left = pa.table({
-    "id": pa.array([1, 2], pa.int64()),
-    "amount": pa.array([1, 2], pa.int32()),
+    "id": pa.array([1, 2, 3], pa.int64()),
+    "amount": pa.array([10, 20, 30], pa.int32()),
 })
 right = pa.table({
-    "id": pa.array([1, 2], pa.int64()),
-    "amount": pa.array([1, 2], pa.int64()),
-    "note": pa.array(["a", "b"], pa.string()),
+    "id": pa.array([2, 3, 4], pa.int64()),
+    "amount": pa.array([20, 31, 40], pa.int64()),
+    "note": pa.array(["a", "b", "c"], pa.string()),
 })
 
 diff = diff_tables(left, right, key=["id"])
 print(diff.summary())
-for change in diff.schema:
-    print(change["column"], change["change"], change["left_type"], change["right_type"])
+print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
+print("removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
 ```
 
 ```
-{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1}
-amount type_changed Int32 Int64
-note added None Utf8
+{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0}
+added ids: [4]
+removed ids: [1]
 ```
 
+A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, `1.00` equals `1.0000`, a timestamp compares by its instant across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+
 Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
 
 `pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra.
@@ -140,15 +142,16 @@ Type comparison uses the full logical Arrow type (timestamp unit and timezone, d
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime`, `time` and `timedelta` are not supported. The normalized-versus-raw split is documented in [`tests/golden/README.md`](tests/golden/README.md).
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
 - A `str` containing a lone (unpaired) surrogate code point (e.g. `'\udc80'`, legal in Python but not encodable as UTF-8) raises `ValueError` naming the exact path on either side, before the two values are ever compared — including a pair DeepDiff would call equal and report as no change, since DeepDiff's scalar equality is plain Python `==` and never hits the encoding problem; DeepDiff does report a plain change for a *differing* pair, and crashes with an unhandled `UnicodeEncodeError` if such a string is ever hashed (a `set`/`frozenset` member). See [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
-- A `str` inside a `tuple` or `frozenset` set item is rendered with Python's `repr()`, which escapes every non-printable character; onix escapes those below `U+0100` (the complete set in that range) and passes higher non-printable code points through literally, since escaping them would mean carrying a Unicode category table. Exact for all of ASCII and all printable text. See [`crates/onix-core/src/path.rs`](crates/onix-core/src/path.rs).
+- A `str` inside a `tuple` or `frozenset` set item is escaped exactly as Python's `repr()` escapes it, against Unicode 16.0.0; on a Python older than 3.14 (an older `unicodedata` table), a code point assigned to Unicode after that Python's own version is escaped by DeepDiff and rendered literally by onix. See [`tests/golden/README.md`](tests/golden/README.md)'s "Pinned versions" section.
 - Adversarially deep input raises `MaxDepthError` instead of crashing: the default `max_depth` is 512 and the hard ceiling is `MAX_DEPTH_CEILING` (20,000). See [`crates/onix-py/src/guard.rs`](crates/onix-py/src/guard.rs).
 - `ignore_order` pairing is `O(N^2)` in unpaired elements per side and carries a polynomial cost in both time and memory with input depth; it has no `max_passes`/`max_diffs` cutoff, so bound the size and depth of untrusted input yourself. See [`crates/onix-core/src/ignore_order/mod.rs`](crates/onix-core/src/ignore_order/mod.rs).
 - A `values_changed` between two multi-line strings runs a `difflib`-style `O(N*M)` line diff on the default path with no opt-out, worst when changes are spread evenly through the text (about 35 s for a heavily edited 1 MB string and growing quadratically, so a few megabytes is minutes), so bound the size of untrusted strings yourself. See [`crates/onix-core/src/unified_diff.rs`](crates/onix-core/src/unified_diff.rs).
 - Ordered sequences (`list` or `tuple`) of scalars (null, bool, number, string, datetime, date) run a `difflib`-style `O(N*M)` matcher on the default path with its popular-element (autojunk) purge disabled for `DeepDiff` parity, worst for sequences of a few repeated values with dense edits (about 620 s for two 8,000-element lists of two repeated values with every other element changed, growing faster than quadratically), so bound the size of untrusted sequences yourself. See [`crates/onix-core/src/lcs.rs`](crates/onix-core/src/lcs.rs).
-- In `diff_tables`, a column name containing an embedded NUL byte (`\0`) arrives truncated at the NUL through the Arrow C Data Interface, and the report shows the truncated name; such names are rare in practice.
-- In `diff_tables`, a list of structs named exactly `key`/`value` with a nullable key and a real map are not distinguished, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type); see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs).
+- `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; and temp disk, because each input is read twice and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: bound the row count and, for duplicate-heavy data, the key width of untrusted input. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced).
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
 
 ## Performance
@@ -200,12 +203,12 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 
 ## Reference
 
-**Python API.** The public surface is `DeepDiff`, `diff_json`, `MaxDepthError`, and `MAX_DEPTH_CEILING`.
+**Python API.** The public surface is `DeepDiff`, `diff_json`, `diff_tables` (returning a `TableDiff`), `MaxDepthError`, and `MAX_DEPTH_CEILING`.
 
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict` (with `str` keys), `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, and `datetime.date` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date` comes back as a real `datetime`/`date` — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the change counts, `.to_json()` both as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` raise `NotImplementedError` until a later version.
+- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, and `.duplicate_keys()` return Arrow tables, and `.cells_changed()` raises `NotImplementedError` until a later version.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -227,7 +230,7 @@ Exit codes:
 ```
 crates/onix-core   # the diff engine (library, no I/O)
 crates/onix-cli    # the `onix` binary (thin CLI over the core)
-crates/onix-arrow  # Arrow table diffing (schema diff; rows in a later version)
+crates/onix-arrow  # Arrow table diffing (schema diff and keyed row diff)
 crates/onix-py     # PyO3 bindings, published as `deepdiff-rs`
 scripts/           # gen_goldens.py: regenerates tests/golden/ from real DeepDiff
 tests/golden       # DeepDiff-generated expected outputs (the compatibility corpus)

deepdiff_rs-0.7.1/crates/onix-arrow/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "onix-arrow"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+publish = false
+description = "Arrow table diffing (schema and keyed rows) built on the onix diff engine"
+
+[lints]
+workspace = true
+
+[dependencies]
+# Pinned to an exact Arrow version so this crate and the `pyo3-arrow` bridge
+# in `onix-py` (which requires `arrow` ^59) resolve to the identical Arrow
+# types — a mismatch would make the record batches handed across the FFI
+# boundary a different, incompatible type. 59.3.0 is the newest 59.x at the
+# time this crate landed. Only the schema/array sub-crates are pulled in: the
+# C Data Interface glue lives in `onix-py` behind `pyo3-arrow`, so this crate
+# needs no `ffi` feature.
+arrow-schema = "=59.3.0"
+arrow-array = "=59.3.0"
+# `i256` for hashing `Decimal256` cells the same way as `Decimal128`.
+arrow-buffer = "=59.3.0"
+# `arrow-cast` decodes dictionary-encoded columns to their value type before
+# hashing, and casts a right-only duplicate key's columns to the left key
+# types; `arrow-select` filters and concatenates the output row batches. The
+# row diff re-reads each side through the caller's re-openable `TableInput`, so
+# no IPC spooling lives here — the Python bindings own that.
+arrow-cast = "=59.3.0"
+arrow-select = "=59.3.0"
+# Row identity is a keyed 128-bit SipHash-1-3 (`siphasher`), keyed from 16
+# bytes of OS randomness per diff (`getrandom`), so the row-matching table on
+# this default path cannot be forced into collisions by chosen input.
+siphasher = "1"
+getrandom = "0.3"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+[dev-dependencies]
+proptest = "1"
+# `half::f16` builds a Float16 test column; the same version arrow-array uses.
+half = "2"

deepdiff_rs-0.7.1/crates/onix-arrow/examples/row_diff_rss.rs

@@ -0,0 +1,189 @@
+//! Measures the keyed row diff's peak memory and wall time, to check the memory
+//! bounds the README states.
+//!
+//! Run under the OS's max-RSS reporter:
+//!
+//! ```sh
+//! cargo build -p onix-arrow --release --example row_diff_rss
+//! # linear shape (default): mostly-matching rows, 1% added/removed, ~2% changed
+//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000
+//! /usr/bin/time -l target/release/examples/row_diff_rss 10000000
+//! # duplicate-heavy shape: every key duplicated, wide string key
+//! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 dup 16
+//! /usr/bin/time -l target/release/examples/row_diff_rss 200000 dup 1024
+//! ```
+//!
+//! Each side is generated on the fly, batch by batch, and nothing is retained
+//! between batches, so the process's peak RSS is the diff's own state, not the
+//! table data. The **linear** shape (`id`, `value` int64 columns) exercises the
+//! per-row hash vectors: the left is ids `0..n`, the right `step..n + step` with
+//! `step = n / 100`, so 1% removed, 1% added, ~2% changed. The **dup** shape
+//! (`key` Utf8 of the given width, `value` int64) makes every key appear twice
+//! on each side, so every distinct key is a duplicate and the whole
+//! `duplicate_keys` report is materialized — the term that scales with distinct
+//! duplicated keys times the key width.
+
+use std::sync::Arc;
+
+use arrow_array::{ArrayRef, Int64Array, RecordBatch, RecordBatchReader, StringArray};
+use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
+use onix_arrow::{TableDiffError, TableDiffOptions, TableInput, diff_tables};
+
+const BATCH: i64 = 65_536;
+
+/// The generated table shape.
+#[derive(Clone, Copy)]
+enum Shape {
+    /// `(id, value)` int64 columns; `id_offset` shifts the key range and
+    /// `change_every` perturbs a fraction of values.
+    Linear { id_offset: i64, change_every: i64 },
+    /// `(key, value)`; `key` is a `key_width`-byte string and each key value
+    /// appears twice, so every key is a duplicate.
+    Dup { key_width: usize },
+}
+
+/// A table generated on demand, retaining nothing between batches.
+struct Generated {
+    schema: SchemaRef,
+    rows: i64,
+    shape: Shape,
+}
+
+impl TableInput for Generated {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+
+    fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
+        Ok(Box::new(GenReader {
+            schema: self.schema.clone(),
+            rows: self.rows,
+            shape: self.shape,
+            next: 0,
+        }))
+    }
+}
+
+struct GenReader {
+    schema: SchemaRef,
+    rows: i64,
+    shape: Shape,
+    next: i64,
+}
+
+impl Iterator for GenReader {
+    type Item = Result<RecordBatch, ArrowError>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.next >= self.rows {
+            return None;
+        }
+        let end = (self.next + BATCH).min(self.rows);
+        let columns: Vec<ArrayRef> = match self.shape {
+            Shape::Linear {
+                id_offset,
+                change_every,
+            } => {
+                let ids: Int64Array = (self.next..end).map(|i| Some(i + id_offset)).collect();
+                // The value is a function of the id, so a shared key holds the
+                // same value on both sides except every `change_every`-th id.
+                let values: Int64Array = (self.next..end)
+                    .map(|i| {
+                        let id = i + id_offset;
+                        Some(if id % change_every == 0 { id + 1 } else { id })
+                    })
+                    .collect();
+                vec![Arc::new(ids), Arc::new(values)]
+            }
+            Shape::Dup { key_width } => {
+                // Key value `i / 2`, so each distinct key appears twice.
+                let keys: StringArray = (self.next..end)
+                    .map(|i| Some(format!("{:0>width$}", i / 2, width = key_width)))
+                    .collect();
+                let values: Int64Array = (self.next..end).map(Some).collect();
+                vec![Arc::new(keys), Arc::new(values)]
+            }
+        };
+        self.next = end;
+
+        Some(RecordBatch::try_new(self.schema.clone(), columns))
+    }
+}
+
+impl RecordBatchReader for GenReader {
+    fn schema(&self) -> SchemaRef {
+        self.schema.clone()
+    }
+}
+
+fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    let rows: i64 = args
+        .get(1)
+        .and_then(|a| a.parse().ok())
+        .unwrap_or(1_000_000);
+    let dup = args.get(2).is_some_and(|a| a == "dup");
+    let key_width: usize = args.get(3).and_then(|a| a.parse().ok()).unwrap_or(16);
+
+    let (schema, left_shape, right_shape, key) = if dup {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("key", DataType::Utf8, false),
+            Field::new("value", DataType::Int64, false),
+        ]));
+        (
+            schema,
+            Shape::Dup { key_width },
+            Shape::Dup { key_width },
+            "key",
+        )
+    } else {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("value", DataType::Int64, false),
+        ]));
+        let step = (rows / 100).max(1);
+        (
+            schema,
+            Shape::Linear {
+                id_offset: 0,
+                change_every: i64::MAX,
+            },
+            Shape::Linear {
+                id_offset: step,
+                change_every: 50,
+            },
+            "id",
+        )
+    };
+
+    let left = Generated {
+        schema: schema.clone(),
+        rows,
+        shape: left_shape,
+    };
+    let right = Generated {
+        schema,
+        rows,
+        shape: right_shape,
+    };
+
+    let start = std::time::Instant::now();
+    let diff = diff_tables(&left, &right, &TableDiffOptions::new(vec![key.to_string()]))
+        .expect("diff succeeds");
+    let elapsed = start.elapsed();
+    let summary = diff.summary();
+
+    println!(
+        "rows per side: {rows}{}",
+        if dup {
+            format!(" (dup, key_width={key_width})")
+        } else {
+            String::new()
+        }
+    );
+    println!("wall: {:.2}s", elapsed.as_secs_f64());
+    println!(
+        "rows_added={} rows_removed={} rows_changed={} duplicate_keys={}",
+        summary.rows_added, summary.rows_removed, summary.rows_changed, summary.duplicate_keys
+    );
+}