deepdiff-rs 0.8.2__tar.gz → 0.9.1__tar.gz

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.1}/Cargo.lock

@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.8.2"
+version = "0.9.1"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -646,7 +646,7 @@ dependencies = [
 
 [[package]]
 name = "onix-cli"
-version = "0.8.2"
+version = "0.9.1"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -654,7 +654,7 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.8.2"
+version = "0.9.1"
 dependencies = [
  "proptest",
  "serde",
@@ -665,7 +665,7 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.8.2"
+version = "0.9.1"
 dependencies = [
  "arrow-array",
  "arrow-ipc",
@@ -997,7 +997,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 3.0.4",
+ "syn 3.0.5",
 ]
 
 [[package]]
@@ -1050,9 +1050,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "3.0.4"
+version = "3.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
+checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1199,7 +1199,7 @@ dependencies = [
  "bumpalo",
  "proc-macro2",
  "quote",
- "syn 3.0.4",
+ "syn 3.0.5",
  "wasm-bindgen-shared",
 ]
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.1}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.8.2"
+version = "0.9.1"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.8.2
+Version: 0.9.1
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -56,7 +56,7 @@ From source:
 ```sh
 cd crates/onix-py
 uv tool install maturin              # the build tool (skip if already installed)
-uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb for the table-diff tests
+uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb/pandas for the table-diff tests
 uv run --group test maturin develop --release
 ```
 
@@ -118,15 +118,15 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
+It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
 
 ```python
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
 left = pa.table({
-    "id": pa.array([1, 2, 3], pa.int64()),
-    "amount": pa.array([10, 20, 30], pa.int32()),
+    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
+    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
 })
 right = pa.table({
     "id": pa.array([2, 3, 4], pa.int64()),
@@ -135,22 +135,20 @@ right = pa.table({
 })
 
 diff = diff_tables(left, right, key=["id"])
-print(diff.summary())
-print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
-print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
+print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
+print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate keys:", pa.table(diff.duplicate_keys()).to_pylist())
 ```
 
 ```
-{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
-added ids: [4]
-cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
+{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 1, 'null_keys': 0, 'cells_changed': 1} added ids: [4] removed ids: [1]
+cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
 A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra.
+`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
 
 ## Performance
 
@@ -206,7 +204,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions, including which types a `dict` key may be); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date`/`time`/`timedelta` comes back as a real one of those — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
+- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -238,10 +236,11 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested). `int`s must fit in `i64`/`u64`, `float`s must be finite, and anything else — a custom object, an arbitrary-precision `int`, or a non-finite `float` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes** and **Sets** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested). `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported type (a `tuple`, `set` or `frozenset` subclass including `namedtuple`, a `datetime`/`date`/`time`/`timedelta` subclass such as pandas' `Timestamp`) raises `TypeError` rather than being diffed as its base type, because DeepDiff reports each value's own type name. A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. Both are described in [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
+- **Non-finite floats** (`NaN`, `Infinity`, `-Infinity`) compare and hash like real Python: two `NaN`s are never equal, `Infinity == Infinity`, `to_json()` renders the same bare `NaN`/`Infinity`/`-Infinity` tokens Python's `json.dumps` does, and `ignore_order` matching treats every `NaN` as one shared item, matching `DeepHash`. The one divergence, always deterministic: this crate's value model carries no Python object identity, so two independently-obtained `NaN`s always compare unequal here, where DeepDiff sometimes reports no difference (`t1 is t2`) or lets one collapse into another (a `set` member, an ordered-list match) when the two objects, or their containers, happen to be the same one. See [`tests/golden/README.md`](tests/golden/README.md)'s "Non-finite floats" section.
 - A `str` containing a lone (unpaired) surrogate code point (e.g. `'\udc80'`, legal in Python but not encodable as UTF-8) raises `ValueError` naming the exact path on either side, before the two values are ever compared — including a pair DeepDiff would call equal and report as no change, since DeepDiff's scalar equality is plain Python `==` and never hits the encoding problem; DeepDiff does report a plain change for a *differing* pair, and crashes with an unhandled `UnicodeEncodeError` if such a string is ever hashed (a `set`/`frozenset` member). See [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - A `str` inside a `tuple` or `frozenset` set item is escaped exactly as Python's `repr()` escapes it, against Unicode 16.0.0; on a Python older than 3.14 (an older `unicodedata` table), a code point assigned to Unicode after that Python's own version is escaped by DeepDiff and rendered literally by onix. See [`tests/golden/README.md`](tests/golden/README.md)'s "Pinned versions" section.
 - Adversarially deep input raises `MaxDepthError` instead of crashing: the default `max_depth` is 512 and the hard ceiling is `MAX_DEPTH_CEILING` (20,000). See [`crates/onix-py/src/guard.rs`](crates/onix-py/src/guard.rs).
@@ -252,6 +251,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
 - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.1}/README.md

@@ -38,7 +38,7 @@ From source:
 ```sh
 cd crates/onix-py
 uv tool install maturin              # the build tool (skip if already installed)
-uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb for the table-diff tests
+uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb/pandas for the table-diff tests
 uv run --group test maturin develop --release
 ```
 
@@ -100,15 +100,15 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
+It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
 
 ```python
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
 left = pa.table({
-    "id": pa.array([1, 2, 3], pa.int64()),
-    "amount": pa.array([10, 20, 30], pa.int32()),
+    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
+    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
 })
 right = pa.table({
     "id": pa.array([2, 3, 4], pa.int64()),
@@ -117,22 +117,20 @@ right = pa.table({
 })
 
 diff = diff_tables(left, right, key=["id"])
-print(diff.summary())
-print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
-print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
+print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
+print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate keys:", pa.table(diff.duplicate_keys()).to_pylist())
 ```
 
 ```
-{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
-added ids: [4]
-cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
+{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 1, 'null_keys': 0, 'cells_changed': 1} added ids: [4] removed ids: [1]
+cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
 A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra.
+`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
 
 ## Performance
 
@@ -188,7 +186,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions, including which types a `dict` key may be); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date`/`time`/`timedelta` comes back as a real one of those — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
+- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -220,10 +218,11 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested). `int`s must fit in `i64`/`u64`, `float`s must be finite, and anything else — a custom object, an arbitrary-precision `int`, or a non-finite `float` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes** and **Sets** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested). `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported type (a `tuple`, `set` or `frozenset` subclass including `namedtuple`, a `datetime`/`date`/`time`/`timedelta` subclass such as pandas' `Timestamp`) raises `TypeError` rather than being diffed as its base type, because DeepDiff reports each value's own type name. A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. Both are described in [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
+- **Non-finite floats** (`NaN`, `Infinity`, `-Infinity`) compare and hash like real Python: two `NaN`s are never equal, `Infinity == Infinity`, `to_json()` renders the same bare `NaN`/`Infinity`/`-Infinity` tokens Python's `json.dumps` does, and `ignore_order` matching treats every `NaN` as one shared item, matching `DeepHash`. The one divergence, always deterministic: this crate's value model carries no Python object identity, so two independently-obtained `NaN`s always compare unequal here, where DeepDiff sometimes reports no difference (`t1 is t2`) or lets one collapse into another (a `set` member, an ordered-list match) when the two objects, or their containers, happen to be the same one. See [`tests/golden/README.md`](tests/golden/README.md)'s "Non-finite floats" section.
 - A `str` containing a lone (unpaired) surrogate code point (e.g. `'\udc80'`, legal in Python but not encodable as UTF-8) raises `ValueError` naming the exact path on either side, before the two values are ever compared — including a pair DeepDiff would call equal and report as no change, since DeepDiff's scalar equality is plain Python `==` and never hits the encoding problem; DeepDiff does report a plain change for a *differing* pair, and crashes with an unhandled `UnicodeEncodeError` if such a string is ever hashed (a `set`/`frozenset` member). See [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - A `str` inside a `tuple` or `frozenset` set item is escaped exactly as Python's `repr()` escapes it, against Unicode 16.0.0; on a Python older than 3.14 (an older `unicodedata` table), a code point assigned to Unicode after that Python's own version is escaped by DeepDiff and rendered literally by onix. See [`tests/golden/README.md`](tests/golden/README.md)'s "Pinned versions" section.
 - Adversarially deep input raises `MaxDepthError` instead of crashing: the default `max_depth` is 512 and the hard ceiling is `MAX_DEPTH_CEILING` (20,000). See [`crates/onix-py/src/guard.rs`](crates/onix-py/src/guard.rs).
@@ -234,6 +233,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
 - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.1}/crates/onix-arrow/src/error.rs

@@ -121,6 +121,25 @@ pub enum TableDiffError {
         /// The column whose two renderings were equal.
         column: String,
     },
+    /// [`crate::TableDiff::to_json`] would embed more row objects than
+    /// [`crate::MAX_JSON_ROWS`] allows (see its own doc for what that caps
+    /// and why). Use the Arrow-returning members instead, or export the
+    /// batches directly, for a diff this large.
+    TooManyJsonRows {
+        /// The number of row objects `to_json()` would have embedded.
+        rows: usize,
+        /// The cap that was exceeded.
+        max: usize,
+    },
+    /// [`crate::TableDiff::to_json`]'s `serde_json` serialization failed.
+    /// Its input is a fixed set of already-rendered strings, numbers, and
+    /// nested objects/arrays, so this does not happen in practice; it is a
+    /// typed error rather than a panic because a public API must return,
+    /// not abort, on an unexpected failure.
+    Json {
+        /// The underlying `serde_json` error's message.
+        message: String,
+    },
 }
 
 impl fmt::Display for TableDiffError {
@@ -170,6 +189,15 @@ impl fmt::Display for TableDiffError {
                 "internal invariant: a value change in column {column:?} rendered identically \
                  on both sides, which the common-form rendering is designed to prevent"
             ),
+            TableDiffError::TooManyJsonRows { rows, max } => write!(
+                f,
+                "to_json() would embed {rows} row objects, more than the {max}-row cap; use \
+                 the Arrow-returning members (rows_added(), rows_removed(), cells_changed(), \
+                 duplicate_keys()) or export the batches directly for a diff this large"
+            ),
+            TableDiffError::Json { message } => {
+                write!(f, "failed to serialize the table diff to JSON: {message}")
+            }
         }
     }
 }
@@ -278,6 +306,27 @@ mod tests {
         assert!(message.contains("changed-row count"));
     }
 
+    #[test]
+    fn too_many_json_rows_message_names_count_and_cap() {
+        let error = TableDiffError::TooManyJsonRows {
+            rows: 20_000,
+            max: 10_000,
+        };
+        let message = error.to_string();
+        assert!(message.contains("20000"));
+        assert!(message.contains("10000-row cap"));
+        assert!(message.contains("rows_added()"));
+    }
+
+    #[test]
+    fn json_message_names_the_underlying_error() {
+        let error = TableDiffError::Json {
+            message: "unexpected end of input".to_string(),
+        };
+        let message = error.to_string();
+        assert!(message.contains("unexpected end of input"));
+    }
+
     #[test]
     fn error_implements_std_error() {
         let error = TableDiffError::EmptyKey;

deepdiff_rs-0.9.1/crates/onix-arrow/src/json_rows.rs

@@ -0,0 +1,141 @@
+//! Renders an Arrow [`RecordBatch`] to a JSON array of row objects, for
+//! [`crate::TableDiff::to_json`]. Reuses [`crate::row_diff::SideRenderer`],
+//! the same renderer [`crate::TableDiff::cells_changed`]'s `old_value`/
+//! `new_value` use, so no cell renders two different ways in the crate.
+//!
+//! [`MAX_JSON_ROWS`] bounds the row count: the JSON holds one object per row
+//! and one owned string per non-null cell, so memory is proportional to
+//! rows × columns × cell width, uncapped past that row count otherwise. A
+//! batch over the cap is refused with [`crate::TableDiffError::TooManyJsonRows`].
+
+use arrow_array::RecordBatch;
+use arrow_cast::display::FormatOptions;
+use serde_json::{Map, Value as JsonValue};
+
+use crate::error::TableDiffError;
+use crate::row_diff::{SideRenderer, cell_is_null};
+
+/// The maximum number of row objects this module will render for one
+/// [`RecordBatch`]; see the module doc for what this bounds.
+pub const MAX_JSON_ROWS: usize = 10_000;
+
+/// Renders every row of `batch` to a JSON object keyed by column name, or
+/// [`TableDiffError::TooManyJsonRows`] if `batch` has more than
+/// [`MAX_JSON_ROWS`] rows. A null cell renders as JSON `null`; every other
+/// cell renders through [`SideRenderer`], the same renderer
+/// [`crate::TableDiff::cells_changed`] uses.
+///
+/// # Errors
+///
+/// - [`TableDiffError::TooManyJsonRows`] if `batch.num_rows()` exceeds
+///   [`MAX_JSON_ROWS`].
+/// - [`TableDiffError::Render`] if a cell cannot be rendered to its
+///   canonical string (naming the column), the same failure
+///   [`crate::TableDiff::cells_changed`] can raise.
+pub(crate) fn rows_to_json(batch: &RecordBatch) -> Result<Vec<JsonValue>, TableDiffError> {
+    if batch.num_rows() > MAX_JSON_ROWS {
+        return Err(TableDiffError::TooManyJsonRows {
+            rows: batch.num_rows(),
+            max: MAX_JSON_ROWS,
+        });
+    }
+
+    let format_options = FormatOptions::default();
+    let schema = batch.schema();
+    let columns = schema
+        .fields()
+        .iter()
+        .zip(batch.columns())
+        .map(|(field, array)| {
+            SideRenderer::new(array, &format_options)
+                .map(|renderer| (field.name(), array, renderer))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let mut rows = Vec::with_capacity(batch.num_rows());
+    for row in 0..batch.num_rows() {
+        let mut object = Map::with_capacity(columns.len());
+        for (name, array, renderer) in &columns {
+            let value = if cell_is_null(array, row) {
+                JsonValue::Null
+            } else {
+                JsonValue::String(renderer.render(row, name)?)
+            };
+            object.insert((*name).clone(), value);
+        }
+        rows.push(JsonValue::Object(object));
+    }
+
+    Ok(rows)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{Int64Array, RecordBatch, StringArray};
+    use arrow_schema::{DataType, Field, Schema};
+
+    use super::{MAX_JSON_ROWS, rows_to_json};
+    use crate::error::TableDiffError;
+
+    fn batch(rows: usize) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("name", DataType::Utf8, true),
+        ]));
+        let ids: Vec<i64> = (0..rows).map(|i| i64::try_from(i).unwrap()).collect();
+        let names: Vec<Option<String>> = (0..rows)
+            .map(|i| {
+                if i % 2 == 0 {
+                    None
+                } else {
+                    Some(format!("n{i}"))
+                }
+            })
+            .collect();
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int64Array::from(ids)),
+                Arc::new(StringArray::from(names)),
+            ],
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn renders_one_object_per_row_with_column_keys() {
+        let rows = rows_to_json(&batch(2)).unwrap();
+        assert_eq!(rows.len(), 2);
+        assert_eq!(rows[0]["id"], serde_json::json!("0"));
+        assert_eq!(rows[0]["name"], serde_json::Value::Null);
+        assert_eq!(rows[1]["id"], serde_json::json!("1"));
+        assert_eq!(rows[1]["name"], serde_json::json!("n1"));
+    }
+
+    #[test]
+    fn empty_batch_renders_no_rows() {
+        assert_eq!(
+            rows_to_json(&batch(0)).unwrap(),
+            Vec::<serde_json::Value>::new()
+        );
+    }
+
+    #[test]
+    fn batch_over_the_cap_is_refused() {
+        let error = rows_to_json(&batch(MAX_JSON_ROWS + 1)).unwrap_err();
+        assert_eq!(
+            error,
+            TableDiffError::TooManyJsonRows {
+                rows: MAX_JSON_ROWS + 1,
+                max: MAX_JSON_ROWS,
+            }
+        );
+    }
+
+    #[test]
+    fn batch_at_the_cap_is_accepted() {
+        assert!(rows_to_json(&batch(MAX_JSON_ROWS)).is_ok());
+    }
+}

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.1}/crates/onix-arrow/src/lib.rs

@@ -102,12 +102,14 @@
 //! ```
 
 mod error;
+mod json_rows;
 mod options;
 mod row_diff;
 mod schema;
 mod table_diff;
 
 pub use error::{Side, TableDiffError};
+pub use json_rows::MAX_JSON_ROWS;
 pub use options::TableDiffOptions;
 pub use row_diff::{MemoryInput, TableInput};
 pub use schema::{ChangeKind, SchemaChange, diff_schemas};