deepdiff-rs 0.8.2__tar.gz → 0.9.0__tar.gz

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/Cargo.lock

@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.8.2"
+version = "0.9.0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -646,7 +646,7 @@ dependencies = [
 
 [[package]]
 name = "onix-cli"
-version = "0.8.2"
+version = "0.9.0"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -654,7 +654,7 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.8.2"
+version = "0.9.0"
 dependencies = [
  "proptest",
  "serde",
@@ -665,7 +665,7 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.8.2"
+version = "0.9.0"
 dependencies = [
  "arrow-array",
  "arrow-ipc",

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.8.2"
+version = "0.9.0"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.8.2
+Version: 0.9.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -56,7 +56,7 @@ From source:
 ```sh
 cd crates/onix-py
 uv tool install maturin              # the build tool (skip if already installed)
-uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb for the table-diff tests
+uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb/pandas for the table-diff tests
 uv run --group test maturin develop --release
 ```
 
@@ -118,15 +118,15 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
+It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
 
 ```python
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
 left = pa.table({
-    "id": pa.array([1, 2, 3], pa.int64()),
-    "amount": pa.array([10, 20, 30], pa.int32()),
+    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
+    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
 })
 right = pa.table({
     "id": pa.array([2, 3, 4], pa.int64()),
@@ -135,22 +135,20 @@ right = pa.table({
 })
 
 diff = diff_tables(left, right, key=["id"])
-print(diff.summary())
-print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
-print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
+print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
+print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate keys:", pa.table(diff.duplicate_keys()).to_pylist())
 ```
 
 ```
-{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
-added ids: [4]
-cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
+{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 1, 'null_keys': 0, 'cells_changed': 1} added ids: [4] removed ids: [1]
+cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
 A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra.
+`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
 
 ## Performance
 
@@ -206,7 +204,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions, including which types a `dict` key may be); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date`/`time`/`timedelta` comes back as a real one of those — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
+- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -252,6 +250,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
 - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/README.md

@@ -38,7 +38,7 @@ From source:
 ```sh
 cd crates/onix-py
 uv tool install maturin              # the build tool (skip if already installed)
-uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb for the table-diff tests
+uv sync --group test                 # creates .venv, installs pytest, pinned deepdiff, and pyarrow/polars/duckdb/pandas for the table-diff tests
 uv run --group test maturin develop --release
 ```
 
@@ -100,15 +100,15 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
 
 `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
 
-It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
+It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell, carrying the key columns, `column`, `old_value`/`new_value`, and `change` (`became_null`/`became_non_null`, `type_changed`, or `value_changed`), ordered by the canonical string rendering of the key columns (nulls first), then left-schema column order — the exact rendering and change-classification rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
 
 ```python
 import pyarrow as pa
 from deepdiff_rs import diff_tables
 
 left = pa.table({
-    "id": pa.array([1, 2, 3], pa.int64()),
-    "amount": pa.array([10, 20, 30], pa.int32()),
+    "id": pa.array([1, 2, 3, 9, 9], pa.int64()),  # 9 is a duplicate key
+    "amount": pa.array([10, 20, 30, 90, 91], pa.int32()),
 })
 right = pa.table({
     "id": pa.array([2, 3, 4], pa.int64()),
@@ -117,22 +117,20 @@ right = pa.table({
 })
 
 diff = diff_tables(left, right, key=["id"])
-print(diff.summary())
-print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
-print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
+print(diff.summary(), "added ids:", pa.table(diff.rows_added()).column("id").to_pylist(), "removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
+print("cells changed:", pa.table(diff.cells_changed()).to_pylist(), "duplicate keys:", pa.table(diff.duplicate_keys()).to_pylist())
 ```
 
 ```
-{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
-added ids: [4]
-cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
+{'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 1, 'null_keys': 0, 'cells_changed': 1} added ids: [4] removed ids: [1]
+cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}] duplicate keys: [{'id': 9, 'left_count': 2, 'right_count': 0}]
 ```
 
 A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 
-Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
+Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` consumes it with no pyarrow needed, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`; `pandas.api.interchange.from_dataframe(diff.schema_arrow)` also works, but pandas' own implementation of that protocol needs pyarrow installed regardless of which path you take.
 
-`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra.
+`pyarrow` is optional: install it with `pip install deepdiff-rs[arrow]`. It is needed only for `to_pyarrow()` and for passing pyarrow objects in — importing `deepdiff_rs` and diffing polars or DuckDB tables need it not at all. Passing an object that implements neither Arrow protocol raises `TypeError`; calling `to_pyarrow()` without pyarrow installed raises `ImportError` naming the extra. `diff.to_json()` gives the whole diff — schema, summary, and `rows_added`/`rows_removed`/`cells_changed`/`duplicate_keys` in full, one JSON object per row — as a single string with no pyarrow, polars, or pandas needed at all; see [Known limitations](#known-limitations) for its row cap.
 
 ## Performance
 
@@ -188,7 +186,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
 - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions, including which types a `dict` key may be); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date`/`time`/`timedelta` comes back as a real one of those — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
 - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
 - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
-- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
+- `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the full diff (schema, summary, and every row-level member, capped at 10,000 embedded rows); `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
 
 **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
 
@@ -234,6 +232,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
 - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
 

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/crates/onix-arrow/src/error.rs

@@ -121,6 +121,25 @@ pub enum TableDiffError {
         /// The column whose two renderings were equal.
         column: String,
     },
+    /// [`crate::TableDiff::to_json`] would embed more row objects than
+    /// [`crate::MAX_JSON_ROWS`] allows (see its own doc for what that caps
+    /// and why). Use the Arrow-returning members instead, or export the
+    /// batches directly, for a diff this large.
+    TooManyJsonRows {
+        /// The number of row objects `to_json()` would have embedded.
+        rows: usize,
+        /// The cap that was exceeded.
+        max: usize,
+    },
+    /// [`crate::TableDiff::to_json`]'s `serde_json` serialization failed.
+    /// Its input is a fixed set of already-rendered strings, numbers, and
+    /// nested objects/arrays, so this does not happen in practice; it is a
+    /// typed error rather than a panic because a public API must return,
+    /// not abort, on an unexpected failure.
+    Json {
+        /// The underlying `serde_json` error's message.
+        message: String,
+    },
 }
 
 impl fmt::Display for TableDiffError {
@@ -170,6 +189,15 @@ impl fmt::Display for TableDiffError {
                 "internal invariant: a value change in column {column:?} rendered identically \
                  on both sides, which the common-form rendering is designed to prevent"
             ),
+            TableDiffError::TooManyJsonRows { rows, max } => write!(
+                f,
+                "to_json() would embed {rows} row objects, more than the {max}-row cap; use \
+                 the Arrow-returning members (rows_added(), rows_removed(), cells_changed(), \
+                 duplicate_keys()) or export the batches directly for a diff this large"
+            ),
+            TableDiffError::Json { message } => {
+                write!(f, "failed to serialize the table diff to JSON: {message}")
+            }
         }
     }
 }
@@ -278,6 +306,27 @@ mod tests {
         assert!(message.contains("changed-row count"));
     }
 
+    #[test]
+    fn too_many_json_rows_message_names_count_and_cap() {
+        let error = TableDiffError::TooManyJsonRows {
+            rows: 20_000,
+            max: 10_000,
+        };
+        let message = error.to_string();
+        assert!(message.contains("20000"));
+        assert!(message.contains("10000-row cap"));
+        assert!(message.contains("rows_added()"));
+    }
+
+    #[test]
+    fn json_message_names_the_underlying_error() {
+        let error = TableDiffError::Json {
+            message: "unexpected end of input".to_string(),
+        };
+        let message = error.to_string();
+        assert!(message.contains("unexpected end of input"));
+    }
+
     #[test]
     fn error_implements_std_error() {
         let error = TableDiffError::EmptyKey;

deepdiff_rs-0.9.0/crates/onix-arrow/src/json_rows.rs

@@ -0,0 +1,141 @@
+//! Renders an Arrow [`RecordBatch`] to a JSON array of row objects, for
+//! [`crate::TableDiff::to_json`]. Reuses [`crate::row_diff::SideRenderer`],
+//! the same renderer [`crate::TableDiff::cells_changed`]'s `old_value`/
+//! `new_value` use, so no cell renders two different ways in the crate.
+//!
+//! [`MAX_JSON_ROWS`] bounds the row count: the JSON holds one object per row
+//! and one owned string per non-null cell, so memory is proportional to
+//! rows × columns × cell width, uncapped past that row count otherwise. A
+//! batch over the cap is refused with [`crate::TableDiffError::TooManyJsonRows`].
+
+use arrow_array::RecordBatch;
+use arrow_cast::display::FormatOptions;
+use serde_json::{Map, Value as JsonValue};
+
+use crate::error::TableDiffError;
+use crate::row_diff::{SideRenderer, cell_is_null};
+
+/// The maximum number of row objects this module will render for one
+/// [`RecordBatch`]; see the module doc for what this bounds.
+pub const MAX_JSON_ROWS: usize = 10_000;
+
+/// Renders every row of `batch` to a JSON object keyed by column name, or
+/// [`TableDiffError::TooManyJsonRows`] if `batch` has more than
+/// [`MAX_JSON_ROWS`] rows. A null cell renders as JSON `null`; every other
+/// cell renders through [`SideRenderer`], the same renderer
+/// [`crate::TableDiff::cells_changed`] uses.
+///
+/// # Errors
+///
+/// - [`TableDiffError::TooManyJsonRows`] if `batch.num_rows()` exceeds
+///   [`MAX_JSON_ROWS`].
+/// - [`TableDiffError::Render`] if a cell cannot be rendered to its
+///   canonical string (naming the column), the same failure
+///   [`crate::TableDiff::cells_changed`] can raise.
+pub(crate) fn rows_to_json(batch: &RecordBatch) -> Result<Vec<JsonValue>, TableDiffError> {
+    if batch.num_rows() > MAX_JSON_ROWS {
+        return Err(TableDiffError::TooManyJsonRows {
+            rows: batch.num_rows(),
+            max: MAX_JSON_ROWS,
+        });
+    }
+
+    let format_options = FormatOptions::default();
+    let schema = batch.schema();
+    let columns = schema
+        .fields()
+        .iter()
+        .zip(batch.columns())
+        .map(|(field, array)| {
+            SideRenderer::new(array, &format_options)
+                .map(|renderer| (field.name(), array, renderer))
+        })
+        .collect::<Result<Vec<_>, _>>()?;
+
+    let mut rows = Vec::with_capacity(batch.num_rows());
+    for row in 0..batch.num_rows() {
+        let mut object = Map::with_capacity(columns.len());
+        for (name, array, renderer) in &columns {
+            let value = if cell_is_null(array, row) {
+                JsonValue::Null
+            } else {
+                JsonValue::String(renderer.render(row, name)?)
+            };
+            object.insert((*name).clone(), value);
+        }
+        rows.push(JsonValue::Object(object));
+    }
+
+    Ok(rows)
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use arrow_array::{Int64Array, RecordBatch, StringArray};
+    use arrow_schema::{DataType, Field, Schema};
+
+    use super::{MAX_JSON_ROWS, rows_to_json};
+    use crate::error::TableDiffError;
+
+    fn batch(rows: usize) -> RecordBatch {
+        let schema = Arc::new(Schema::new(vec![
+            Field::new("id", DataType::Int64, false),
+            Field::new("name", DataType::Utf8, true),
+        ]));
+        let ids: Vec<i64> = (0..rows).map(|i| i64::try_from(i).unwrap()).collect();
+        let names: Vec<Option<String>> = (0..rows)
+            .map(|i| {
+                if i % 2 == 0 {
+                    None
+                } else {
+                    Some(format!("n{i}"))
+                }
+            })
+            .collect();
+        RecordBatch::try_new(
+            schema,
+            vec![
+                Arc::new(Int64Array::from(ids)),
+                Arc::new(StringArray::from(names)),
+            ],
+        )
+        .unwrap()
+    }
+
+    #[test]
+    fn renders_one_object_per_row_with_column_keys() {
+        let rows = rows_to_json(&batch(2)).unwrap();
+        assert_eq!(rows.len(), 2);
+        assert_eq!(rows[0]["id"], serde_json::json!("0"));
+        assert_eq!(rows[0]["name"], serde_json::Value::Null);
+        assert_eq!(rows[1]["id"], serde_json::json!("1"));
+        assert_eq!(rows[1]["name"], serde_json::json!("n1"));
+    }
+
+    #[test]
+    fn empty_batch_renders_no_rows() {
+        assert_eq!(
+            rows_to_json(&batch(0)).unwrap(),
+            Vec::<serde_json::Value>::new()
+        );
+    }
+
+    #[test]
+    fn batch_over_the_cap_is_refused() {
+        let error = rows_to_json(&batch(MAX_JSON_ROWS + 1)).unwrap_err();
+        assert_eq!(
+            error,
+            TableDiffError::TooManyJsonRows {
+                rows: MAX_JSON_ROWS + 1,
+                max: MAX_JSON_ROWS,
+            }
+        );
+    }
+
+    #[test]
+    fn batch_at_the_cap_is_accepted() {
+        assert!(rows_to_json(&batch(MAX_JSON_ROWS)).is_ok());
+    }
+}

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/crates/onix-arrow/src/lib.rs

@@ -102,12 +102,14 @@
 //! ```
 
 mod error;
+mod json_rows;
 mod options;
 mod row_diff;
 mod schema;
 mod table_diff;
 
 pub use error::{Side, TableDiffError};
+pub use json_rows::MAX_JSON_ROWS;
 pub use options::TableDiffOptions;
 pub use row_diff::{MemoryInput, TableInput};
 pub use schema::{ChangeKind, SchemaChange, diff_schemas};

{deepdiff_rs-0.8.2 → deepdiff_rs-0.9.0}/crates/onix-arrow/src/row_diff.rs

@@ -66,11 +66,13 @@
 //!
 //! # Per-cell changes
 //!
-//! [`diff_cells`] reports, for every changed row, which cells differ. A cell is
-//! reported changed **if and only if its [`hash_cell`] contribution differs**
-//! between the two matched rows — the same helper the row hash is built from, so
-//! the cell list and the row-changed decision can never drift. Each reported
-//! cell is labelled:
+//! [`diff_cells`] reports, for every changed row, which cells differ, as one
+//! output row per differing cell: the key columns, then `column`, `old_value`,
+//! `new_value`, and `change` (see [`diff_cells`] for the exact output order).
+//! A cell is reported changed **if and only if its [`hash_cell`] contribution
+//! differs** between the two matched rows — the same helper the row hash is
+//! built from, so the cell list and the row-changed decision can never drift.
+//! Each reported cell is labelled:
 //!
 //! - `became_null`/`became_non_null` when exactly one side is null;
 //! - `type_changed` when both are non-null and the two sides' types are not
@@ -93,11 +95,14 @@
 //! `0.10000000149011612` against an `f64` `0.1`), a timestamp renders as its UTC
 //! instant with its zone appended when aware (so an aware and a naive timestamp
 //! of the same instant differ), a decimal renders at its native scale, a string
-//! verbatim (decimals and strings match the `DuckDB` oracle), and a duration
+//! verbatim (decimals and strings match the `DuckDB` oracle), a duration
 //! renders as an ISO 8601 `PT<seconds>S` string computed from its value — never
 //! through the Arrow formatter, whose second/millisecond duration formatter can
-//! emit a `<invalid>` sentinel while still succeeding. As a construction guard,
-//! a `value_changed` record whose two renderings are nonetheless equal is a
+//! emit a `<invalid>` sentinel while still succeeding — and a cross-variant
+//! interval renders with its variant appended, so two variants whose human
+//! form would otherwise coincide stay distinct (see [`prepare_render`]). As a
+//! construction guard, a `value_changed` record whose two renderings are
+//! nonetheless equal is a
 //! [`TableDiffError::EqualRenderings`], not a silent row. There is no typed
 //! old/new
 //! column: a long-format table mixes every compared column's type in one column,
@@ -1287,7 +1292,7 @@ fn value_domain(data_type: &DataType) -> ValueDomain {
 /// diff both use. A `Null`-typed column is all-null by definition and carries no
 /// validity buffer, so it is detected by type; every other column defers to its
 /// validity bitmap.
-fn cell_is_null(array: &ArrayRef, row: usize) -> bool {
+pub(crate) fn cell_is_null(array: &ArrayRef, row: usize) -> bool {
     matches!(array.data_type(), DataType::Null) || array.is_null(row)
 }
 
@@ -1351,13 +1356,16 @@ fn render_duration(raw: i64, unit: TimeUnit) -> String {
 /// One side's cell renderer: the Arrow formatter for most types, or a
 /// dependency-free [`render_duration`] for a `Duration` column (which the
 /// formatter can render as a `<invalid>` sentinel).
-enum SideRenderer<'a> {
+pub(crate) enum SideRenderer<'a> {
     Formatter(ArrayFormatter<'a>),
     Duration(&'a ArrayRef, TimeUnit),
 }
 
 impl<'a> SideRenderer<'a> {
-    fn new(array: &'a ArrayRef, opts: &'a FormatOptions<'a>) -> Result<Self, TableDiffError> {
+    pub(crate) fn new(
+        array: &'a ArrayRef,
+        opts: &'a FormatOptions<'a>,
+    ) -> Result<Self, TableDiffError> {
         if let DataType::Duration(unit) = array.data_type() {
             Ok(SideRenderer::Duration(array, *unit))
         } else {
@@ -1366,7 +1374,7 @@ impl<'a> SideRenderer<'a> {
         }
     }
 
-    fn render(&self, row: usize, column: &str) -> Result<String, TableDiffError> {
+    pub(crate) fn render(&self, row: usize, column: &str) -> Result<String, TableDiffError> {
         match self {
             SideRenderer::Formatter(formatter) => render_value(formatter, row, column),
             SideRenderer::Duration(array, unit) => {