deepdiff-rs 0.7.1__tar.gz → 0.8.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/Cargo.lock +4 -4
  2. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/Cargo.toml +1 -1
  3. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/PKG-INFO +9 -9
  4. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/README.md +8 -8
  5. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/Cargo.toml +7 -4
  6. deepdiff_rs-0.8.0/crates/onix-arrow/examples/row_diff_rss.rs +252 -0
  7. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/error.rs +67 -18
  8. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/lib.rs +18 -7
  9. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/row_diff.rs +1776 -207
  10. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/table_diff.rs +34 -31
  11. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/Cargo.toml +1 -1
  12. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/arrow.rs +16 -13
  13. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_table_diff.py +5 -6
  14. deepdiff_rs-0.8.0/crates/onix-py/tests/test_table_row_diff.py +722 -0
  15. deepdiff_rs-0.7.1/crates/onix-arrow/examples/row_diff_rss.rs +0 -189
  16. deepdiff_rs-0.7.1/crates/onix-py/tests/test_table_row_diff.py +0 -368
  17. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/examples/type_stack_cost.rs +0 -0
  18. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/options.rs +0 -0
  19. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-arrow/src/schema.rs +0 -0
  20. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/Cargo.toml +0 -0
  21. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/examples/stack_frame_cost.rs +0 -0
  22. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/datetime.rs +0 -0
  23. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/datetime_tests.rs +0 -0
  24. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/array.rs +0 -0
  25. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/dispatch.rs +0 -0
  26. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/mod.rs +0 -0
  27. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/object.rs +0 -0
  28. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/options.rs +0 -0
  29. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/scalar.rs +0 -0
  30. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/set.rs +0 -0
  31. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/diff/tests.rs +0 -0
  32. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/error.rs +0 -0
  33. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/distance.rs +0 -0
  34. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/fxhash.rs +0 -0
  35. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/hash.rs +0 -0
  36. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/memo.rs +0 -0
  37. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/mod.rs +0 -0
  38. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/pairing.rs +0 -0
  39. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/ignore_order/tests.rs +0 -0
  40. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/lcs.rs +0 -0
  41. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/lcs_tests.rs +0 -0
  42. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/lib.rs +0 -0
  43. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/path.rs +0 -0
  44. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/report.rs +0 -0
  45. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/report_tests.rs +0 -0
  46. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/test_support.rs +0 -0
  47. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/unified_diff.rs +0 -0
  48. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/unified_diff_tests.rs +0 -0
  49. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/value.rs +0 -0
  50. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/src/value_tests.rs +0 -0
  51. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/golden.rs +0 -0
  52. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/ignore_order_memory.rs +0 -0
  53. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/memory_footprint.rs +0 -0
  54. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/proptest_diff.rs +0 -0
  55. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-core/tests/proptest_ignore_order.rs +0 -0
  56. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/.python-version +0 -0
  57. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/benchmarks/bench_bindings.py +0 -0
  58. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/convert.rs +0 -0
  59. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/deepdiff.rs +0 -0
  60. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/errors.rs +0 -0
  61. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/fast_path.rs +0 -0
  62. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/guard.rs +0 -0
  63. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/src/lib.rs +0 -0
  64. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_bindings_memory.py +0 -0
  65. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_conversions.py +0 -0
  66. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_datetimes.py +0 -0
  67. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_depth_guard.py +0 -0
  68. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_differential_fuzz.py +0 -0
  69. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_golden_parity.py +0 -0
  70. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_sets.py +0 -0
  71. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_signed_zero.py +0 -0
  72. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_smoke.py +0 -0
  73. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_suite_hygiene.py +0 -0
  74. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/crates/onix-py/tests/test_tuples.py +0 -0
  75. {deepdiff_rs-0.7.1 → deepdiff_rs-0.8.0}/pyproject.toml +0 -0
@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
629
629
 
630
630
  [[package]]
631
631
  name = "onix-arrow"
632
- version = "0.7.1"
632
+ version = "0.8.0"
633
633
  dependencies = [
634
634
  "arrow-array",
635
635
  "arrow-buffer",
@@ -646,7 +646,7 @@ dependencies = [
646
646
 
647
647
  [[package]]
648
648
  name = "onix-cli"
649
- version = "0.7.1"
649
+ version = "0.8.0"
650
650
  dependencies = [
651
651
  "onix-core",
652
652
  "serde_json",
@@ -654,7 +654,7 @@ dependencies = [
654
654
 
655
655
  [[package]]
656
656
  name = "onix-core"
657
- version = "0.7.1"
657
+ version = "0.8.0"
658
658
  dependencies = [
659
659
  "proptest",
660
660
  "serde",
@@ -665,7 +665,7 @@ dependencies = [
665
665
 
666
666
  [[package]]
667
667
  name = "onix-py"
668
- version = "0.7.1"
668
+ version = "0.8.0"
669
669
  dependencies = [
670
670
  "arrow-array",
671
671
  "arrow-ipc",
@@ -3,7 +3,7 @@ resolver = "3"
3
3
  members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
4
4
 
5
5
  [workspace.package]
6
- version = "0.7.1"
6
+ version = "0.8.0"
7
7
  edition = "2024"
8
8
  license = "MIT"
9
9
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: deepdiff-rs
3
- Version: 0.7.1
3
+ Version: 0.8.0
4
4
  Classifier: Programming Language :: Python :: 3
5
5
  Classifier: Programming Language :: Rust
6
6
  Classifier: License :: OSI Approved :: MIT License
@@ -118,7 +118,7 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
118
118
 
119
119
  `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
120
120
 
121
- It reports the **schema** diff (which columns were added, removed, or changed type) and the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated). Rows are matched by the key columns; `rows_added`, `rows_removed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome. The per-cell diff (`cells_changed`) arrives in a later version and raises `NotImplementedError` until then.
121
+ It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
122
122
 
123
123
  ```python
124
124
  import pyarrow as pa
@@ -137,16 +137,16 @@ right = pa.table({
137
137
  diff = diff_tables(left, right, key=["id"])
138
138
  print(diff.summary())
139
139
  print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
140
- print("removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
140
+ print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
141
141
  ```
142
142
 
143
143
  ```
144
- {'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0}
144
+ {'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
145
145
  added ids: [4]
146
- removed ids: [1]
146
+ cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
147
147
  ```
148
148
 
149
- A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, `1.00` equals `1.0000`, a timestamp compares by its instant across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
149
+ A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
150
150
 
151
151
  Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
152
152
 
@@ -168,8 +168,8 @@ Type comparison uses the full logical Arrow type (timestamp unit and timezone, d
168
168
  - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
169
169
  - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
170
170
  - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
171
- - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; and temp disk, because each input is read twice and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: bound the row count and, for duplicate-heavy data, the key width of untrusted input. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
172
- - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced).
171
+ - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
172
+ - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
173
173
  - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
174
174
 
175
175
  ## Performance
@@ -226,7 +226,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
226
226
  - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict` (with `str` keys), `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, and `datetime.date` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date` comes back as a real `datetime`/`date` — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
227
227
  - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
228
228
  - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
229
- - `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, and `.duplicate_keys()` return Arrow tables, and `.cells_changed()` raises `NotImplementedError` until a later version.
229
+ - `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
230
230
 
231
231
  **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
232
232
 
@@ -100,7 +100,7 @@ Pass `--ignore-order` to compare every list by value instead of by position, mir
100
100
 
101
101
  `diff_tables` compares two tables the way `DeepDiff` compares two objects. It takes any object implementing the [Arrow PyCapsule interface](https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html) — a pyarrow `Table` or `RecordBatch`, a polars `DataFrame`, a DuckDB relation — and imports it with no Python round trip. The two tables are matched on a required, non-empty set of key columns (the table's primary key).
102
102
 
103
- It reports the **schema** diff (which columns were added, removed, or changed type) and the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated). Rows are matched by the key columns; `rows_added`, `rows_removed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome. The per-cell diff (`cells_changed`) arrives in a later version and raises `NotImplementedError` until then.
103
+ It reports the **schema** diff (which columns were added, removed, or changed type), the keyed **row** diff (which rows were added, removed, or changed, and which keys are duplicated), and the per-cell diff (`cells_changed`): one row per changed cell of a changed row, carrying the key columns, the `column`, its `old_value`/`new_value` in a canonical string rendering (a decimal at its native scale and a string verbatim, both matching DuckDB; a timestamp as its UTC instant, keeping its zone when aware; a cross-variant interval with its variant appended; a duration as ISO 8601 `PT<seconds>S`, never through the Arrow formatter (which can emit a `<invalid>` sentinel); numbers of differing width at the wider type, so an `f32` `0.1` shows as `0.10000000149011612` against an `f64` `0.1` — the exact rules are in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs)), and a `change`: `became_null`/`became_non_null` for a one-sided null, `type_changed` when the two types are not losslessly comparable (different value kinds, an aware-versus-naive timestamp, or a cross-variant interval), otherwise `value_changed` — a lossless type difference (integer or float widening, a time/duration unit change, a decimal scale change) is `value_changed` only when the value truly differs, never merely for the type. Its rows are ordered by the canonical string rendering of the key columns (so numeric keys sort as strings — `10` before `2` — nulls first), then left-schema column order. Rows are matched by the key columns; `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` return Arrow tables, and `summary()` counts each outcome.
104
104
 
105
105
  ```python
106
106
  import pyarrow as pa
@@ -119,16 +119,16 @@ right = pa.table({
119
119
  diff = diff_tables(left, right, key=["id"])
120
120
  print(diff.summary())
121
121
  print("added ids:", pa.table(diff.rows_added()).column("id").to_pylist())
122
- print("removed ids:", pa.table(diff.rows_removed()).column("id").to_pylist())
122
+ print("cells changed:", pa.table(diff.cells_changed()).to_pylist())
123
123
  ```
124
124
 
125
125
  ```
126
- {'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0}
126
+ {'columns_added': 1, 'columns_removed': 0, 'columns_type_changed': 1, 'rows_added': 1, 'rows_removed': 1, 'rows_changed': 1, 'duplicate_keys': 0, 'null_keys': 0, 'cells_changed': 1}
127
127
  added ids: [4]
128
- removed ids: [1]
128
+ cells changed: [{'id': 3, 'column': 'amount', 'old_value': '30', 'new_value': '31', 'change': 'value_changed'}]
129
129
  ```
130
130
 
131
- A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, `1.00` equals `1.0000`, a timestamp compares by its instant across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
131
+ A key appearing more than once on either side is reported in `duplicate_keys` (with `left_count` and `right_count`) and excluded from the added/removed/changed sets; a null key matches its counterpart and is counted in `null_keys`. Rows are compared by the non-key columns present on *both* sides, with onix's value semantics (integers and integral floats fold together, all NaNs compare equal, `1.00` equals `1.0000`, a timestamp compares by its instant and a time or duration by its value across units, dictionary-encoded values equal their plain form, and null equals null); a nested non-key column is out of scope and is skipped rather than compared. The exact value-comparison rules are documented on the hashing functions in [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
132
132
 
133
133
  Type comparison uses the full logical Arrow type (timestamp unit and timezone, decimal precision and scale, and so on), but physical encodings that carry the same logical type compare equal — a dictionary-encoded string equals a plain string, polars' `Utf8View` equals pyarrow's `Utf8`, the list variants normalize together, and a map compares equal however a library spells it — so the same table read through pyarrow, polars, or DuckDB reports no spurious type changes. The full normalization rules are documented on `normalized_type` (and `map_entries`) in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs); nullability is ignored but reported in each record. Column names must be unique on each side; a repeated name raises `ValueError`. `diff.schema_arrow` is the same result as an Arrow table: it implements `__arrow_c_stream__`, so `polars.DataFrame(diff.schema_arrow)` and `pandas` consume it directly, and `diff.schema_arrow.to_pyarrow()` returns a `pyarrow.Table`.
134
134
 
@@ -150,8 +150,8 @@ Type comparison uses the full logical Arrow type (timestamp unit and timezone, d
150
150
  - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
151
151
  - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
152
152
  - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
153
- - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; and temp disk, because each input is read twice and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: bound the row count and, for duplicate-heavy data, the key width of untrusted input. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
154
- - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced).
153
+ - `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M) and time is `N·log N` (about 2 s for a 10M-row pair), plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, and the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns). Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss`, single run, macOS on an Apple M-series laptop, 2026-09-05, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
154
+ - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
155
155
  - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.
156
156
 
157
157
  ## Performance
@@ -208,7 +208,7 @@ Both reports carry their full methodology, fairness rules, and the reproduce com
208
208
  - `DeepDiff(t1, t2, ignore_order=False, max_depth=None)`: diffs two live Python objects of supported value types — `None`, `bool`, `int`, `float`, `str`, `dict` (with `str` keys), `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, and `datetime.date` (see [Known limitations](#known-limitations) for the exact restrictions and exclusions); `.to_json()` returns the DeepDiff-compatible JSON string, `.to_dict()` the same report as a dict — with Python types preserved, so a value the diff found in a `tuple`, `set` or `frozenset` comes back as one and a `datetime`/`date` comes back as a real `datetime`/`date` — and the instance is falsy when there is no difference. The `set_item_added`/`set_item_removed` categories are lists of path strings, each ending in the item itself (`root['a'][2]`, `root['x']`, `root[(1, 2)]`).
209
209
  - `diff_json(a, b, ignore_order=False, max_depth=None) -> str`: diffs two JSON strings entirely in Rust and returns the report as a JSON string.
210
210
  - `MaxDepthError` (a `ValueError` subclass) is raised when input exceeds `max_depth`; `MAX_DEPTH_CEILING` (20,000) is the hard upper bound on `max_depth`.
211
- - `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, and `.duplicate_keys()` return Arrow tables, and `.cells_changed()` raises `NotImplementedError` until a later version.
211
+ - `diff_tables(left, right, key=[...]) -> TableDiff`: diffs two Arrow tables (see [Diffing tables](#diffing-tables)). `TableDiff.schema` is the list of changed columns, `.schema_arrow` the same as an Arrow table, `.summary()` the schema and row change counts, `.to_json()` the schema diff as JSON; `.rows_added()`, `.rows_removed()`, `.cells_changed()`, and `.duplicate_keys()` return Arrow tables.
212
212
 
213
213
  **CLI.** `onix diff <a.json> <b.json> [--max-depth N] [--ignore-order] [--timing]` reads both files as JSON and prints a compact, single-line DeepDiff-compatible report to stdout (`{}` when there is no difference).
214
214
 
@@ -22,10 +22,13 @@ arrow-array = "=59.3.0"
22
22
  # `i256` for hashing `Decimal256` cells the same way as `Decimal128`.
23
23
  arrow-buffer = "=59.3.0"
24
24
  # `arrow-cast` decodes dictionary-encoded columns to their value type before
25
- # hashing, and casts a right-only duplicate key's columns to the left key
26
- # types; `arrow-select` filters and concatenates the output row batches. The
27
- # row diff re-reads each side through the caller's re-openable `TableInput`, so
28
- # no IPC spooling lives here — the Python bindings own that.
25
+ # hashing, casts a right-only duplicate key's columns to the left key types,
26
+ # strips a timestamp's timezone before rendering, and renders each changed
27
+ # cell's value to its canonical string (`display::ArrayFormatter`);
28
+ # `arrow-select` filters the materialized rows, concatenates the output row
29
+ # batches, and `take`s the key columns into the per-cell diff. The row diff
30
+ # re-reads each side through the caller's re-openable `TableInput`, so no IPC
31
+ # spooling lives here — the Python bindings own that.
29
32
  arrow-cast = "=59.3.0"
30
33
  arrow-select = "=59.3.0"
31
34
  # Row identity is a keyed 128-bit SipHash-1-3 (`siphasher`), keyed from 16
@@ -0,0 +1,252 @@
1
+ //! Measures the keyed row diff's peak memory and wall time, to check the memory
2
+ //! bounds the README states.
3
+ //!
4
+ //! Run under the OS's max-RSS reporter:
5
+ //!
6
+ //! ```sh
7
+ //! cargo build -p onix-arrow --release --example row_diff_rss
8
+ //! # linear shape (default): mostly-matching rows, 1% added/removed, ~2% changed
9
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 1000000
10
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 10000000
11
+ //! # same shape with no changed rows: the cell pass materializes nothing, the
12
+ //! # pass-one baseline the ~2%-changed run is measured against
13
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 nochange
14
+ //! # every row changed (narrow int cells): the cell pass at full width
15
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 allchange
16
+ //! # every row changed with a wide (1 KB) string cell: the rendering worst case
17
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 100000 wide 1024
18
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 200000 wide 1024
19
+ //! # duplicate-heavy shape: every key duplicated, wide string key
20
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 1000000 dup 16
21
+ //! /usr/bin/time -l target/release/examples/row_diff_rss 200000 dup 1024
22
+ //! ```
23
+ //!
24
+ //! Each side is generated on the fly, batch by batch, and nothing is retained
25
+ //! between batches, so the process's peak RSS is the diff's own state, not the
26
+ //! table data. The **linear** shape (`id`, `value` int64 columns) exercises the
27
+ //! per-row hash vectors: the left is ids `0..n`, the right `step..n + step` with
28
+ //! `step = n / 100`, so 1% removed, 1% added, ~2% changed — and the cell pass
29
+ //! materializes those ~2% changed rows on both sides. The **nochange** variant
30
+ //! keeps the 1% added/removed but makes every shared row equal, so the cell pass
31
+ //! materializes nothing: the difference in peak RSS between it and the default
32
+ //! run is the cell pass's cost. The **allchange** variant drops the offset and
33
+ //! changes every shared row (no added/removed), so the cell pass materializes
34
+ //! and renders every row. The **wide** shape (`id` int64, `value` a
35
+ //! `value_width`-byte Utf8 that differs between the sides) changes every row too
36
+ //! and renders `value_width` bytes per changed cell — the rendering worst case,
37
+ //! whose peak RSS scales with changed cells times cell width. The **dup** shape
38
+ //! (`key` Utf8 of the given width, `value` int64) makes every key appear twice
39
+ //! on each side, so every distinct
40
+ //! key is a duplicate and the whole `duplicate_keys` report is materialized —
41
+ //! the term that scales with distinct duplicated keys times the key width.
42
+
43
+ use std::sync::Arc;
44
+
45
+ use arrow_array::{ArrayRef, Int64Array, RecordBatch, RecordBatchReader, StringArray};
46
+ use arrow_schema::{ArrowError, DataType, Field, Schema, SchemaRef};
47
+ use onix_arrow::{TableDiffError, TableDiffOptions, TableInput, diff_tables};
48
+
49
+ const BATCH: i64 = 65_536;
50
+
51
+ /// The generated table shape.
52
+ #[derive(Clone, Copy)]
53
+ enum Shape {
54
+ /// `(id, value)` int64 columns; `id_offset` shifts the key range and
55
+ /// `change_every` perturbs a fraction of values.
56
+ Linear { id_offset: i64, change_every: i64 },
57
+ /// `(key, value)`; `key` is a `key_width`-byte string and each key value
58
+ /// appears twice, so every key is a duplicate.
59
+ Dup { key_width: usize },
60
+ /// `(id, value)` where `value` is a `value_width`-byte string filled with
61
+ /// `fill`; the two sides share every id but differ in `fill`, so every row
62
+ /// is changed and every changed cell renders `value_width` bytes — the
63
+ /// wide-cell worst case for the per-cell diff's rendering memory.
64
+ Wide { value_width: usize, fill: u8 },
65
+ }
66
+
67
+ /// A table generated on demand, retaining nothing between batches.
68
+ struct Generated {
69
+ schema: SchemaRef,
70
+ rows: i64,
71
+ shape: Shape,
72
+ }
73
+
74
+ impl TableInput for Generated {
75
+ fn schema(&self) -> SchemaRef {
76
+ self.schema.clone()
77
+ }
78
+
79
+ fn open(&self) -> Result<Box<dyn RecordBatchReader + Send>, TableDiffError> {
80
+ Ok(Box::new(GenReader {
81
+ schema: self.schema.clone(),
82
+ rows: self.rows,
83
+ shape: self.shape,
84
+ next: 0,
85
+ }))
86
+ }
87
+ }
88
+
89
+ struct GenReader {
90
+ schema: SchemaRef,
91
+ rows: i64,
92
+ shape: Shape,
93
+ next: i64,
94
+ }
95
+
96
+ impl Iterator for GenReader {
97
+ type Item = Result<RecordBatch, ArrowError>;
98
+
99
+ fn next(&mut self) -> Option<Self::Item> {
100
+ if self.next >= self.rows {
101
+ return None;
102
+ }
103
+ let end = (self.next + BATCH).min(self.rows);
104
+ let columns: Vec<ArrayRef> = match self.shape {
105
+ Shape::Linear {
106
+ id_offset,
107
+ change_every,
108
+ } => {
109
+ let ids: Int64Array = (self.next..end).map(|i| Some(i + id_offset)).collect();
110
+ // The value is a function of the id, so a shared key holds the
111
+ // same value on both sides except every `change_every`-th id.
112
+ let values: Int64Array = (self.next..end)
113
+ .map(|i| {
114
+ let id = i + id_offset;
115
+ Some(if id % change_every == 0 { id + 1 } else { id })
116
+ })
117
+ .collect();
118
+ vec![Arc::new(ids), Arc::new(values)]
119
+ }
120
+ Shape::Dup { key_width } => {
121
+ // Key value `i / 2`, so each distinct key appears twice.
122
+ let keys: StringArray = (self.next..end)
123
+ .map(|i| Some(format!("{:0>width$}", i / 2, width = key_width)))
124
+ .collect();
125
+ let values: Int64Array = (self.next..end).map(Some).collect();
126
+ vec![Arc::new(keys), Arc::new(values)]
127
+ }
128
+ Shape::Wide { value_width, fill } => {
129
+ let ids: Int64Array = (self.next..end).map(Some).collect();
130
+ let cell = String::from_utf8(vec![fill; value_width]).unwrap();
131
+ let values: StringArray = (self.next..end).map(|_| Some(cell.as_str())).collect();
132
+ vec![Arc::new(ids), Arc::new(values)]
133
+ }
134
+ };
135
+ self.next = end;
136
+
137
+ Some(RecordBatch::try_new(self.schema.clone(), columns))
138
+ }
139
+ }
140
+
141
+ impl RecordBatchReader for GenReader {
142
+ fn schema(&self) -> SchemaRef {
143
+ self.schema.clone()
144
+ }
145
+ }
146
+
147
+ fn main() {
148
+ let args: Vec<String> = std::env::args().collect();
149
+ let rows: i64 = args
150
+ .get(1)
151
+ .and_then(|a| a.parse().ok())
152
+ .unwrap_or(1_000_000);
153
+ // Mode selects the shape: "dup" (duplicate-heavy), "wide" (all rows changed,
154
+ // wide string value column), "nochange"/"allchange"/default (linear int).
155
+ let mode = args.get(2).map_or("", String::as_str);
156
+ let width: usize = args
157
+ .get(3)
158
+ .and_then(|a| a.parse().ok())
159
+ .unwrap_or(if mode == "wide" { 1024 } else { 16 });
160
+
161
+ let (schema, left_shape, right_shape, key, label) = match mode {
162
+ "dup" => {
163
+ let schema = Arc::new(Schema::new(vec![
164
+ Field::new("key", DataType::Utf8, false),
165
+ Field::new("value", DataType::Int64, false),
166
+ ]));
167
+ let shape = Shape::Dup { key_width: width };
168
+ (
169
+ schema,
170
+ shape,
171
+ shape,
172
+ "key",
173
+ format!(" (dup, key_width={width})"),
174
+ )
175
+ }
176
+ "wide" => {
177
+ let schema = Arc::new(Schema::new(vec![
178
+ Field::new("id", DataType::Int64, false),
179
+ Field::new("value", DataType::Utf8, false),
180
+ ]));
181
+ (
182
+ schema,
183
+ Shape::Wide {
184
+ value_width: width,
185
+ fill: b'a',
186
+ },
187
+ Shape::Wide {
188
+ value_width: width,
189
+ fill: b'b',
190
+ },
191
+ "id",
192
+ format!(" (wide, value_width={width}, all changed)"),
193
+ )
194
+ }
195
+ _ => {
196
+ let schema = Arc::new(Schema::new(vec![
197
+ Field::new("id", DataType::Int64, false),
198
+ Field::new("value", DataType::Int64, false),
199
+ ]));
200
+ let step = (rows / 100).max(1);
201
+ // `nochange`: 1% added/removed but no changed rows (the pass-one
202
+ // baseline). `allchange`: no added/removed, every row changed.
203
+ // Default: 1% added/removed, ~2% changed.
204
+ let (id_offset, change_every, label) = match mode {
205
+ "nochange" => (step, i64::MAX, " (nochange baseline)"),
206
+ "allchange" => (0, 1, " (all changed)"),
207
+ _ => (step, 50, ""),
208
+ };
209
+ (
210
+ schema,
211
+ Shape::Linear {
212
+ id_offset: 0,
213
+ change_every: i64::MAX,
214
+ },
215
+ Shape::Linear {
216
+ id_offset,
217
+ change_every,
218
+ },
219
+ "id",
220
+ label.to_string(),
221
+ )
222
+ }
223
+ };
224
+
225
+ let left = Generated {
226
+ schema: schema.clone(),
227
+ rows,
228
+ shape: left_shape,
229
+ };
230
+ let right = Generated {
231
+ schema,
232
+ rows,
233
+ shape: right_shape,
234
+ };
235
+
236
+ let start = std::time::Instant::now();
237
+ let diff = diff_tables(&left, &right, &TableDiffOptions::new(vec![key.to_string()]))
238
+ .expect("diff succeeds");
239
+ let elapsed = start.elapsed();
240
+ let summary = diff.summary();
241
+
242
+ println!("rows per side: {rows}{label}");
243
+ println!("wall: {:.2}s", elapsed.as_secs_f64());
244
+ println!(
245
+ "rows_added={} rows_removed={} rows_changed={} duplicate_keys={} cells_changed={}",
246
+ summary.rows_added,
247
+ summary.rows_removed,
248
+ summary.rows_changed,
249
+ summary.duplicate_keys,
250
+ summary.cells_changed
251
+ );
252
+ }
@@ -31,9 +31,8 @@ impl fmt::Display for Side {
31
31
 
32
32
  /// Errors that can occur while diffing two tables.
33
33
  ///
34
- /// Marked `#[non_exhaustive]` because the later row-diff slices add their own
35
- /// variants (streaming/read failures and the like); matching on it must keep
36
- /// a wildcard arm.
34
+ /// Marked `#[non_exhaustive]` so future work can add variants without a
35
+ /// breaking change; matching on it must keep a wildcard arm.
37
36
  #[derive(Debug, Clone, PartialEq, Eq)]
38
37
  #[non_exhaustive]
39
38
  pub enum TableDiffError {
@@ -95,13 +94,32 @@ pub enum TableDiffError {
95
94
  /// The underlying error's message.
96
95
  message: String,
97
96
  },
98
- /// A member of [`crate::TableDiff`] that a later version fills in was
99
- /// asked for before that version exists. The per-cell changes
100
- /// (`cells_changed`) arrive with a later version; `rows_added`,
101
- /// `rows_removed`, and `duplicate_keys` are available.
102
- NotImplemented {
103
- /// The name of the member that is not implemented yet.
104
- feature: &'static str,
97
+ /// A cell value could not be rendered to its canonical string for the
98
+ /// per-cell diff — for example a temporal value outside the range the
99
+ /// formatter can format. Reported as a typed error rather than written into
100
+ /// the output as error prose (which a real string cell could not be told
101
+ /// apart from).
102
+ Render {
103
+ /// The column whose cell could not be rendered.
104
+ column: String,
105
+ /// The underlying formatting error's message.
106
+ message: String,
107
+ },
108
+ /// The per-cell diff would need more than `u32::MAX` changed rows on one
109
+ /// side, which its row-index arrays cannot address. Bound the changed-row
110
+ /// count of untrusted input.
111
+ TooManyChangedRows {
112
+ /// The number of changed rows that overflowed.
113
+ rows: usize,
114
+ },
115
+ /// A `value_changed` cell rendered identically on both sides — a broken
116
+ /// invariant, not a caller error. The per-cell diff renders each side in a
117
+ /// common comparison form so a value change always shows two different
118
+ /// strings; this variant guards that guarantee and cannot fire for any real
119
+ /// input.
120
+ EqualRenderings {
121
+ /// The column whose two renderings were equal.
122
+ column: String,
105
123
  },
106
124
  }
107
125
 
@@ -137,10 +155,20 @@ impl fmt::Display for TableDiffError {
137
155
  needs the key to be the same type on both sides"
138
156
  ),
139
157
  TableDiffError::Read { message } => write!(f, "failed to read table data: {message}"),
140
- TableDiffError::NotImplemented { feature } => write!(
158
+ TableDiffError::Render { column, message } => write!(
141
159
  f,
142
- "{feature} is not implemented in this version; \
143
- the per-cell changes arrive in a later version"
160
+ "could not render a value of column {column:?} to its canonical string: {message}"
161
+ ),
162
+ TableDiffError::TooManyChangedRows { rows } => write!(
163
+ f,
164
+ "the per-cell diff has {rows} changed rows on one side, more than the \
165
+ {} its row-index arrays can address; bound the changed-row count",
166
+ u32::MAX
167
+ ),
168
+ TableDiffError::EqualRenderings { column } => write!(
169
+ f,
170
+ "internal invariant: a value change in column {column:?} rendered identically \
171
+ on both sides, which the common-form rendering is designed to prevent"
144
172
  ),
145
173
  }
146
174
  }
@@ -220,13 +248,34 @@ mod tests {
220
248
  }
221
249
 
222
250
  #[test]
223
- fn not_implemented_message_names_the_feature() {
224
- let error = TableDiffError::NotImplemented {
225
- feature: "rows_added",
251
+ fn render_message_names_the_column() {
252
+ let error = TableDiffError::Render {
253
+ column: "ts".to_string(),
254
+ message: "Cast error".to_string(),
255
+ };
256
+ let message = error.to_string();
257
+ assert!(message.contains("\"ts\""));
258
+ assert!(message.contains("Cast error"));
259
+ }
260
+
261
+ #[test]
262
+ fn equal_renderings_message_reads_as_an_invariant() {
263
+ let error = TableDiffError::EqualRenderings {
264
+ column: "d".to_string(),
265
+ };
266
+ let message = error.to_string();
267
+ assert!(message.contains("\"d\""));
268
+ assert!(message.contains("invariant"));
269
+ }
270
+
271
+ #[test]
272
+ fn too_many_changed_rows_message_names_the_count() {
273
+ let error = TableDiffError::TooManyChangedRows {
274
+ rows: 5_000_000_000,
226
275
  };
227
276
  let message = error.to_string();
228
- assert!(message.contains("rows_added"));
229
- assert!(message.contains("not implemented"));
277
+ assert!(message.contains("5000000000"));
278
+ assert!(message.contains("changed-row count"));
230
279
  }
231
280
 
232
281
  #[test]
@@ -1,10 +1,11 @@
1
1
  //! Table diffing for onix, over Apache Arrow.
2
2
  //!
3
3
  //! [`diff_tables`] compares two tables presented as [`TableInput`]s and returns
4
- //! a [`TableDiff`] carrying both the **schema** diff — which columns were added,
5
- //! removed, or changed type — and the keyed **row** diff — which rows were
6
- //! added, removed, or changed, and which keys are duplicated. The per-cell diff
7
- //! (`cells_changed`) arrives in a later version.
4
+ //! a [`TableDiff`] carrying the **schema** diff — which columns were added,
5
+ //! removed, or changed type — the keyed **row** diff — which rows were added,
6
+ //! removed, or changed, and which keys are duplicated — and the per-cell diff
7
+ //! (`cells_changed`), reporting which columns changed in each changed row and
8
+ //! how.
8
9
  //!
9
10
  //! The two tables are matched on a required, non-empty set of key columns (the
10
11
  //! table's primary key), carried in [`TableDiffOptions`]. Every key column must
@@ -14,9 +15,10 @@
14
15
  //!
15
16
  //! # Inputs
16
17
  //!
17
- //! The row diff reads each side twice (once to hash every row, once to
18
- //! materialize only the differing rows), so [`diff_tables`] takes a
19
- //! re-openable [`TableInput`] rather than a single-use `RecordBatchReader`.
18
+ //! The row diff reads each side more than once (to hash every row, to
19
+ //! materialize the added/removed rows, and to materialize the changed rows for
20
+ //! the per-cell diff), so [`diff_tables`] takes a re-openable [`TableInput`]
21
+ //! rather than a single-use `RecordBatchReader`.
20
22
  //! In-memory tables use [`MemoryInput`]; a caller whose data is a one-shot
21
23
  //! stream spools it to a temporary Arrow IPC file first and implements
22
24
  //! [`TableInput`] over that file (as the Python bindings do).
@@ -162,6 +164,15 @@ pub const MAX_NESTING_DEPTH: usize = 128;
162
164
  /// nested key, a run-end-encoded column, or a type combination Arrow cannot
163
165
  /// build.
164
166
  /// - [`TableDiffError::Read`] if a batch cannot be read from either input.
167
+ /// - [`TableDiffError::Render`] if a changed cell's value cannot be rendered to
168
+ /// its canonical string — for example an out-of-range temporal value the
169
+ /// formatter cannot format.
170
+ /// - [`TableDiffError::TooManyChangedRows`] if one side has more than
171
+ /// `u32::MAX` changed rows, which the per-cell diff's row-index arrays cannot
172
+ /// address.
173
+ /// - [`TableDiffError::EqualRenderings`] never fires for real input: it guards
174
+ /// the internal invariant that a `value_changed` cell always renders two
175
+ /// different strings.
165
176
  pub fn diff_tables(
166
177
  left: &impl TableInput,
167
178
  right: &impl TableInput,