deepdiff-rs 0.10.0__tar.gz → 0.11.0__tar.gz

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/Cargo.lock

@@ -629,7 +629,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-arrow"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -646,7 +646,7 @@ dependencies = [
 
 [[package]]
 name = "onix-cli"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -654,8 +654,10 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
+ "num-bigint",
+ "num-traits",
  "proptest",
  "serde",
  "serde_json",
@@ -665,11 +667,12 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.10.0"
+version = "0.11.0"
 dependencies = [
  "arrow-array",
  "arrow-ipc",
  "arrow-schema",
+ "num-bigint",
  "onix-arrow",
  "onix-core",
  "pyo3",

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py", "crates/onix-arrow"]
 
 [workspace.package]
-version = "0.10.0"
+version = "0.11.0"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.10.0
+Version: 0.11.0
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License
@@ -195,7 +195,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on a seeded 5 GB parquet pair and its 1M-row subset — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' rather than the several-fold gap the earlier single-threaded row diff showed; see the linked results for the figures and the before/after; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared, still trails by a much larger multiple, see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -236,7 +236,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. An `int` of any magnitude converts and keeps its exact value (an arbitrary-precision integer beyond `i64`/`u64` is compared, ordered, hashed and rendered by its full digits), while a custom object raises `TypeError` naming the exact path it was found at. Under `ignore_order`, comparing an integer beyond `f64::MAX` (about `2**1024`) reports the change deterministically where real DeepDiff raises `OverflowError` (its distance function calls `float()` on it unguarded), a documented crash-class divergence like the datetime one below (see [`tests/golden/README.md`](tests/golden/README.md)). Reading JSON *text* (`diff_json` and the CLI, not the Python-object path), an integer beyond `i64`/`u64` parses as the nearest `float`, so two documents whose integers differ only past that range compare **equal** and diff to `{}` — a limitation of the JSON number reader, not the value model, tracked in [#92](https://github.com/ksco92/onix/issues/92). A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, or `datetime.timedelta` (including `namedtuple` and pandas' `Timestamp`) converts and compares as its base type but carries its own class name into a `type_changes` entry, matching DeepDiff's `type(obj).__name__` reporting, with three exceptions: a calendar-type (`datetime`/`date`/`time`/`timedelta`) subclass held as a `set`/`frozenset` member compares as its base with no `type_changes`, matching DeepDiff, since set membership has no pairwise comparison to report one against; a `tuple`/`frozenset` subclass, including a `namedtuple`, is not accepted as a `set`/`frozenset` member at all and raises `TypeError` where DeepDiff accepts and compares it by value (a documented divergence); and `namedtuple` diffs positionally rather than by field (also documented). A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. See [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
@@ -250,7 +250,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M); the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 67 GB at 16.875M rows/side, see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/README.md

@@ -177,7 +177,7 @@ The engine's own diff-only time and peak resident memory against pinned `deepdif
 | `ignore_order_10k` | 73.475 ms (71.672 ms-73.940 ms) | 12.976 s (12.900 s-13.005 s) | 176.60x | 60.11 MB | 345.19 MB | 5.74x | ✅ |
 | `identical_1m` | 9.254 ms (6.726 ms-9.875 ms) | 15.790 s (15.660 s-15.989 s) | 1706.35x | 315.41 MB | 503.19 MB | 1.60x | ✅ |
 
-Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on a seeded 5 GB parquet pair and its 1M-row subset — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' rather than the several-fold gap the earlier single-threaded row diff showed; see the linked results for the figures and the before/after; regenerated with `perf/arrow/bench_tables.py`.
+Both reports carry their full methodology, fairness rules, and the reproduce command. `perf/RESULTS.md` is an upper bound (JSON parsed straight into the engine, no Python-object conversion); the bindings table is the product-surface number. Regenerate them with `perf/run_bench.sh` and `crates/onix-py/benchmarks/bench_bindings.py` (see [CONTRIBUTING.md](CONTRIBUTING.md)). The Arrow table diff's cost against two hand-rolled baselines — a DuckDB SQL join and a polars anti-join/inner-join diff, on two seeded ~5 GB parquet pairs and their own 1M-row subsets (a five-column narrow fixture, and a wide fixture covering every scalar Arrow type) — is in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md): the row diff now hashes and classifies across every core, so at the 5 GB size its wall time is a small multiple of the two parallelized baselines' for the narrow fixture (down from the several-fold single-threaded gap); the wide fixture, where nearly every row's 34 columns are compared, still trails by a much larger multiple, see the linked results for both; regenerated with `perf/arrow/bench_tables.py`.
 
 ## Reference
 
@@ -218,7 +218,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 ## Known limitations
 
 - Only the core diff is implemented: `exclude_paths`, `significant_digits`, custom operators, `verbose_level != 2`, and delta/patch are not (yet) supported.
-- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. `int`s must fit in `i64`/`u64`, and anything else — a custom object or an arbitrary-precision `int` — raises `TypeError`/`ValueError` naming the exact path it was found at. A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
+- Supported value types are `None`, `bool`, `int`, `float` (`NaN`/`Infinity`/`-Infinity` included), `str`, `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, and `datetime.timedelta`; a `set`/`frozenset` member may be any of these except a `list`, `dict` or `set`, matching Python's own hashability rule, transitively through whatever the member nests, and a `dict` key may be `str`, `None`, `bool`, `int`, `float`, `datetime.datetime`, `datetime.date`, or a `tuple` of those (never nested), or a `tuple`/`datetime`/`date` subclass key (`namedtuple` included), accepted and matched against its base-type value the same way — see [Known limitations](#known-limitations)'s subclass bullet. An `int` of any magnitude converts and keeps its exact value (an arbitrary-precision integer beyond `i64`/`u64` is compared, ordered, hashed and rendered by its full digits), while a custom object raises `TypeError` naming the exact path it was found at. Under `ignore_order`, comparing an integer beyond `f64::MAX` (about `2**1024`) reports the change deterministically where real DeepDiff raises `OverflowError` (its distance function calls `float()` on it unguarded), a documented crash-class divergence like the datetime one below (see [`tests/golden/README.md`](tests/golden/README.md)). Reading JSON *text* (`diff_json` and the CLI, not the Python-object path), an integer beyond `i64`/`u64` parses as the nearest `float`, so two documents whose integers differ only past that range compare **equal** and diff to `{}` — a limitation of the JSON number reader, not the value model, tracked in [#92](https://github.com/ksco92/onix/issues/92). A non-`str` key's path renders via Python's own `repr()`, except a `tuple` key, which splits into one bracket group per element (`root[1][2]` for `(1, 2)`, never `root[(1, 2)]`, with one deliberate exception for a real DeepDiff bug on the empty tuple, and another on a non-finite-float key — see [`tests/golden/README.md`](tests/golden/README.md)). A nested dict value's own `bool`/`None`/`int`/`float` key stringifies the way `json.dumps` does, but its `datetime`/`date`/`tuple` key renders that same `repr()` text where DeepDiff's own `to_json()` raises `TypeError` on one — a superset, not a difference in the findings (see [`tests/golden/README.md`](tests/golden/README.md)). `1`/`1.0`/`True` match as the same key between two dicts (Python `dict`/`set` equality). The **Datetimes**, **Sets** and **Non-finite floats** bullets below cover the deliberate divergences for those types. See [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs) and [`tests/golden/README.md`](tests/golden/README.md).
 - A subclass of a supported `dict`, `list`, `tuple`, `set`, `frozenset`, `datetime.datetime`, `datetime.date`, `datetime.time`, or `datetime.timedelta` (including `namedtuple` and pandas' `Timestamp`) converts and compares as its base type but carries its own class name into a `type_changes` entry, matching DeepDiff's `type(obj).__name__` reporting, with three exceptions: a calendar-type (`datetime`/`date`/`time`/`timedelta`) subclass held as a `set`/`frozenset` member compares as its base with no `type_changes`, matching DeepDiff, since set membership has no pairwise comparison to report one against; a `tuple`/`frozenset` subclass, including a `namedtuple`, is not accepted as a `set`/`frozenset` member at all and raises `TypeError` where DeepDiff accepts and compares it by value (a documented divergence); and `namedtuple` diffs positionally rather than by field (also documented). A `type_changes` entry's `old_type`/`new_type` are type *names* in `to_dict()`, where DeepDiff returns the type objects. See [`tests/golden/README.md`](tests/golden/README.md).
 - **Datetimes** compare by instant, with a naive value read as UTC, matching DeepDiff. A changed pair is reported normalized to UTC (`to_json()` renders `...+00:00`, `to_dict()` returns UTC-aware `datetime`s); everywhere else a datetime keeps its raw value. Three deliberate departures: `to_json()` renders a `date` as `YYYY-MM-DD` where DeepDiff's own `to_json()` raises `TypeError` (a documented superset); a `zoneinfo`/`pytz` tzinfo comes back from `to_dict()` as a fixed-offset `datetime.timezone` carrying the offset it was in force at, not the original zone object; and a set holding both a naive and an aware value at one instant reports both as members, where DeepDiff's own digest cache can report only one (see [`crates/onix-py/src/convert.rs`](crates/onix-py/src/convert.rs)). Comparing two datetimes whose UTC form would leave year 1..=9999 raises `ValueError` naming the path, where DeepDiff raises `OverflowError`; under `ignore_order` DeepDiff's hasher normalizes every datetime and so raises for such a value even when it is only added, removed, or shuffled, where onix hashes by instant and reports it normally (see [`tests/golden/README.md`](tests/golden/README.md)). `truncate_datetime` is not supported; the normalized-versus-raw split is documented there too. A `time`/`timedelta`, unlike a datetime, is never normalized for report (DeepDiff compares `time`/`date`/`timedelta` with a plain `!=`), and a naive `time` is never equal to an aware one; `to_json()` renders a `time` as `time.isoformat()`'s bytes and a `timedelta` as `str(timedelta)`'s, both supersets. Under `ignore_order`, `DeepHash` hashes a `time` by whole seconds-of-day only — dropping the microsecond and any offset, a confirmed upstream quirk — while a `timedelta` hashes exactly; see [`tests/golden/README.md`](tests/golden/README.md)'s "Known DeepDiff quirks" section.
 - **Sets** are diffed deterministically, where DeepDiff's own answers depend on the order the running process happens to iterate a set in (hash order, and `PYTHONHASHSEED`-dependent for `str` members) or on how its digest cache/computation handles a tuple, frozenset, or calendar member independently of Python's own `==`. Each consequence — entry order, which member of an equality class is reported, set-versus-sequence coercion, and a tuple/frozenset member's own (positional, not order-/repetition-insensitive) matching rule — is shown with both tools' output in [`tests/golden/README.md`](tests/golden/README.md)'s "Set iteration order" section. A report holding a `frozenset` value also serializes to JSON here, where DeepDiff's own `to_json()` raises `TypeError` — a superset, not a difference in the findings.
@@ -232,7 +232,7 @@ perf/              # cross-language benchmark harness and RESULTS.md
 - `diff_tables` inherits some Arrow-interchange quirks: a column name with an embedded NUL byte (`\0`) arrives truncated at the NUL through the C Data Interface (the report shows the truncated name; rare in practice); a list of structs named exactly `key`/`value` with a nullable key is not distinguished from a real map, so a migration between the two is not reported as a type change (polars exports both as the same Arrow type — see `map_entries` in [`crates/onix-arrow/src/schema.rs`](crates/onix-arrow/src/schema.rs)); and a polars all-null (`Null`-typed) column fails at Arrow C import with a `ValueError` (`the datatype "Null" doesn't expect buffer at index 0`), so give such a column a concrete type first (a pyarrow all-`None` column, inferred as `null`, works and compares as all-null).
 - In `diff_tables`, DuckDB labels a `TIMESTAMP WITH TIME ZONE` column with the connection's *session* time zone when it exports to Arrow (a UTC session as `Timestamp(µs, "UTC")`, an `America/New_York` session as `Timestamp(µs, "America/New_York")`), so on a non-UTC machine such a column can be reported as a type change against a UTC column from another library. Run `SET TimeZone='UTC'` on the DuckDB connection first for a deterministic, machine-independent result.
 - `diff_tables` refuses a column whose Arrow type is nested deeper than `MAX_NESTING_DEPTH` (128) with a `MaxDepthError`, because comparing arbitrarily deep nesting would overflow the native stack; 128 is far beyond any real schema. Importing a schema nested many thousands of levels deep is also slow regardless, a cost of the Arrow C Data Interface itself.
-- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
+- `diff_tables`'s row diff costs, per call: RAM for a 32-byte hash per row per side (sorted), so peak memory is linear in row count (about 75 MB at 1M rows/side, 660 MB at 10M); the diff hashes and classifies across all cores by default, running single-threaded only when both sides stay under 50,000 rows and under 64 MB of decoded data (either bound reached first runs the parallel path), and appends the per-row hashes into shared per-partition buffers so there is no separate combined copy; the parallel path adds only the in-flight batches (worker count times batch size) plus the shared buffers' reallocation slack, measured at the linear shape as a peak of about 629 MB single-threaded and 857 MB at 18 threads at 8M rows/side (+228 MB) and about 2.9 GB and 3.6 GB at 37M rows/side (+685 MB) — roughly 10-15 bytes per row per side, under one extra copy of the 32-byte hash vectors, and within the run-to-run slack the single-threaded peak itself shows (2.9-4.2 GB at 37M); `threads=1` runs the sequential path; the size gate peeks each side only up to 50,000 rows or 64 MB of decoded data before choosing, reading the left side first; the peek holds at most 64 MB plus one producer batch per side — a streamed 49,999-row/side pair of 8 KB cells in 100-row batches peaks at about 78 MB at 18 threads and 14 MB single-threaded, rising to about 1.6 GB when the whole side arrives as one batch (see the size-gate peek table in [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md)); wall time is `N·log N` work spread across the workers, plus a second term for the duplicate-key report, which holds the key values of every *distinct duplicated* key — an all-duplicate 200k-rows/side table (100k distinct duplicated keys) peaks at about 37 MB with 16-byte keys and 1.05 GB with 1 KB keys, 1M rows/side (500k distinct) at about 165 MB with 16-byte keys; a third term for the per-cell diff, which re-reads both inputs, holds the changed rows of both sides, and renders every changed cell to a string held once in the output — so its cost is the number of changed cells times the cell width, not the changed-row count alone (every value is rendered in full). Measured, same method: with two narrow `int64` columns, 85 MB at 1M rows/side with 2% changed and 629 MB with every row changed; with a 1 KB `string` cell changed on every row, 1.18 GB at 100k rows/side and 2.34 GB at 200k (about 11.6 KB per changed row, linear, so on the order of 11.6 GB at 1M); the full wide benchmark fixture (34 columns, nearly every cell changed) measures about 67 GB at 16.875M rows/side, see [`perf/arrow/RESULTS.md`](perf/arrow/RESULTS.md). Then temp disk, because each input is re-read several times and so is spooled to an anonymous file (`tempfile`: unlinked at once, mode 0600, no predictable name — nothing is left on disk even on abnormal exit), both spools resident at once, peaking at the decoded size of both inputs (about 315 MB — 161 + 154 MB uncompressed Arrow IPC — for the 1M-row fixture pair; on the order of 10 GB for the full 5 GB-per-side fixture pair, which on Linux may be a RAM-backed `tmpfs`), a full temp filesystem raising `ValueError` naming `TMPDIR`. None of these has a built-in cap: for untrusted input, bound the row count, the changed fraction, the column widths (key columns for duplicate-heavy data, changed cells — rendered in full — for wide value columns, and any column's width for the size-gate peek, which buffers decoded cells whether or not they change), and the producer's batch size, since the peek holds up to one whole batch per side beyond the 64 MB bound. Figures are the peak resident set of `cargo run -p onix-arrow --release --example row_diff_rss` (worker count from `ROW_DIFF_THREADS`, rows per generated batch from `ROW_DIFF_BATCH`), single run, macOS on an Apple M-series laptop, 2026-09-06, same method as [Performance](#performance). See [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs).
 - `TableDiff.to_json()` is the one member with a built-in cap: it embeds `rows_added`, `rows_removed`, `cells_changed`, and `duplicate_keys` in full, one JSON object per row, so it refuses with `ValueError` — naming the row count and the cap — once those four together hold more than 10,000 rows. The cap bounds row count only, the same way the row-diff cost bullet above states its per-cell term as changed cells times cell width, not column count or cell width — so a table under the row cap but with wide or large cells can still be large; use the Arrow-returning accessors instead. See [`crates/onix-arrow/src/json_rows.rs`](crates/onix-arrow/src/json_rows.rs).
 - `diff_tables` compares scalar columns by value (hashed: null, booleans, every integer, float and decimal width, strings and binary in every encoding, timestamps, dates, times, durations, intervals, and dictionaries of these; refused with `ValueError`: run-end encoded columns and any type-and-unit combination Arrow itself cannot build; nested non-key columns skipped, nested key columns refused), with the exact enumeration in the module doc of [`crates/onix-arrow/src/row_diff.rs`](crates/onix-arrow/src/row_diff.rs). It also refuses a key column whose type differs across the two inputs after encoding normalization (the conservative choice: a primary key that changed type is refused rather than guessed, not coerced). A row whose only difference is a lossless type change — an `Int32` widened to `Int64`, a timestamp unit change at the same instant — hashes equal on both sides, so it is in neither `rows_changed` nor `cells_changed`; the column's type change is still reported in `schema`.
 - Output is byte-identical to DeepDiff except for the cases listed above and two path-rendering quirks; [`tests/golden/README.md`](tests/golden/README.md) enumerates every accepted exception, including integers past `2^53` (the limit of exact `f64` representation) inside ordered scalar lists and `ignore_order` pairing among naive datetimes, which DeepDiff ranks using the *process's local timezone* while onix reads a naive value as UTC everywhere.

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/Cargo.toml

@@ -13,6 +13,11 @@ workspace = true
 serde = "1"
 serde_json = "1"
 unicode-general-category = "1.1"
+# Arbitrary-precision integers (a Python `int` outside i64/u64 range). Pinned
+# to the same 0.5 line arrow-buffer already resolves in this workspace, so no
+# second copy enters the tree.
+num-bigint = "0.5"
+num-traits = "0.2"
 
 [dev-dependencies]
 proptest = "1"

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/diff/mod.rs

@@ -189,7 +189,7 @@
 //!   risking the very overflow they guard against, and `scoped`, the shared
 //!   push/pop path-buffer helper every container loop below uses.
 //! - `scalar` — leaf-level comparison: scalar/numeric equality
-//!   (`numbers_equal`, `floats_equal`, `number_as_i128`) and the
+//!   (`numbers_equal`, `floats_equal`) and the
 //!   `type_changes`/`values_changed` finding builders (`type_change_report`,
 //!   `scalar_diff`, `numeric_diff`) `diff_at` dispatches to for a
 //!   non-container pair.

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/diff/scalar.rs

@@ -191,9 +191,10 @@ pub(crate) fn numeric_diff(
 /// An int and a float are never equal (mirroring `DeepDiff` always reporting
 /// that pairing as a `type_changes`, never a numeric comparison). Within the
 /// same kind: floats compare by exact IEEE-754 `==` (see [`floats_equal`]);
-/// ints compare by value across `i64`/`u64` representations (see
-/// [`number_as_i128`]), so `9_000_000_000_000_000_000u64` and its `i64`
-/// counterpart compare equal even though they use different representations.
+/// ints compare by value across every representation via
+/// [`Number::integer_cmp`], so `9_000_000_000_000_000_000u64` and its `i64`
+/// counterpart — or an arbitrary-precision integer and its equal — compare
+/// equal even though they use different representations.
 pub(crate) fn numbers_equal(old: &Number, new: &Number) -> bool {
     if old.is_f64() != new.is_f64() {
         return false;
@@ -208,11 +209,7 @@ pub(crate) fn numbers_equal(old: &Number, new: &Number) -> bool {
             .expect("Number::is_f64 guarantees as_f64 succeeds");
         floats_equal(old_f, new_f)
     } else {
-        let old_int =
-            number_as_i128(old).expect("non-float Number always has an i64 or u64 representation");
-        let new_int =
-            number_as_i128(new).expect("non-float Number always has an i64 or u64 representation");
-        old_int == new_int
+        old.integer_cmp(new).is_eq()
     }
 }
 /// Compares two floats for exact equality.
@@ -233,11 +230,3 @@ fn floats_equal(a: f64, b: f64) -> bool {
         a == b
     }
 }
-/// Converts a non-float [`Number`] to `i128`, so ints stored as `u64` and
-/// `i64` compare equal by value regardless of which representation
-/// the compact `Number` preserves from `serde_json`'s original parse.
-pub(crate) fn number_as_i128(n: &Number) -> Option<i128> {
-    n.as_i64()
-        .map(i128::from)
-        .or_else(|| n.as_u64().map(i128::from))
-}

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/diff/tests.rs

@@ -37,7 +37,7 @@ fn map_deeper_than(map: &Map<String, Value>, limit: usize) -> bool {
     super::dispatch::map_deeper_than(&cobj(map), limit)
 }
 fn number_as_i128(n: &Number) -> Option<i128> {
-    super::scalar::number_as_i128(&cnum(n))
+    cnum(n).as_i128()
 }
 
 /// Wraps `leaf` in `depth` single-key (`"k"`) nested dicts, so `leaf`

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/ignore_order/distance.rs

@@ -695,7 +695,9 @@ fn coerce_to_python_str(value: &Value) -> Option<String> {
             } else if let Some(i) = n.as_i64() {
                 Some(i.to_string())
             } else {
-                n.as_u64().map(|u| u.to_string())
+                n.as_u64()
+                    .map(|u| u.to_string())
+                    .or_else(|| n.as_big().map(ToString::to_string))
             }
         }
         // `str(x)` is the identity for a value already a `str`, including

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/ignore_order/fxhash.rs

@@ -68,6 +68,17 @@ pub(crate) type HashSet<T> = std::collections::HashSet<T, BuildHasherDefault<FxH
 /// changes which values these tables treat as the same item, not their
 /// per-lookup cost.
 ///
+/// An integer beyond `i128` is carried as `ItemKey::BigInt` (hence a
+/// `DistKey` leaf, via `number_key`) and, inside a hashable tuple, as
+/// [`ScalarKey::Big`](crate::lcs::ScalarKey) reaching the `tuple_ids` memo key
+/// through `PyHashPart::Scalar`; both hash and compare by its magnitude digits,
+/// which carry ample entropy and so need no `mix_float_bits` treatment, but at
+/// `O(digits)` rather than `O(1)` per lookup — a cost proportional to that one
+/// operand's *digit length* (the number of machine words in its magnitude),
+/// which the `ignore_order` element-count cap does not bound: a single
+/// astronomically large integer costs its own digit length per hash and per
+/// comparison regardless of how few elements the diff holds.
+///
 /// For those remaining `ignore_order`-only tables the trade is deliberate and
 /// measured. Re-keying them to `SipHash` (`RandomState`) added a material,
 /// measured double-digit-percentage per-call cost on the pairing-heavy

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/ignore_order/hash.rs

@@ -8,6 +8,8 @@ use std::collections::{BTreeMap, BTreeSet};
 use std::hash::{Hash, Hasher};
 use std::rc::Rc;
 
+use num_bigint::BigInt;
+
 use crate::lcs::{ScalarKey, mix_float_bits, python_scalar_key};
 use crate::value::Value;
 
@@ -84,8 +86,9 @@ impl Hash for DistKey {
 
 /// Hashes a value consistently with its structural `PartialEq` (equal values
 /// hash equal): a per-variant discriminant, then the fields that equality
-/// compares — numbers through [`number_key`] (so `-0.0`/`0.0` agree and an
-/// int and an equal-valued float stay distinct), a datetime by its instant, a
+/// compares — numbers through [`number_key`] (so `-0.0`/`0.0` agree, an
+/// int and an equal-valued float stay distinct, and an integer beyond `i128`
+/// hashes by its digits at `O(digits)`), a datetime by its instant, a
 /// date by its ordinal, a list/tuple by its length and elements (order and
 /// repetition preserving), a set/frozenset by its canonical members, a dict by
 /// its sorted keys and their values. The exact byte sequence is unspecified;
@@ -183,12 +186,19 @@ pub(crate) enum ItemKey {
     /// `true`/`false` — tagged distinctly from any integer of the same
     /// value (never collides with `Int`).
     Bool(bool),
-    /// A JSON value `serde_json` parsed as an integer (no decimal point or
-    /// exponent) — see [`mod@crate::diff`]'s `python_type_name` for the same
-    /// int/float split used throughout this crate. Always exact: a
-    /// `serde_json::Number`'s non-float representation is always an `i64`
-    /// or `u64`, both of which fit losslessly in `i128`.
+    /// An integer whose value fits `i128` — see [`mod@crate::diff`]'s
+    /// `python_type_name` for the same int/float split used throughout this
+    /// crate. Covers every `i64`/`u64` value (both fit losslessly) and any
+    /// arbitrary-precision integer that happens to fit `i128`; a value equal
+    /// across those representations shares one key.
     Int(i128),
+    /// An arbitrary-precision integer whose magnitude exceeds `i128`, in its
+    /// own bucket — never collides with [`ItemKey::Int`] because that arm
+    /// only ever holds a value `i128` can represent (see [`number_key`],
+    /// which sends every fitting value there first). Boxed so this rare arm
+    /// does not widen the enum (a `BigInt` dwarfs the other arms' payloads),
+    /// which would grow every key in the `ignore_order` hash tables.
+    BigInt(Box<BigInt>),
     /// A float, keyed by [`deephash_float_bits`] — its exact bit pattern for
     /// any finite value (kept as its own bucket even when whole-numbered:
     /// `5.0` never collides with `Int(5)`; see this type's own doc), but one
@@ -279,6 +289,7 @@ impl std::hash::Hash for ItemKey {
             Self::Null => {}
             Self::Bool(b) => b.hash(state),
             Self::Int(i) => i.hash(state),
+            Self::BigInt(b) => b.hash(state),
             Self::Float(bits) => mix_float_bits(*bits).hash(state),
             Self::Str(s) => s.hash(state),
             Self::DateTime(instant) => instant.hash(state),
@@ -723,12 +734,13 @@ fn number_key(n: &crate::value::Number) -> ItemKey {
             .expect("Number::is_f64 guarantees as_f64 succeeds");
         return ItemKey::Float(deephash_float_bits(f));
     }
-    if let Some(i) = n.as_i64() {
-        return ItemKey::Int(i128::from(i));
+    if let Some(i) = n.as_i128() {
+        return ItemKey::Int(i);
     }
-    ItemKey::Int(i128::from(
-        n.as_u64()
-            .expect("a non-f64 Number always has an i64 or u64 repr"),
+    ItemKey::BigInt(Box::new(
+        n.as_big()
+            .expect("a non-float Number that overflows i128 is an arbitrary-precision integer")
+            .clone(),
     ))
 }
 
@@ -772,29 +784,13 @@ fn keyed(value: &Value, memo: &IgnoreOrderMemo, want_part: bool) -> (ItemKey, Op
         Value::Date(value) => (ItemKey::Date(value.ordinal()), part()),
         Value::Time(value) => (ItemKey::Time(value.hash_seconds_of_day()), part()),
         Value::TimeDelta(value) => (ItemKey::TimeDelta(value.value()), part()),
-        Value::Number(n) => {
-            let number = if n.is_f64() {
-                let f = n
-                    .as_f64()
-                    .expect("Number::is_f64 guarantees as_f64 succeeds");
-                // See `deephash_float_bits`: it is the identity on every
-                // finite float but `-0.0`, so an integral float like `2.0`
-                // keeps a distinct `Float` key from the integer `2` (this
-                // deliberately does NOT take the ordered path's `ScalarKey`
-                // integral-to-`Int` canonicalization — the two paths have
-                // genuinely different number semantics), and it collapses
-                // every `NaN` onto one shared key regardless of bits.
-                ItemKey::Float(deephash_float_bits(f))
-            } else if let Some(i) = n.as_i64() {
-                ItemKey::Int(i128::from(i))
-            } else {
-                let u = n
-                    .as_u64()
-                    .expect("a non-f64 serde_json::Number always has an i64 or u64 repr");
-                ItemKey::Int(i128::from(u))
-            };
-            (number, part())
-        }
+        // See `number_key` (and `deephash_float_bits`): an integral float
+        // like `2.0` keeps a distinct `Float` key from the integer `2` (this
+        // deliberately does NOT take the ordered path's `ScalarKey`
+        // integral-to-`Int` canonicalization — the two paths have genuinely
+        // different number semantics), every `NaN` collapses onto one shared
+        // key, and an integer keys by value regardless of magnitude.
+        Value::Number(n) => (number_key(n), part()),
         Value::Array(items) => (
             ItemKey::List(items.iter().map(|i| item_key(i, memo)).collect()),
             None,

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/ignore_order/memo.rs

@@ -142,7 +142,10 @@ use super::hash::{
 /// and the `A * R` entries one pairing records just share those [`Rc`](std::rc::Rc)s. The
 /// *lookup* cost is not constant, though — every probe hashes both keys (a
 /// full walk of each value) and, on a bucket match, compares them structurally
-/// — so per-pair work is proportional to record size (see [`super::hash::DistKey`]).
+/// — so per-pair work is proportional to record size (see [`super::hash::DistKey`]),
+/// and a leaf that is an integer beyond `i128` (an [`ItemKey::BigInt`], via
+/// `number_key`) additionally costs its own *digit length* per hash and per
+/// comparison, which the element-count cap does not bound.
 type DistanceKey = (DistKey, DistKey);
 
 /// The per-top-level-diff caches described in this module's doc: container-pair
@@ -154,14 +157,22 @@ type DistanceKey = (DistKey, DistKey);
 /// recursive diff, and dropped when it returns. No eviction and no tuning
 /// knobs: each is bounded by the number of distinct queries one diff makes.
 /// `member_content`'s own key can itself carry a nested dict's keys (see its
-/// own field doc), so a lookup there is not always a flat comparison.
+/// own field doc), so a lookup there is not always a flat comparison. A leaf
+/// that is an integer beyond `i128` is keyed by an [`ItemKey::BigInt`] (in the
+/// `cache`'s `DistKey`s and, when a set member, in `member_content`), hashed
+/// and compared by its magnitude at `O(digit length)` — a per-lookup cost the
+/// element-count bounds above do not cap (see `super::fxhash`'s doc).
 ///
 /// [`rough_distance`]: super::distance::rough_distance
 pub(crate) struct IgnoreOrderMemo {
     cache: RefCell<HashMap<DistanceKey, f64>>,
     /// Interns each distinct hashable-tuple identity to its place in
     /// `tuple_digests`, so a nested tuple can be named by one [`TupleId`]
-    /// inside its parent's identity instead of by a copy of its own.
+    /// inside its parent's identity instead of by a copy of its own. A tuple
+    /// member that is an integer beyond `i128` reaches this `FxHash` key as a
+    /// [`ScalarKey::Big`](crate::lcs::ScalarKey) (via `PyHashPart::Scalar`),
+    /// hashed and compared by its magnitude digits at `O(digit length)` — the
+    /// same per-lookup cost `super::fxhash`'s doc notes for `ItemKey::BigInt`.
     tuple_ids: RefCell<HashMap<PyHashKey, TupleId>>,
     /// The digest assigned to each interned identity, indexed by
     /// [`TupleId::index`].

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/ignore_order/tests.rs

@@ -336,6 +336,40 @@ fn item_key_handles_a_u64_beyond_i64_range() {
     assert_eq!(ignore_order_diff(&a, &b), json!({}));
 }
 
+#[test]
+fn item_key_distinguishes_arbitrary_precision_integers() {
+    use crate::value::Number;
+
+    let big = |s: &str| CValue::Number(Number::from_bigint(s.parse().expect("integer")));
+    let memo = IgnoreOrderMemo::new();
+    let k = |v: &CValue| super::hash::item_key(v, &memo);
+
+    // A big int fitting i128 keys as `Int`; one beyond i128 as `BigInt`. Two
+    // equal big ints share a key; different ones do not; and neither collides
+    // with a small int or an equal-valued float (numbers stay type-distinct
+    // under DeepHash — see `hash::ItemKey`'s doc).
+    let two_pow_100 = "1267650600228229401496703205376";
+    let beyond_i128 = "1".to_string() + &"0".repeat(40);
+    assert_eq!(k(&big(two_pow_100)), k(&big(two_pow_100)));
+    assert_ne!(
+        k(&big(two_pow_100)),
+        k(&big("1267650600228229401496703205377"))
+    );
+    assert_ne!(k(&big(two_pow_100)), k(&big(&beyond_i128)));
+    assert_ne!(k(&big(two_pow_100)), k(&cv(&json!(1))));
+    assert_ne!(
+        k(&big(two_pow_100)),
+        k(&CValue::Number(Number::from_f64(2f64.powi(100)))),
+    );
+
+    // Hashing a beyond-i128 `ItemKey::BigInt` agrees with its equality: the
+    // same value re-inserts as a duplicate, a different one as a new entry.
+    let mut seen = std::collections::HashSet::new();
+    assert!(seen.insert(k(&big(&beyond_i128))));
+    assert!(!seen.insert(k(&big(&beyond_i128))));
+    assert!(seen.insert(k(&big(two_pow_100))));
+}
+
 #[test]
 fn numeric_pairing_at_two_distances_reuses_the_used_check_across_buckets() {
     // "5" (added) has candidates at two distinct distances: "4"

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/lcs.rs

@@ -58,6 +58,9 @@
 
 use std::collections::HashMap;
 
+use num_bigint::BigInt;
+use num_traits::{FromPrimitive, ToPrimitive};
+
 use crate::value::Value;
 
 /// A bucket key for grouping list elements that compare equal the way
@@ -91,6 +94,15 @@ pub(crate) enum ScalarKey {
     /// compares correctly instead of failing to compile or colliding.
     Str(Vec<u8>),
     Int(i128),
+    /// An integer-valued scalar whose magnitude exceeds `i128` and is not
+    /// exactly representable as an `f64` — a big int that shares Python `==`
+    /// with no float, so it needs its own bucket. An exactly-representable
+    /// big int takes [`ScalarKey::Float`] instead (so it collapses with the
+    /// equal float, matching Python's `10**20 == 1e20`); see
+    /// [`python_scalar_key`]. Boxed so this rare arm does not widen the enum
+    /// (a `BigInt` is far larger than the `i128`/`Vec` the other arms hold),
+    /// which would grow every key in the scalar-list matcher's hash table.
+    Big(Box<BigInt>),
     /// Bit pattern of a non-integral (or too-large-to-be-exact) float —
     /// hashed through [`mix_float_bits`]; see this type's hand-written `Hash`.
     Float(u64),
@@ -154,6 +166,7 @@ impl std::hash::Hash for ScalarKey {
             Self::Null => {}
             Self::Str(s) => s.hash(state),
             Self::Int(i) => i.hash(state),
+            Self::Big(b) => b.hash(state),
             Self::Float(bits) => mix_float_bits(*bits).hash(state),
             // A `Value` node's address is 8/16-byte-aligned like any other
             // pointer, so its low bits carry no entropy; avalanche it the
@@ -264,7 +277,23 @@ pub(crate) fn python_scalar_key(value: &Value) -> Option<ScalarKey> {
             {
                 return Some(ScalarKey::Int(i));
             }
-            let f = n.as_f64().expect("a serde_json Number is i64, u64, or f64");
+            if let Some(big) = n.as_big() {
+                // An arbitrary-precision integer collapses with a float that
+                // is exactly equal to it (Python's own `10**20 == 1e20`): if
+                // it round-trips through `f64` losslessly it shares that
+                // float's `Float` key (the same rule the `>2^53` integral
+                // float below already uses), otherwise it keeps its own `Big`
+                // key. So `10**20`/`1e20` pair but `10**23`/`1e23` do not —
+                // see `tests/golden/README.md`.
+                let as_float = big.to_f64().unwrap_or(f64::INFINITY);
+                if as_float.is_finite()
+                    && BigInt::from_f64(as_float).is_some_and(|rounded| &rounded == big)
+                {
+                    return Some(ScalarKey::Float(as_float.to_bits()));
+                }
+                return Some(ScalarKey::Big(Box::new(big.clone())));
+            }
+            let f = n.as_f64().expect("a non-integer Number is a float");
             if f.is_nan() {
                 return Some(ScalarKey::Nan(std::ptr::from_ref(value) as usize));
             }

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/lcs_tests.rs

@@ -817,3 +817,48 @@ fn build_b2j_purges_only_above_the_autojunk_threshold() {
     let unpurged = super::build_b2j(&b_keys, false);
     assert!(unpurged.contains_key(&scalar_key(&json!("pop"))));
 }
+
+// --- arbitrary-precision integer scalar keys ----------------------------
+
+/// A big integer collapses onto the same `ScalarKey` as a float exactly equal
+/// to it (Python's `10**20 == 1e20`), and stays distinct from a float it does
+/// not equal — the ordered-list matcher's Python-`==` rule. Floats are built
+/// directly from their exact `f64` bits here, not parsed from text, so this
+/// pins the collapse logic independently of the JSON reader's own float
+/// rounding.
+#[test]
+fn big_integer_scalar_key_collapses_with_its_equal_float_only() {
+    use crate::value::{Number, Value};
+
+    let bigint = |s: &str| Value::Number(Number::from_bigint(s.parse().expect("integer")));
+    let float = |f: f64| Value::Number(Number::from_f64(f));
+
+    // 2^70 is exactly representable, so the big int and the float share a key.
+    let two_pow_70 = "1180591620717411303424";
+    assert_eq!(
+        super::python_scalar_key(&bigint(two_pow_70)),
+        super::python_scalar_key(&float(2f64.powi(70))),
+    );
+
+    // 10^23 is NOT exactly representable (`10**23 != 1e23` in f64), so the big
+    // int keeps its own `Big` key and never matches the float `1e23`.
+    let ten_pow_23 = "100000000000000000000000";
+    assert_ne!(
+        super::python_scalar_key(&bigint(ten_pow_23)),
+        super::python_scalar_key(&float(1e23)),
+    );
+    assert!(matches!(
+        super::python_scalar_key(&bigint(ten_pow_23)),
+        Some(super::ScalarKey::Big(_))
+    ));
+
+    // Two equal big ints share a key; two different ones do not.
+    assert_eq!(
+        super::python_scalar_key(&bigint(ten_pow_23)),
+        super::python_scalar_key(&bigint(ten_pow_23)),
+    );
+    assert_ne!(
+        super::python_scalar_key(&bigint(ten_pow_23)),
+        super::python_scalar_key(&bigint("100000000000000000000001")),
+    );
+}

{deepdiff_rs-0.10.0 → deepdiff_rs-0.11.0}/crates/onix-core/src/path.rs

@@ -609,8 +609,11 @@ fn number_repr(n: &Number) -> String {
     if let Some(i) = n.as_i64() {
         return i.to_string();
     }
-    n.as_u64()
-        .expect("a non-float Number always has an i64 or u64 representation")
+    if let Some(u) = n.as_u64() {
+        return u.to_string();
+    }
+    n.as_big()
+        .expect("a non-float Number is an i64, a u64, or an arbitrary-precision integer")
         .to_string()
 }