deepdiff-rs 0.5.3__tar.gz → 0.5.5__tar.gz

{deepdiff_rs-0.5.3 → deepdiff_rs-0.5.5}/Cargo.lock

@@ -127,7 +127,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "onix-cli"
-version = "0.5.3"
+version = "0.5.5"
 dependencies = [
  "onix-core",
  "serde_json",
@@ -135,7 +135,7 @@ dependencies = [
 
 [[package]]
 name = "onix-core"
-version = "0.5.3"
+version = "0.5.5"
 dependencies = [
  "proptest",
  "serde",
@@ -145,7 +145,7 @@ dependencies = [
 
 [[package]]
 name = "onix-py"
-version = "0.5.3"
+version = "0.5.5"
 dependencies = [
  "onix-core",
  "pyo3",

{deepdiff_rs-0.5.3 → deepdiff_rs-0.5.5}/Cargo.toml

@@ -3,7 +3,7 @@ resolver = "3"
 members = ["crates/onix-core", "crates/onix-py"]
 
 [workspace.package]
-version = "0.5.3"
+version = "0.5.5"
 edition = "2024"
 license = "MIT"
 

{deepdiff_rs-0.5.3 → deepdiff_rs-0.5.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: deepdiff-rs
-Version: 0.5.3
+Version: 0.5.5
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Rust
 Classifier: License :: OSI Approved :: MIT License

{deepdiff_rs-0.5.3 → deepdiff_rs-0.5.5}/crates/onix-core/src/ignore_order/memo.rs

@@ -179,6 +179,15 @@ pub(crate) struct IgnoreOrderMemo {
     /// [`BTreeMap`] for the same collision-immunity reason as `node_table`.
     member_content: RefCell<BTreeMap<MemberContent, RepId>>,
     enabled: bool,
+    /// Total number of times [`Self::put`] has actually run — every distance
+    /// *recomputation*, not just the distinct entries it leaves behind (a
+    /// repeated `put` for a key already in `cache` overwrites the entry
+    /// rather than growing [`Self::cache_len`], so this is the only signal
+    /// that would rise if a caller re-derived a distance it should have
+    /// gotten from [`Self::get`] instead). Test-only: see
+    /// [`Self::put_count`]'s own doc for what it guards.
+    #[cfg(test)]
+    puts: std::cell::Cell<usize>,
 }
 
 impl IgnoreOrderMemo {
@@ -191,6 +200,8 @@ impl IgnoreOrderMemo {
             node_table: RefCell::new(BTreeMap::new()),
             member_content: RefCell::new(BTreeMap::new()),
             enabled: true,
+            #[cfg(test)]
+            puts: std::cell::Cell::new(0),
         }
     }
 
@@ -206,6 +217,7 @@ impl IgnoreOrderMemo {
             node_table: RefCell::new(BTreeMap::new()),
             member_content: RefCell::new(BTreeMap::new()),
             enabled: false,
+            puts: std::cell::Cell::new(0),
         }
     }
 
@@ -229,6 +241,18 @@ impl IgnoreOrderMemo {
         self.cache.borrow().len()
     }
 
+    /// The number of times [`Self::put`] has run — see that counter's own
+    /// field doc. A deep, self-similar nesting whose distance is computed
+    /// through this cache must see this count grow *linearly* with depth: a
+    /// cache whose lookup is broken (every candidate pair is recomputed
+    /// rather than reused) instead compounds `~2x` per level, which this
+    /// deterministically catches without timing anything — see
+    /// `super::tests::deep_nested_ignore_order_memoizes_distance_computations_linearly`.
+    #[cfg(test)]
+    pub(crate) fn put_count(&self) -> usize {
+        self.puts.get()
+    }
+
     /// The cached distance for `key`, if present.
     pub(crate) fn get(&self, key: &DistanceKey) -> Option<f64> {
         self.cache.borrow().get(key).copied()
@@ -236,6 +260,8 @@ impl IgnoreOrderMemo {
 
     /// Records `value` for `key` (moving the already-cloned key in).
     pub(crate) fn put(&self, key: DistanceKey, value: f64) {
+        #[cfg(test)]
+        self.puts.set(self.puts.get() + 1);
         self.cache.borrow_mut().insert(key, value);
     }
 

{deepdiff_rs-0.5.3 → deepdiff_rs-0.5.5}/crates/onix-core/src/ignore_order/tests.rs

@@ -1332,15 +1332,25 @@ proptest! {
     }
 }
 
-#[test]
-fn deep_nested_ignore_order_completes_quickly_with_memoization() {
-    use std::time::Instant;
-
-    // A single-element nested list of depth `d` is `d` nodes — tiny, legal
-    // input. Unmemoized, `ignore_order` pairing re-diffs each level twice
-    // (once to score the pair's distance, once to record it), compounding to
-    // `~2x` cost per level: this used to take ~1-2s at depth 20 and hang for
-    // tens of seconds by depth 25. The distance memo collapses it to linear.
+/// A memoized deep-nested `ignore_order` diff must recompute each level's
+/// pairing distance exactly once — replaces a wall-clock guard that was
+/// flaky under parallel CI (issue #33). Measured directly instead, with no
+/// clock in the loop:
+///
+/// [`super::pairing::compute_pairs`] recomputes a container pair's distance
+/// only on an [`IgnoreOrderMemo`] cache miss (see [`IgnoreOrderMemo::put`]'s
+/// field doc), so [`IgnoreOrderMemo::put_count`] — every recomputation, not
+/// just the distinct entries a repeated `put` leaves behind — is a direct,
+/// deterministic stand-in for the "re-diffs each level twice, compounding
+/// `~2x` per level" cost the timing bound used to catch. A single-element
+/// nested list of depth `d` has exactly `d - 1` container-pair candidates
+/// (the outermost wrapper is never itself paired against anything), so a
+/// working memo leaves `put_count() == depth - 1`; confirmed by temporarily
+/// forcing [`IgnoreOrderMemo::get`] to always return `None` and re-running
+/// this test, which then reported `2^(depth - 1) - 1` puts (8,191 at depth
+/// 14 alone, where the working memo leaves 13) instead of failing on a clock.
+#[test]
+fn deep_nested_ignore_order_memoizes_distance_computations_linearly() {
     let opts = DiffOptions {
         ignore_order: true,
         max_depth: 100_000,
@@ -1353,28 +1363,31 @@ fn deep_nested_ignore_order_completes_quickly_with_memoization() {
         crate::value::Value::from(value)
     };
 
-    let (a, b) = (build(20, 1), build(20, 2));
-    let started = Instant::now();
-    let report = crate::diff::diff_with_options(&a, &b, &opts).expect("depth 20 diffs");
-    let depth_20 = started.elapsed();
-    assert!(
-        !report.is_empty(),
-        "depth-20 unequal input must report a change"
-    );
-    assert!(
-        depth_20.as_millis() < 500,
-        "depth-20 nested ignore_order took {depth_20:?}, over the 500ms bound"
-    );
+    let recomputations_at = |depth: usize| -> usize {
+        let (a, b) = (build(depth, 1), build(depth, 2));
+        let memo = IgnoreOrderMemo::new();
+        let report = crate::diff::diff_with_options_memo(&a, &b, &opts, &memo)
+            .unwrap_or_else(|e| panic!("depth-{depth} diffs: {e}"));
+        assert!(
+            !report.is_empty(),
+            "depth-{depth} unequal input must report a change"
+        );
+        memo.put_count()
+    };
 
-    // Depth 25 previously hung (>30s); it must now complete at all.
-    let (a, b) = (build(25, 1), build(25, 2));
-    let started = Instant::now();
-    let _ = crate::diff::diff_with_options(&a, &b, &opts).expect("depth 25 diffs");
-    let depth_25 = started.elapsed();
-    assert!(
-        depth_25.as_secs() < 5,
-        "depth-25 nested ignore_order took {depth_25:?}, over the 5s bound"
-    );
+    // Depth 25 previously hung for tens of seconds unmemoized; both depths
+    // are kept so a regression shows up well before it would need to.
+    for depth in [20usize, 25] {
+        let puts = recomputations_at(depth);
+        assert_eq!(
+            puts,
+            depth - 1,
+            "depth-{depth} nested ignore_order recomputed {puts} distances, \
+             expected exactly {} (one per level) — the distance memo is not \
+             being hit",
+            depth - 1
+        );
+    }
 }
 
 // --- tuples under ignore_order -------------------------------------------
@@ -2487,100 +2500,88 @@ fn a_deeply_nested_set_member_hashes_and_compares_without_native_recursion() {
         .expect("set-member hashing and comparison complete on a small stack");
 }
 
-/// Interning `K` set members must stay near linear in `K`, never quadratic.
-///
-/// - **Float bit-pattern collision (`SetFloat*`, `ListFloat`) — the runtime
-///   guard.** A float carrying an integer or half-integer has ~50 trailing zero
-///   bits; on the `FxHash` tables the crate keeps (e.g. `HashedList` for an
-///   `ignore_order` list), a run of them collides unless the float bits are
-///   mixed first ([`crate::lcs::mix_float_bits`]). Reverting the mixing turns
-///   the float rows here red, so they genuinely guard it.
-/// - **Benign near-linearity (`SetIntPair`).** A run of plain int 2-tuples
-///   exercises the set-member tables on the *shape* of the crafted
-///   hash-flooding attack, but does **not** stand in for the attack: sequential
-///   ints do not collide under `FxHash`, so this row stays green even if the
-///   tables were reverted to `FxHash`. The adversarial hazard — the tables are
-///   keyed by attacker-controlled content and reached with the default
-///   `ignore_order=false` — is guarded at the **type level** instead: the tables
-///   are [`BTreeMap`]s, and [`super::hash::MemberHashKey`]/
-///   [`super::hash::MemberContent`] no longer derive `Hash`, so putting them
-///   back on an `FxHash` map fails to compile (`E0599`). This row is a plain
-///   regression check that the `BTreeMap` path itself scales.
+/// Interning `K` set/list members must never collapse onto one hash bucket —
+/// replaces a wall-clock `K -> 2K` diff-time ratio that was flaky under
+/// parallel CI (issue #33). Measured directly instead, with no diff and no
+/// clock in the loop:
 ///
-/// Each asserts the `K -> 2K` diff-time ratio stays under `3.0` — a linear (or
-/// `n log n`) pass is `~2x`, a quadratic one `~4x`. Sized to run well under a
-/// second.
-#[test]
-fn set_and_list_member_interning_scales_near_linearly() {
-    use crate::value::Number;
-
-    #[derive(Clone, Copy)]
-    enum Shape {
-        SetIntFloat,
-        SetHalfFloat,
-        SetIntPair,
-        ListFloat,
-    }
-
-    let f = |x: f64| CValue::Number(Number::from_f64(x).expect("finite"));
-    let i = |n: i64| CValue::Number(Number::from_i64(n));
-
-    let build = |k: usize, shape: Shape| -> (CValue, CValue, DiffOptions) {
-        #[allow(clippy::cast_precision_loss)]
-        let member = |n: usize| -> CValue {
-            let n_i = i64::try_from(n).expect("test sizes fit i64");
-            match shape {
-                Shape::SetIntFloat => CValue::Tuple(vec![f(n as f64)].into_boxed_slice()),
-                Shape::SetHalfFloat => CValue::Tuple(vec![f(n as f64 + 0.5)].into_boxed_slice()),
-                Shape::SetIntPair => CValue::Tuple(vec![i(n_i), i(n_i)].into_boxed_slice()),
-                Shape::ListFloat => f(n as f64),
-            }
-        };
-        let side = |extra: bool| {
-            let mut items: Vec<CValue> = (0..k).map(member).collect();
-            if extra {
-                items.push(CValue::Str("sentinel".to_string().into_boxed_str()));
-            }
-            match shape {
-                Shape::ListFloat => CValue::Array(items.into_boxed_slice()),
-                _ => CValue::Set(SetItems::new(items)),
-            }
-        };
-        let opts = DiffOptions {
-            ignore_order: matches!(shape, Shape::ListFloat),
-            ..DiffOptions::default()
-        };
-        // Differ by one member so the whole-value fast path can't short-circuit.
-        (side(false), side(true), opts)
-    };
+/// [`super::hash::item_key`]'s `Float` arm hashes through
+/// [`crate::lcs::mix_float_bits`] before the bits ever reach one of this
+/// module's `FxHash` tables (e.g. [`super::hash::HashedList`], the table an
+/// `ignore_order` list's items are matched through) — an integral or
+/// half-integer float's raw bit pattern shares dozens of trailing zero bits
+/// over the range this test uses, and `hashbrown` picks a table's bucket
+/// from a hash's *low* bits (see `mix_float_bits`'s own doc), so an unmixed
+/// run of them would all name the same bucket and degrade every lookup to a
+/// linear scan. This hashes real [`super::hash::ItemKey`]s through the real
+/// [`FxHasher`] `HashedList` itself uses and checks the low 32 bits (a proxy
+/// wide enough that no realistic table capacity at these sizes reads outside
+/// it) of `K`, and separately `2K`, keys: with the mixing intact they land in
+/// (almost) pairwise-distinct buckets, so the distinct count grows linearly
+/// with `K`; reverting or breaking the mix collapses the whole run onto a
+/// handful of buckets regardless of `K`, which the linearity assertion below
+/// catches immediately (confirmed by temporarily reverting `hash.rs`'s
+/// `Float` arm to hash the raw bits and re-running this test, which then
+/// fails on both the distinctness and the growth checks).
+#[test]
+fn float_hash_buckets_stay_distinct_and_grow_linearly_with_member_count() {
+    use std::collections::HashSet;
+    use std::hash::{Hash, Hasher};
 
-    let best_diff = |k: usize, shape: Shape| -> f64 {
-        let (a, b, opts) = build(k, shape);
-        let _ = crate::diff::diff_with_options(&a, &b, &opts).expect("diffs cleanly"); // warm
-        (0..5)
-            .map(|_| {
-                let start = std::time::Instant::now();
-                let _ = crate::diff::diff_with_options(&a, &b, &opts).expect("diffs cleanly");
-                start.elapsed().as_secs_f64()
-            })
-            .fold(f64::INFINITY, f64::min)
+    let low_bucket = |n: usize, half: bool| -> u32 {
+        #[allow(
+            clippy::cast_precision_loss,
+            reason = "n stays well under 2^53 here; exactness is not the point of the probe"
+        )]
+        let value = n as f64 + if half { 0.5 } else { 0.0 };
+        let key = item_key(&json!(value));
+        let mut hasher = FxHasher::default();
+        key.hash(&mut hasher);
+        #[allow(
+            clippy::cast_possible_truncation,
+            reason = "only the low 32 bits are the bucket proxy under test"
+        )]
+        {
+            hasher.finish() as u32
+        }
     };
 
-    for (name, shape) in [
-        ("set int-float", Shape::SetIntFloat),
-        ("set half-float", Shape::SetHalfFloat),
-        ("set int-pair", Shape::SetIntPair),
-        ("ignore_order list float", Shape::ListFloat),
-    ] {
+    for (name, half) in [("integral", false), ("half-integer", true)] {
         let k = 10_000;
-        let t1 = best_diff(k, shape);
-        let t2 = best_diff(2 * k, shape);
-        let ratio = t2 / t1;
-        assert!(
-            ratio < 3.0,
-            "{name}: K->2K ratio {ratio:.2} (t1={t1:.4}s t2={t2:.4}s) is super-linear — \
-             keys are colliding in their interning table"
-        );
+        let distinct_count = |count: usize| -> usize {
+            (0..count)
+                .map(|n| low_bucket(n, half))
+                .collect::<HashSet<_>>()
+                .len()
+        };
+
+        let distinct_1x = distinct_count(k);
+        let distinct_2x = distinct_count(2 * k);
+
+        #[allow(
+            clippy::cast_precision_loss,
+            reason = "k is a small test constant; the ratio only needs two significant figures"
+        )]
+        {
+            assert!(
+                (distinct_1x as f64) > 0.99 * (k as f64),
+                "{name} floats: only {distinct_1x}/{k} distinct hash buckets at K \
+                 — keys are colliding in their interning table"
+            );
+            assert!(
+                (distinct_2x as f64) > 0.99 * (2.0 * k as f64),
+                "{name} floats: only {distinct_2x}/{} distinct hash buckets at 2K \
+                 — keys are colliding in their interning table",
+                2 * k
+            );
+
+            let ratio = distinct_2x as f64 / distinct_1x as f64;
+            assert!(
+                (1.9..=2.1).contains(&ratio),
+                "{name} floats: distinct-bucket count did not grow linearly \
+                 with K (K->2K ratio {ratio:.3}, distinct_1x={distinct_1x}, distinct_2x={distinct_2x})"
+            );
+        }
     }
 }