lzcomplexity 1.0.2__tar.gz → 2.0.0__tar.gz

{lzcomplexity-1.0.2 → lzcomplexity-2.0.0}/Cargo.lock

@@ -238,7 +238,7 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
 
 [[package]]
 name = "lzcomplexity-cli"
-version = "1.0.2"
+version = "2.0.0"
 dependencies = [
  "clap",
  "flate2",
@@ -250,7 +250,7 @@ dependencies = [
 
 [[package]]
 name = "lzcomplexity-core"
-version = "1.0.2"
+version = "2.0.0"
 dependencies = [
  "cdivsufsort",
  "rand",
@@ -260,7 +260,7 @@ dependencies = [
 
 [[package]]
 name = "lzcomplexity-py"
-version = "1.0.2"
+version = "2.0.0"
 dependencies = [
  "lzcomplexity-core",
  "pyo3",

lzcomplexity-2.0.0/Cargo.toml

@@ -0,0 +1,50 @@
+[workspace]
+resolver = "2"
+members = ["crates/lzcomplexity-core", "crates/lzcomplexity-py"]
+
+[workspace.package]
+version = "2.0.0" # x-release-please-version
+edition = "2021"
+rust-version = "1.75"
+license = "MIT"
+authors = [
+    "Efrén Aragón Pérez <efrenaragon96@gmail.com>",
+    "Daniel Estévez Moya <kemossabee@gmail.com>",
+    "Ernesto Estévez Rams <estevez@fisica.uh.cu>",
+]
+repository = "https://github.com/pleros-ai/lzcomplexity"
+
+[workspace.dependencies]
+rayon = "1.10"
+# PINNED EXACTLY, AND NOT AS A SUPPLY-CHAIN MEASURE.
+#
+# The block shuffle behind `emc` draws from `ChaCha8Rng::seed_from_u64` and
+# `Rng::gen_range`, so the exact byte stream these two crates produce is part of
+# this library's observable output: it decides which surrogates are drawn, and
+# therefore the value `emc` returns. Two things depend on it.
+#
+#   1. `emc` is documented as reproducible for a given input on any machine at any
+#      thread count. See docs/project/determinism.md.
+#   2. The C++ backend on `main` reproduces these crates' *internal* algorithms by
+#      hand -- rand_core's PCG32 seed expansion, ChaCha8 with a 64-word buffer, and
+#      gen_range's `(range << leading_zeros) - 1` rejection zone -- so that both
+#      backends return identical values. See modules/core/inc/lz/rng.h there, and
+#      docs/project/cpp-parity.md.
+#
+# A minor-version bump of either crate may change that stream without any API
+# change and without any warning, which would silently break both properties. Do
+# not relax these to caret requirements. To upgrade deliberately: re-run the C++
+# `rng_parity` test (ctest) against the new stream, regenerate its vectors if it
+# fails, and treat the resulting `emc` change as a breaking change.
+rand = "=0.8.6"
+rand_chacha = "=0.3.1"
+cdivsufsort = "2.0"
+clap = { version = "4.5", features = ["derive"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+flate2 = "1.0"
+
+[profile.release]
+lto = "thin"
+codegen-units = 1
+opt-level = 3

{lzcomplexity-1.0.2 → lzcomplexity-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lzcomplexity
-Version: 1.0.2
+Version: 2.0.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: Environment :: Console
 Classifier: Programming Language :: Python :: 3

lzcomplexity-2.0.0/crates/lzcomplexity-core/src/shuffle.rs

@@ -0,0 +1,358 @@
+//! Shuffle-based complexity measures.
+
+use rayon::prelude::*;
+
+use crate::lz76::{lz76_factorization, LzArgs, LzShuffle};
+use crate::sequence::{shuffle_copy_seeded, Sequence};
+use crate::NO_ALPHABET;
+
+/// Estimate the "naive M" satisfying `N = M * 2^M`, mirroring `max_block_size` in C++.
+pub fn max_block_size(size: usize) -> usize {
+    if size < 10 {
+        return 1;
+    }
+    let mut m: usize = 10;
+    let mut m_old: usize = 1;
+    let mut count: u32 = 0;
+    while count < 100 {
+        let est = ((size as f64) / (m_old as f64)).ln() / 2f64.ln();
+        m = est.round() as usize;
+        count += 1;
+        if m == m_old {
+            break;
+        }
+        m_old = m;
+    }
+    m
+}
+
+/// Compute the random-shuffle factorization vector and `mm`. Mirrors
+/// `ShuffleFactorization` from the C++.
+pub fn shuffle_factorization(seq: &Sequence, args: &LzArgs) -> (Vec<i32>, usize) {
+    let mut mm: i64 = args.block_size as i64;
+    if mm <= 0 {
+        mm = max_block_size(seq.len()) as i64;
+        if seq.len() > 50 {
+            mm += 10;
+        }
+    }
+    let mm = mm as usize;
+
+    // Index 0 unused, indices 1..=mm hold the per-block-size complexity.
+    let mut res: Vec<i32> = vec![0; mm + 3];
+
+    // Hash the sequence content ONCE; each per-block seed only differs by the
+    // `idx` term (bit-identical seeds → identical shuffles → identical EMC).
+    let base = fnv1a(seq.as_bytes());
+    let computed: Vec<(usize, i32)> = (1..=mm)
+        .into_par_iter()
+        .map(|idx| {
+            let seed = seed_from_base(base, idx);
+            let rand_seq = shuffle_copy_seeded(seq, idx as u32, (seq.len() / 2) as u32, seed);
+            let c = lz76_factorization(&rand_seq, args);
+            (idx, c as i32)
+        })
+        .collect();
+    for (idx, c) in computed {
+        res[idx] = c;
+    }
+    (res, mm)
+}
+
+/// Effective measure complexity via the block-entropy estimator.
+///
+/// For each block size `l`, `h_rand[l]` is `C_LZ(u^{RS(l)})`, the LZ complexity
+/// of `u` block-shuffled at scale `l`. The surrogate is (idealised) an i.i.d.
+/// stream of `l`-blocks whose entropy rate is `H(l)/l`, so multiplying back
+/// recovers a block-entropy estimate `H_l = l · C_LZ(u^{RS(l)}) · g`, with
+/// `g = log_k(N)/N` and `ĥ = C_LZ(u) · g` the entropy density of `u`. That
+/// gives one *finite-scale excess entropy* per block size — the subextensive
+/// part of the block entropy at scale `l`:
+///
+/// ```text
+/// Ê(l) = H_l − l·ĥ = l · g · ( C_LZ(u^{RS(l)}) − C_LZ(u) )
+/// ```
+///
+/// Theory pins down the shape of that ladder: `E(l) = Σ_{j≤l} [h(j) − h]` with
+/// every increment `h(j) − h ≥ 0`, so `E(l)` is non-negative and non-decreasing
+/// in `l` and rises to `E` (Crutchfield & Feldman, *Chaos* 13:25 (2003),
+/// Lemma 1 and Eq. 52). The raw ladder violates both properties, because each
+/// `Ê(l)` is an *independent* surrogate draw rather than a slice of one
+/// consistent block-entropy curve. We therefore project it onto that cone —
+/// non-negative isotonic regression, see [`non_negative_isotonic`] — and read
+/// the answer off the projection:
+///
+/// ```text
+/// summands[l−1] = Ê_fit(l) − Ê_fit(l−1)   ≥ 0     (the estimate of h(l) − h)
+/// Ê             = Ê_fit(mm) = Σ summands  ≥ 0
+/// ```
+///
+/// Every block size informs the fit, and the returned scalar can no longer go
+/// negative — a mutual information never does.
+///
+/// # Why not sum the raw differences
+///
+/// Versions ≤ 1.0.1 summed `(H_l − H_{l−1}) − ĥ` directly. That is the same
+/// textbook identity written the other way round, `Σ_l [h(l) − h] = H(L) − L·h`,
+/// so the `H_l` telescoped and the total collapsed *exactly* to
+/// `mm · g · (C_LZ(u^{RS(mm)}) − C_LZ(u))`: only the largest block size reached
+/// the scalar, the other `mm − 1` factorizations cancelled algebraically, and a
+/// single unlucky surrogate draw could push the result below zero. Projecting
+/// first is what couples the scales together.
+///
+/// `multi_information` stays the *raw* `l = 1` contrast `H_1 − ĥ`, unchanged
+/// from earlier versions: it is a separate quantity (the multi-information rate
+/// of the source), not a term of the sum, and it is the best-calibrated number
+/// this estimator produces.
+pub fn shuffle_entropy_calculation(
+    seq: &Sequence,
+    args: &LzArgs,
+    complexity: i32,
+    h_rand: &[i32],
+    mm: usize,
+) -> LzShuffle {
+    let mut result = LzShuffle {
+        max_block_size: mm as i32,
+        ..Default::default()
+    };
+    if args.get_shuffle_terms {
+        result.summands = vec![0.0; mm];
+    }
+    let log_base = if args.log_base == NO_ALPHABET {
+        seq.alphabet_size().max(2)
+    } else {
+        args.log_base.max(2)
+    } as f64;
+
+    let n = seq.len() as f64;
+    if n <= 1.0 || mm == 0 {
+        return result;
+    }
+    // g = log_k(N) / N, so ĥ = C_LZ(u)·g and H_l = l·C_LZ(u^{RS(l)})·g.
+    let g = (n.ln() / log_base.ln()) / n;
+    let h_hat = complexity as f64 * g; // entropy density of the original sequence
+
+    // The raw ladder Ê(l) = H_l − l·ĥ, one finite-scale excess entropy per
+    // block size. `h_rand` is 1-indexed, so `len` is how many scales it holds.
+    let len = mm.min(h_rand.len().saturating_sub(1));
+    if len == 0 {
+        return result;
+    }
+    let raw: Vec<f64> = (1..=len)
+        .map(|l| l as f64 * (h_rand[l] as f64 * g - h_hat))
+        .collect();
+
+    // The l = 1 contrast, before any projection.
+    result.multi_information = raw[0];
+
+    // Project onto {non-negative, non-decreasing} and read off the increments.
+    let fitted = non_negative_isotonic(&raw);
+    let mut prev = 0.0f64; // Ê_fit(0) = 0
+    for l in 1..=len {
+        let term = fitted[l - 1] - prev;
+        if args.get_shuffle_terms {
+            result.summands[l - 1] = term;
+        }
+        prev = fitted[l - 1];
+    }
+    result.emc_value = prev; // = Ê_fit(len)
+    result
+}
+
+/// L2 projection of `y` onto the cone of non-negative, non-decreasing sequences.
+///
+/// The isotonic part is the pool-adjacent-violators algorithm (Ayer et al.,
+/// *Ann. Math. Statist.* 26:641 (1955)): walk left to right, and whenever the
+/// newest block averages below its predecessor, merge the two and re-check.
+/// Blocks are stored as `(sum, count)` pairs, so this is O(n) overall. The
+/// non-negative projection is then the pointwise positive part, which preserves
+/// monotonicity.
+fn non_negative_isotonic(y: &[f64]) -> Vec<f64> {
+    let mut sums: Vec<f64> = Vec::with_capacity(y.len());
+    let mut counts: Vec<usize> = Vec::with_capacity(y.len());
+    for &v in y {
+        sums.push(v);
+        counts.push(1);
+        while sums.len() > 1 {
+            let k = sums.len();
+            let last = sums[k - 1] / counts[k - 1] as f64;
+            let prev = sums[k - 2] / counts[k - 2] as f64;
+            if prev <= last {
+                break;
+            }
+            sums[k - 2] += sums[k - 1];
+            counts[k - 2] += counts[k - 1];
+            sums.pop();
+            counts.pop();
+        }
+    }
+    let mut out = Vec::with_capacity(y.len());
+    for (s, c) in sums.iter().zip(counts.iter()) {
+        let level = (s / *c as f64).max(0.0);
+        out.extend(std::iter::repeat(level).take(*c));
+    }
+    out
+}
+
+/// `lz76RandomShuffleComplexity` — full pipeline.
+pub fn lz76_random_shuffle_complexity(seq: &Sequence, args: &LzArgs) -> LzShuffle {
+    lz76_random_shuffle_complexity_with(seq, args, lz76_factorization(seq, args) as i32)
+}
+
+/// Same, but with an already-computed original-sequence complexity (so callers
+/// that already have it — e.g. the `lz76` driver — skip one factorization).
+pub fn lz76_random_shuffle_complexity_with(
+    seq: &Sequence,
+    args: &LzArgs,
+    complexity: i32,
+) -> LzShuffle {
+    let (h_rand, mm) = shuffle_factorization(seq, args);
+    shuffle_entropy_calculation(seq, args, complexity, &h_rand, mm)
+}
+
+/// `lz76PairedShuffleComplexity` — shuffle on the merged (Z) sequence.
+pub fn lz76_paired_shuffle_complexity(seq: &Sequence, args: &LzArgs) -> LzShuffle {
+    let mid = seq.len() / 2;
+    let (past, future) = seq.split_at(mid);
+    let merged = merge_sequences(&past, &future);
+    let complexity = lz76_factorization(&merged, args) as i32;
+    let (h_rand, mm) = shuffle_factorization(&merged, args);
+    shuffle_entropy_calculation(&merged, args, complexity, &h_rand, mm)
+}
+
+/// Mirror of `internal::MergeSequences` — element-wise pairing into a new alphabet.
+pub fn merge_sequences(s1: &Sequence, s2: &Sequence) -> Sequence {
+    let max_iter = s1.len().min(s2.len());
+    let mut buf = Vec::with_capacity(max_iter);
+    let mut lookup = vec![0u8; 65536];
+    let mut next_symbol: u8 = b'0';
+    let a = s1.as_bytes();
+    let b = s2.as_bytes();
+    for i in 0..max_iter {
+        let key = ((a[i] as u16) << 8) | (b[i] as u16);
+        let entry = &mut lookup[key as usize];
+        if *entry == 0 {
+            *entry = next_symbol;
+            next_symbol = next_symbol.wrapping_add(1);
+        }
+        buf.push(*entry);
+    }
+    Sequence::from_bytes_with_alphabet(buf, s1.alphabet_size().saturating_mul(s2.alphabet_size()))
+}
+
+/// FNV-1a hash of the sequence bytes (the content-dependent part of the seed).
+fn fnv1a(bytes: &[u8]) -> u64 {
+    let mut h: u64 = 14695981039346656037; // FNV offset basis (64-bit)
+    for &c in bytes {
+        h ^= c as u64;
+        h = h.wrapping_mul(1099511628211);
+    }
+    h
+}
+
+/// Derive the per-block-size seed from the (once-computed) content hash.
+fn seed_from_base(base: u64, idx: usize) -> u64 {
+    base ^ ((idx as u64).wrapping_mul(0x9E3779B97F4A7C15))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A binary sequence (alphabet 2 -> log_base auto = 2) of length 100, plus
+    /// `g = log_2(100)/100`. The block-entropy arithmetic only reads the length
+    /// and the alphabet, so the content is irrelevant to these tests.
+    fn fixture() -> (Sequence, LzArgs, f64) {
+        let mut data = vec![b'0'; 50];
+        data.extend(std::iter::repeat(b'1').take(50));
+        let seq = Sequence::from_bytes(data);
+        let mut args = LzArgs::new();
+        args.get_shuffle_terms = true;
+        let n = 100.0f64;
+        (seq, args, (n.ln() / 2f64.ln()) / n)
+    }
+
+    #[test]
+    fn emc_projects_the_excess_ladder() {
+        let (seq, args, g) = fixture();
+        // c_1 = 20, c_2 = 15, c_3 = 12 against C_LZ(u) = 10, so the raw ladder
+        // Ê(l) = l·g·(c_l − 10) is [10g, 10g, 6g] — non-monotone at l = 3.
+        let r = shuffle_entropy_calculation(&seq, &args, 10, &[0, 20, 15, 12], 3);
+
+        // Pooling all three violators gives one block at (10 + 10 + 6)/3 g.
+        let level = (26.0 / 3.0) * g;
+        assert!((r.emc_value - level).abs() < 1e-12, "{}", r.emc_value);
+        assert!((r.summands[0] - level).abs() < 1e-12);
+        assert_eq!(r.summands[1], 0.0);
+        assert_eq!(r.summands[2], 0.0);
+        // multi_information is the *raw* l = 1 contrast, not the fitted one.
+        assert!((r.multi_information - 10.0 * g).abs() < 1e-12);
+    }
+
+    #[test]
+    fn emc_keeps_the_closed_form_when_the_ladder_is_monotone() {
+        let (seq, args, g) = fixture();
+        // c_l = 11 for every l gives Ê(l) = l·g, already strictly increasing,
+        // so the projection is the identity and the total is the old
+        // closed form mm·g·(c_mm − C_LZ(u)).
+        let r = shuffle_entropy_calculation(&seq, &args, 10, &[0, 11, 11, 11], 3);
+
+        assert!((r.emc_value - 3.0 * g).abs() < 1e-12, "{}", r.emc_value);
+        for s in &r.summands {
+            assert!((s - g).abs() < 1e-12, "{s}");
+        }
+    }
+
+    #[test]
+    fn emc_is_never_negative() {
+        let (seq, args, g) = fixture();
+        // Every surrogate less complex than the original: the raw ladder is
+        // [−10g, −20g, −30g], entirely below zero.
+        let r = shuffle_entropy_calculation(&seq, &args, 20, &[0, 10, 10, 10], 3);
+
+        assert_eq!(r.emc_value, 0.0);
+        assert!(r.summands.iter().all(|&s| s == 0.0), "{:?}", r.summands);
+        // The diagnostic still shows the negative contrast.
+        assert!((r.multi_information + 10.0 * g).abs() < 1e-12);
+    }
+
+    #[test]
+    fn isotonic_pools_adjacent_violators() {
+        // 3.0 then 2.0 violates monotonicity; they pool to 2.5.
+        assert_eq!(
+            non_negative_isotonic(&[1.0, 3.0, 2.0, 4.0]),
+            vec![1.0, 2.5, 2.5, 4.0]
+        );
+        // Already isotonic: unchanged apart from the non-negativity clamp.
+        assert_eq!(non_negative_isotonic(&[-1.0, 0.5]), vec![0.0, 0.5]);
+        // A strictly decreasing input pools into one block.
+        assert_eq!(non_negative_isotonic(&[6.0, 3.0, 0.0]), vec![3.0, 3.0, 3.0]);
+        assert_eq!(non_negative_isotonic(&[]), Vec::<f64>::new());
+    }
+
+    #[test]
+    fn summands_are_non_negative_and_sum_to_the_value() {
+        // A real end-to-end run: period-8 motif, n = 2000.
+        let data: Vec<u8> = b"00010111".iter().copied().cycle().take(2000).collect();
+        let seq = Sequence::from_bytes(data);
+        let mut args = LzArgs::new();
+        args.get_shuffle_terms = true;
+        let r = lz76_random_shuffle_complexity(&seq, &args);
+
+        assert_eq!(r.summands.len(), r.max_block_size as usize);
+        assert!(r.emc_value > 0.0, "expected structure, got {}", r.emc_value);
+        assert!(r.summands.iter().all(|&s| s >= 0.0), "{:?}", r.summands);
+        let total: f64 = r.summands.iter().sum();
+        assert!(
+            (total - r.emc_value).abs() < 1e-12,
+            "{total} vs {}",
+            r.emc_value
+        );
+        // The ladder is non-decreasing, so partial sums climb to the total.
+        let mut acc = 0.0;
+        for &s in &r.summands {
+            acc += s;
+            assert!(acc <= r.emc_value + 1e-12);
+        }
+    }
+}

{lzcomplexity-1.0.2 → lzcomplexity-2.0.0}/crates/lzcomplexity-py/src/lib.rs

@@ -177,9 +177,28 @@ fn h(
 /// content. Higher values mean more structure than random noise of the same
 /// length and alphabet.
 ///
+/// For each block size ``l = 1 ... max_block_size`` the sequence is shuffled in
+/// blocks of length ``l`` and re-factorized, which gives one estimate of the
+/// *excess entropy accumulated up to scale l*. Excess entropy is a mutual
+/// information between the past and the future, so that ladder must be
+/// non-negative and non-decreasing in ``l``; the raw estimates are not, because
+/// each rests on its own surrogate draw. They are projected onto that shape
+/// (non-negative isotonic regression) before the terms are read off, so
+/// **every returned value is non-negative** and every block size informs the
+/// total.
+///
 /// The shuffle is seeded deterministically from the sequence content, so
 /// repeated calls on the same input produce identical results.
 ///
+/// .. warning::
+///
+///    Treat the value as an **ordinal index at fixed sequence length**, not as
+///    a bit count. It is biased upward (roughly 2-3x the analytic excess
+///    entropy for order-1 Markov sources) and it scales with
+///    ``max_block_size``, which itself grows with length -- so never compare
+///    values across sequences of different length. Report it alongside
+///    :func:`h`.
+///
 /// Parameters
 /// ----------
 /// seq : str | bytes | list[int] | list[str] | Iterable[int]
@@ -198,9 +217,15 @@ fn h(
 /// Returns
 /// -------
 /// tuple[float, list[float]]
-///     ``(emc_value, summands)`` where ``emc_value`` is the effective measure
-///     complexity and ``summands`` is the list of per-block-size terms whose
-///     sum is ``emc_value``. ``summands[0]`` is the multi-information term.
+///     ``(emc_value, summands)``. ``emc_value`` is the effective measure
+///     complexity, always ``>= 0.0``. ``summands`` has one entry per block
+///     size, ``summands[l-1]`` estimating the scale-``l`` conditional-entropy
+///     excess ``h(l) - h``; every entry is ``>= 0.0`` and they sum to
+///     ``emc_value``. Their running sum is the excess entropy captured up to
+///     each scale, so a profile that keeps climbing past the first few entries
+///     indicates long-range or hierarchical correlation. Zero entries mean
+///     neighbouring scales were pooled by the projection, not that the scale
+///     was skipped.
 ///
 /// Examples
 /// --------

{lzcomplexity-1.0.2 → lzcomplexity-2.0.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "lzcomplexity"
-version = "1.0.2" # x-release-please-version
+version = "2.0.0" # x-release-please-version
 description = "LempelZiv-76 complexity engine. Suited for complexity analysis of time series."
 authors = [
     { name = "Efrén Aragón Pérez", email = "efrenaragon96@gmail.com" },

{lzcomplexity-1.0.2 → lzcomplexity-2.0.0}/python/lzcomplexity/__init__.pyi

@@ -54,8 +54,12 @@ def emc(
 ) -> Tuple[float, List[float]]:
     """Effective measure complexity via random block shuffling.
 
-    Returns ``(emc_value, summands)`` — the EMC value and the list of
-    per-block-size terms whose sum is the EMC value.
+    Returns ``(emc_value, summands)``. Both the value and every summand are
+    non-negative, and the summands sum to the value; ``summands[l-1]``
+    estimates the scale-``l`` conditional-entropy excess ``h(l) - h``.
+
+    Read the value as an ordinal index at fixed sequence length, not as a bit
+    count, and never compare it across lengths.
     """
     ...
 

lzcomplexity-1.0.2/Cargo.toml

@@ -1,30 +0,0 @@
-[workspace]
-resolver = "2"
-members = ["crates/lzcomplexity-core", "crates/lzcomplexity-py"]
-
-[workspace.package]
-version = "1.0.2" # x-release-please-version
-edition = "2021"
-rust-version = "1.75"
-license = "MIT"
-authors = [
-    "Efrén Aragón Pérez <efrenaragon96@gmail.com>",
-    "Daniel Estévez Moya <kemossabee@gmail.com>",
-    "Ernesto Estévez Rams <estevez@fisica.uh.cu>",
-]
-repository = "https://github.com/pleros-ai/lzcomplexity"
-
-[workspace.dependencies]
-rayon = "1.10"
-rand = "0.8"
-rand_chacha = "0.3"
-cdivsufsort = "2.0"
-clap = { version = "4.5", features = ["derive"] }
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-flate2 = "1.0"
-
-[profile.release]
-lto = "thin"
-codegen-units = 1
-opt-level = 3

lzcomplexity-1.0.2/crates/lzcomplexity-core/src/shuffle.rs

@@ -1,216 +0,0 @@
-//! Shuffle-based complexity measures.
-
-use rayon::prelude::*;
-
-use crate::lz76::{lz76_factorization, LzArgs, LzShuffle};
-use crate::sequence::{shuffle_copy_seeded, Sequence};
-use crate::NO_ALPHABET;
-
-/// Estimate the "naive M" satisfying `N = M * 2^M`, mirroring `max_block_size` in C++.
-pub fn max_block_size(size: usize) -> usize {
-    if size < 10 {
-        return 1;
-    }
-    let mut m: usize = 10;
-    let mut m_old: usize = 1;
-    let mut count: u32 = 0;
-    while count < 100 {
-        let est = ((size as f64) / (m_old as f64)).ln() / 2f64.ln();
-        m = est.round() as usize;
-        count += 1;
-        if m == m_old {
-            break;
-        }
-        m_old = m;
-    }
-    m
-}
-
-/// Compute the random-shuffle factorization vector and `mm`. Mirrors
-/// `ShuffleFactorization` from the C++.
-pub fn shuffle_factorization(seq: &Sequence, args: &LzArgs) -> (Vec<i32>, usize) {
-    let mut mm: i64 = args.block_size as i64;
-    if mm <= 0 {
-        mm = max_block_size(seq.len()) as i64;
-        if seq.len() > 50 {
-            mm += 10;
-        }
-    }
-    let mm = mm as usize;
-
-    // Index 0 unused, indices 1..=mm hold the per-block-size complexity.
-    let mut res: Vec<i32> = vec![0; mm + 3];
-
-    // Hash the sequence content ONCE; each per-block seed only differs by the
-    // `idx` term (bit-identical seeds → identical shuffles → identical EMC).
-    let base = fnv1a(seq.as_bytes());
-    let computed: Vec<(usize, i32)> = (1..=mm)
-        .into_par_iter()
-        .map(|idx| {
-            let seed = seed_from_base(base, idx);
-            let rand_seq = shuffle_copy_seeded(seq, idx as u32, (seq.len() / 2) as u32, seed);
-            let c = lz76_factorization(&rand_seq, args);
-            (idx, c as i32)
-        })
-        .collect();
-    for (idx, c) in computed {
-        res[idx] = c;
-    }
-    (res, mm)
-}
-
-/// Effective measure complexity via the block-entropy estimator.
-///
-/// For each block size `l`, `h_rand[l]` is `C_LZ(u^{RS(l)})`, the LZ complexity
-/// of `u` block-shuffled at scale `l`. From it we form a block-entropy estimate
-/// `H_l = l * C_LZ(u^{RS(l)}) * log_k(N) / N` (with `H_0 = 0`), take the entropy
-/// gain `ΔH_l = H_l - H_{l-1}`, and sum each gain's excess over the original
-/// sequence's entropy density `ĥ = C_LZ(u) * log_k(N) / N`:
-///
-/// ```text
-/// Ê = Σ_{l=1}^{mm} (ΔH_l - ĥ)
-/// ```
-pub fn shuffle_entropy_calculation(
-    seq: &Sequence,
-    args: &LzArgs,
-    complexity: i32,
-    h_rand: &[i32],
-    mm: usize,
-) -> LzShuffle {
-    let mut result = LzShuffle {
-        max_block_size: mm as i32,
-        ..Default::default()
-    };
-    if args.get_shuffle_terms {
-        result.summands = vec![0.0; mm];
-    }
-    let log_base = if args.log_base == NO_ALPHABET {
-        seq.alphabet_size().max(2)
-    } else {
-        args.log_base.max(2)
-    } as f64;
-
-    let n = seq.len() as f64;
-    if n <= 1.0 || mm == 0 {
-        return result;
-    }
-    // g = log_k(N) / N, so ĥ = C_LZ(u)·g and H_l = l·C_LZ(u^{RS(l)})·g.
-    let g = (n.ln() / log_base.ln()) / n;
-    let h_hat = complexity as f64 * g; // entropy density of the original sequence
-
-    let mut emc = 0.0f64;
-    let mut h_prev = 0.0f64; // H_0 = 0
-    for l in 1..=mm {
-        if l >= h_rand.len() {
-            break;
-        }
-        let h_l = l as f64 * h_rand[l] as f64 * g;
-        let term = (h_l - h_prev) - h_hat; // ΔH_l - ĥ
-        emc += term;
-        if args.get_shuffle_terms {
-            result.summands[l - 1] = term;
-        }
-        if l == 1 {
-            result.multi_information = term;
-        }
-        h_prev = h_l;
-    }
-    result.emc_value = emc;
-    result
-}
-
-/// `lz76RandomShuffleComplexity` — full pipeline.
-pub fn lz76_random_shuffle_complexity(seq: &Sequence, args: &LzArgs) -> LzShuffle {
-    lz76_random_shuffle_complexity_with(seq, args, lz76_factorization(seq, args) as i32)
-}
-
-/// Same, but with an already-computed original-sequence complexity (so callers
-/// that already have it — e.g. the `lz76` driver — skip one factorization).
-pub fn lz76_random_shuffle_complexity_with(
-    seq: &Sequence,
-    args: &LzArgs,
-    complexity: i32,
-) -> LzShuffle {
-    let (h_rand, mm) = shuffle_factorization(seq, args);
-    shuffle_entropy_calculation(seq, args, complexity, &h_rand, mm)
-}
-
-/// `lz76PairedShuffleComplexity` — shuffle on the merged (Z) sequence.
-pub fn lz76_paired_shuffle_complexity(seq: &Sequence, args: &LzArgs) -> LzShuffle {
-    let mid = seq.len() / 2;
-    let (past, future) = seq.split_at(mid);
-    let merged = merge_sequences(&past, &future);
-    let complexity = lz76_factorization(&merged, args) as i32;
-    let (h_rand, mm) = shuffle_factorization(&merged, args);
-    shuffle_entropy_calculation(&merged, args, complexity, &h_rand, mm)
-}
-
-/// Mirror of `internal::MergeSequences` — element-wise pairing into a new alphabet.
-pub fn merge_sequences(s1: &Sequence, s2: &Sequence) -> Sequence {
-    let max_iter = s1.len().min(s2.len());
-    let mut buf = Vec::with_capacity(max_iter);
-    let mut lookup = vec![0u8; 65536];
-    let mut next_symbol: u8 = b'0';
-    let a = s1.as_bytes();
-    let b = s2.as_bytes();
-    for i in 0..max_iter {
-        let key = ((a[i] as u16) << 8) | (b[i] as u16);
-        let entry = &mut lookup[key as usize];
-        if *entry == 0 {
-            *entry = next_symbol;
-            next_symbol = next_symbol.wrapping_add(1);
-        }
-        buf.push(*entry);
-    }
-    Sequence::from_bytes_with_alphabet(buf, s1.alphabet_size().saturating_mul(s2.alphabet_size()))
-}
-
-/// FNV-1a hash of the sequence bytes (the content-dependent part of the seed).
-fn fnv1a(bytes: &[u8]) -> u64 {
-    let mut h: u64 = 14695981039346656037; // FNV offset basis (64-bit)
-    for &c in bytes {
-        h ^= c as u64;
-        h = h.wrapping_mul(1099511628211);
-    }
-    h
-}
-
-/// Derive the per-block-size seed from the (once-computed) content hash.
-fn seed_from_base(base: u64, idx: usize) -> u64 {
-    base ^ ((idx as u64).wrapping_mul(0x9E3779B97F4A7C15))
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn emc_block_entropy_formula() {
-        // Binary sequence (alphabet 2 -> log_base auto = 2), n = 100.
-        let mut data = vec![b'0'; 50];
-        data.extend(std::iter::repeat(b'1').take(50));
-        let seq = Sequence::from_bytes(data);
-        let mut args = LzArgs::new();
-        args.get_shuffle_terms = true;
-
-        let complexity = 10i32; // C_LZ(u)
-        let h_rand = vec![0, 20, 15, 12]; // index 0 unused; c_1=20, c_2=15, c_3=12
-        let mm = 3usize;
-        let r = shuffle_entropy_calculation(&seq, &args, complexity, &h_rand, mm);
-
-        // Hand-computed: g = log_2(100)/100, ĥ = 10·g, H_l = l·c_l·g (H_0 = 0).
-        let n = 100.0f64;
-        let g = (n.ln() / 2f64.ln()) / n;
-        let h_hat = 10.0 * g;
-        let (h1, h2, h3) = (1.0 * 20.0 * g, 2.0 * 15.0 * g, 3.0 * 12.0 * g);
-        let expected = (h1 - 0.0 - h_hat) + (h2 - h1 - h_hat) + (h3 - h2 - h_hat);
-        assert!(
-            (r.emc_value - expected).abs() < 1e-12,
-            "{} vs {}",
-            r.emc_value,
-            expected
-        );
-        assert!((r.summands[0] - (h1 - h_hat)).abs() < 1e-12);
-        assert!((r.multi_information - (h1 - h_hat)).abs() < 1e-12);
-    }
-}