roxify 1.12.7 → 1.12.9

package/native/archive.rs

@@ -0,0 +1,220 @@
+use std::io::Cursor;
+use std::path::Path;
+use rayon::prelude::*;
+use tar::{Archive, Builder, Header};
+use walkdir::WalkDir;
+
+pub struct TarPackResult {
+    pub data: Vec<u8>,
+    pub file_list: Vec<(String, u64)>,
+}
+
+pub fn tar_pack_directory_with_list(dir_path: &Path) -> Result<TarPackResult, String> {
+    let base = dir_path;
+
+    let entries: Vec<_> = WalkDir::new(dir_path)
+        .follow_links(false)
+        .into_iter()
+        .filter_map(|e| e.ok())
+        .filter(|e| e.file_type().is_file())
+        .collect();
+
+    let file_data: Vec<(String, Vec<u8>)> = entries
+        .par_iter()
+        .filter_map(|entry| {
+            let full = entry.path();
+            let rel = full.strip_prefix(base).unwrap_or(full);
+            let rel_str = rel.to_string_lossy().replace('\\', "/");
+            match std::fs::read(full) {
+                Ok(data) => Some((rel_str, data)),
+                Err(_) => None,
+            }
+        })
+        .collect();
+
+    let file_list: Vec<(String, u64)> = file_data.iter()
+        .map(|(name, data)| (name.clone(), data.len() as u64))
+        .collect();
+
+    let total_estimate: usize = file_data.iter().map(|(n, d)| 512 + d.len() + 512 + n.len()).sum();
+    let mut buf = Vec::with_capacity(total_estimate + 1024);
+    {
+        let mut builder = Builder::new(&mut buf);
+        for (rel_str, data) in &file_data {
+            let mut header = Header::new_gnu();
+            header.set_size(data.len() as u64);
+            header.set_mode(0o644);
+            header.set_cksum();
+            builder
+                .append_data(&mut header, rel_str, &data[..])
+                .map_err(|e| format!("tar append {}: {}", rel_str, e))?;
+        }
+        builder.finish().map_err(|e| format!("tar finish: {}", e))?;
+    }
+    Ok(TarPackResult { data: buf, file_list })
+}
+
+pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+    tar_pack_directory_with_list(dir_path).map(|r| r.data)
+}
+
+pub fn tar_file_list_fast(tar_data: &[u8]) -> Vec<(String, u64)> {
+    let mut list = Vec::new();
+    let mut pos = 0;
+    while pos + 512 <= tar_data.len() {
+        let header = &tar_data[pos..pos + 512];
+        if header.iter().all(|&b| b == 0) {
+            break;
+        }
+        let name_end = header[..100].iter().position(|&b| b == 0).unwrap_or(100);
+        let name = String::from_utf8_lossy(&header[..name_end]).to_string();
+        let size_str = String::from_utf8_lossy(&header[124..136]);
+        let size = u64::from_str_radix(size_str.trim().trim_matches('\0'), 8).unwrap_or(0);
+        if !name.is_empty() {
+            list.push((name, size));
+        }
+        let data_blocks = (size as usize + 511) / 512;
+        pos += 512 + data_blocks * 512;
+    }
+    list
+}
+
+pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, String> {
+    let mut archive = Archive::new(Cursor::new(tar_data));
+    let mut entries_data: Vec<(std::path::PathBuf, Vec<u8>)> = Vec::new();
+
+    let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
+    for entry in entries {
+        let mut entry = entry.map_err(|e| format!("tar entry: {}", e))?;
+        let path = entry.path().map_err(|e| format!("tar entry path: {}", e))?.to_path_buf();
+
+        let mut safe = std::path::PathBuf::new();
+        for comp in path.components() {
+            if let std::path::Component::Normal(osstr) = comp {
+                safe.push(osstr);
+            }
+        }
+        if safe.as_os_str().is_empty() {
+            continue;
+        }
+
+        let mut data = Vec::with_capacity(entry.size() as usize);
+        std::io::Read::read_to_end(&mut entry, &mut data)
+            .map_err(|e| format!("tar read {:?}: {}", safe, e))?;
+        entries_data.push((safe, data));
+    }
+
+    let dirs: std::collections::HashSet<_> = entries_data.iter()
+        .filter_map(|(p, _)| {
+            let dest = output_dir.join(p);
+            dest.parent().map(|d| d.to_path_buf())
+        })
+        .collect();
+    for dir in &dirs {
+        std::fs::create_dir_all(dir).map_err(|e| format!("mkdir {:?}: {}", dir, e))?;
+    }
+
+    let written: Vec<String> = entries_data.par_iter()
+        .filter_map(|(safe, data)| {
+            let dest = output_dir.join(safe);
+            match std::fs::write(&dest, data) {
+                Ok(_) => Some(safe.to_string_lossy().to_string()),
+                Err(_) => None,
+            }
+        })
+        .collect();
+
+    Ok(written)
+}
+
+pub fn is_tar(data: &[u8]) -> bool {
+    if data.len() < 263 {
+        return false;
+    }
+    &data[257..262] == b"ustar"
+}
+
+pub fn tar_file_list(tar_data: &[u8]) -> Result<Vec<(String, u64)>, String> {
+    let mut archive = Archive::new(Cursor::new(tar_data));
+    let mut list = Vec::new();
+    let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
+    for entry in entries {
+        let entry = entry.map_err(|e| format!("tar entry: {}", e))?;
+        let path = entry
+            .path()
+            .map_err(|e| format!("tar path: {}", e))?
+            .to_string_lossy()
+            .to_string();
+        let size = entry.size();
+        list.push((path, size));
+    }
+    Ok(list)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+
+    #[test]
+    fn test_tar_roundtrip() {
+        let tmp = std::env::temp_dir().join("rox_tar_test");
+        let _ = fs::remove_dir_all(&tmp);
+        fs::create_dir_all(tmp.join("sub")).unwrap();
+        fs::write(tmp.join("hello.txt"), b"Hello TAR").unwrap();
+        fs::write(tmp.join("sub/nested.txt"), b"Nested!").unwrap();
+
+        let tar_data = tar_pack_directory(&tmp).unwrap();
+        assert!(is_tar(&tar_data));
+
+        let list = tar_file_list(&tar_data).unwrap();
+        assert_eq!(list.len(), 2);
+
+        let out = std::env::temp_dir().join("rox_tar_test_out");
+        let _ = fs::remove_dir_all(&out);
+        fs::create_dir_all(&out).unwrap();
+
+        let written = tar_unpack(&tar_data, &out).unwrap();
+        assert_eq!(written.len(), 2);
+        assert_eq!(fs::read_to_string(out.join("hello.txt")).unwrap(), "Hello TAR");
+        assert_eq!(fs::read_to_string(out.join("sub/nested.txt")).unwrap(), "Nested!");
+
+        let _ = fs::remove_dir_all(&tmp);
+        let _ = fs::remove_dir_all(&out);
+    }
+
+    #[test]
+    fn test_tar_zstd_roundtrip() {
+        use std::io::Write;
+
+        let tmp = std::env::temp_dir().join("rox_tar_zstd_test");
+        let _ = fs::remove_dir_all(&tmp);
+        fs::create_dir_all(tmp.join("a/b")).unwrap();
+        fs::write(tmp.join("root.txt"), b"root file content").unwrap();
+        fs::write(tmp.join("a/mid.txt"), b"mid level").unwrap();
+        fs::write(tmp.join("a/b/deep.txt"), b"deep nested file").unwrap();
+
+        let tar_data = tar_pack_directory(&tmp).unwrap();
+        assert!(is_tar(&tar_data));
+
+        let mut encoder = zstd::stream::Encoder::new(Vec::new(), 3).unwrap();
+        encoder.write_all(&tar_data).unwrap();
+        let compressed = encoder.finish().unwrap();
+
+        let decompressed = crate::core::zstd_decompress_bytes(&compressed, None).unwrap();
+        assert!(is_tar(&decompressed));
+
+        let out = std::env::temp_dir().join("rox_tar_zstd_test_out");
+        let _ = fs::remove_dir_all(&out);
+        fs::create_dir_all(&out).unwrap();
+
+        let written = tar_unpack(&decompressed, &out).unwrap();
+        assert_eq!(written.len(), 3);
+        assert_eq!(fs::read_to_string(out.join("root.txt")).unwrap(), "root file content");
+        assert_eq!(fs::read_to_string(out.join("a/mid.txt")).unwrap(), "mid level");
+        assert_eq!(fs::read_to_string(out.join("a/b/deep.txt")).unwrap(), "deep nested file");
+
+        let _ = fs::remove_dir_all(&tmp);
+        let _ = fs::remove_dir_all(&out);
+    }
+}

package/native/audio.rs

@@ -0,0 +1,151 @@
+/// WAV container for binary data.
+///
+/// Encodes raw bytes as 8-bit unsigned PCM mono samples (44100 Hz).
+/// Header is exactly 44 bytes. Total overhead: 44 bytes.
+///
+/// Compared to PNG (stored deflate): PNG overhead grows with data size
+/// (zlib framing, filter bytes, chunk CRCs). WAV overhead is constant.
+
+const WAV_HEADER_SIZE: usize = 44;
+const SAMPLE_RATE: u32 = 44100;
+const BITS_PER_SAMPLE: u16 = 8;
+const NUM_CHANNELS: u16 = 1;
+
+/// Pack raw bytes into a WAV file (8-bit PCM, mono, 44100 Hz).
+/// The bytes are stored directly as unsigned PCM samples.
+/// Returns the complete WAV file as a Vec<u8>.
+pub fn bytes_to_wav(data: &[u8]) -> Vec<u8> {
+    let data_size = data.len() as u32;
+    let file_size = WAV_HEADER_SIZE as u32 - 8 + data_size; // RIFF chunk size
+
+    let byte_rate = SAMPLE_RATE * NUM_CHANNELS as u32 * (BITS_PER_SAMPLE as u32 / 8);
+    let block_align = NUM_CHANNELS * (BITS_PER_SAMPLE / 8);
+
+    let mut wav = Vec::with_capacity(WAV_HEADER_SIZE + data.len());
+
+    // RIFF header
+    wav.extend_from_slice(b"RIFF");
+    wav.extend_from_slice(&file_size.to_le_bytes());
+    wav.extend_from_slice(b"WAVE");
+
+    // fmt sub-chunk
+    wav.extend_from_slice(b"fmt ");
+    wav.extend_from_slice(&16u32.to_le_bytes()); // sub-chunk size (PCM = 16)
+    wav.extend_from_slice(&1u16.to_le_bytes());  // audio format (1 = PCM)
+    wav.extend_from_slice(&NUM_CHANNELS.to_le_bytes());
+    wav.extend_from_slice(&SAMPLE_RATE.to_le_bytes());
+    wav.extend_from_slice(&byte_rate.to_le_bytes());
+    wav.extend_from_slice(&block_align.to_le_bytes());
+    wav.extend_from_slice(&BITS_PER_SAMPLE.to_le_bytes());
+
+    // data sub-chunk
+    wav.extend_from_slice(b"data");
+    wav.extend_from_slice(&data_size.to_le_bytes());
+    wav.extend_from_slice(data);
+
+    wav
+}
+
+/// Extract raw bytes from a WAV file.
+/// Returns the PCM data (the original bytes) or an error.
+pub fn wav_to_bytes(wav: &[u8]) -> Result<Vec<u8>, String> {
+    if wav.len() < WAV_HEADER_SIZE {
+        return Err("WAV data too short".to_string());
+    }
+
+    // Validate RIFF header
+    if &wav[0..4] != b"RIFF" {
+        return Err("Not a RIFF file".to_string());
+    }
+    if &wav[8..12] != b"WAVE" {
+        return Err("Not a WAVE file".to_string());
+    }
+
+    // Find the "data" sub-chunk (skip fmt and any other chunks)
+    let mut offset = 12; // past "RIFF" + size + "WAVE"
+    loop {
+        if offset + 8 > wav.len() {
+            return Err("data chunk not found".to_string());
+        }
+        let chunk_id = &wav[offset..offset + 4];
+        let chunk_size = u32::from_le_bytes([
+            wav[offset + 4],
+            wav[offset + 5],
+            wav[offset + 6],
+            wav[offset + 7],
+        ]) as usize;
+
+        if chunk_id == b"data" {
+            let data_start = offset + 8;
+            let data_end = data_start + chunk_size;
+            if data_end > wav.len() {
+                // Allow truncation: return what we have
+                return Ok(wav[data_start..].to_vec());
+            }
+            return Ok(wav[data_start..data_end].to_vec());
+        }
+
+        // Skip this chunk (+ padding byte if odd size)
+        offset += 8 + chunk_size;
+        if chunk_size % 2 != 0 {
+            offset += 1; // RIFF chunks are word-aligned
+        }
+    }
+}
+
+/// Check if a buffer starts with a RIFF/WAVE header.
+pub fn is_wav(buf: &[u8]) -> bool {
+    buf.len() >= 12 && &buf[0..4] == b"RIFF" && &buf[8..12] == b"WAVE"
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_wav_roundtrip() {
+        let data = b"Hello, World! This is roxify audio container test data.";
+        let wav = bytes_to_wav(data);
+
+        // Check header
+        assert_eq!(&wav[0..4], b"RIFF");
+        assert_eq!(&wav[8..12], b"WAVE");
+        assert_eq!(wav.len(), 44 + data.len());
+
+        // Roundtrip
+        let recovered = wav_to_bytes(&wav).expect("decode should succeed");
+        assert_eq!(recovered, data);
+    }
+
+    #[test]
+    fn test_wav_empty() {
+        let data: &[u8] = b"";
+        let wav = bytes_to_wav(data);
+        assert_eq!(wav.len(), 44);
+        let recovered = wav_to_bytes(&wav).expect("decode empty");
+        assert!(recovered.is_empty());
+    }
+
+    #[test]
+    fn test_wav_large() {
+        let data = vec![0xAB_u8; 1024 * 1024]; // 1 MB
+        let wav = bytes_to_wav(&data);
+        assert_eq!(wav.len(), 44 + 1024 * 1024);
+        let recovered = wav_to_bytes(&wav).expect("decode large");
+        assert_eq!(recovered, data);
+    }
+
+    #[test]
+    fn test_is_wav() {
+        let wav = bytes_to_wav(b"test");
+        assert!(is_wav(&wav));
+        assert!(!is_wav(b"not a wav"));
+        assert!(!is_wav(b"RIFF1234XXXX")); // RIFF but not WAVE
+    }
+
+    #[test]
+    fn test_invalid_wav() {
+        assert!(wav_to_bytes(b"short").is_err());
+        assert!(wav_to_bytes(b"NOT a RIFF file!").is_err());
+    }
+}

package/native/bench_hybrid.rs

@@ -0,0 +1,145 @@
+use std::time::Instant;
+
+mod rans_byte;
+mod bwt;
+mod mtf;
+mod context_mixing;
+mod pool;
+mod hybrid;
+
+fn bench_roundtrip(name: &str, data: &[u8]) {
+    let compressor = hybrid::HybridCompressor::new(false, 4);
+
+    let start = Instant::now();
+    let (compressed, stats) = compressor.compress(data).unwrap();
+    let compress_time = start.elapsed();
+
+    let start = Instant::now();
+    let decompressed = compressor.decompress(&compressed).unwrap();
+    let decompress_time = start.elapsed();
+
+    let ratio = (compressed.len() as f64) / (data.len() as f64) * 100.0;
+    let compress_mbps = (data.len() as f64 / 1_048_576.0) / compress_time.as_secs_f64();
+    let decompress_mbps = (data.len() as f64 / 1_048_576.0) / decompress_time.as_secs_f64();
+
+    assert_eq!(decompressed, data, "ROUND-TRIP FAILED for {}", name);
+
+    println!("=== {} ===", name);
+    println!("  Input:       {} bytes", data.len());
+    println!("  Compressed:  {} bytes ({:.1}%)", compressed.len(), ratio);
+    println!("  Reduction:   {:.1}%", 100.0 - ratio);
+    println!("  Compress:    {:.1} ms ({:.1} MB/s)", compress_time.as_secs_f64() * 1000.0, compress_mbps);
+    println!("  Decompress:  {:.1} ms ({:.1} MB/s)", decompress_time.as_secs_f64() * 1000.0, decompress_mbps);
+    println!("  Entropy:     {:.2} bits/byte", stats.entropy_bits);
+    println!();
+}
+
+fn bench_zstd(name: &str, data: &[u8], level: i32) {
+    let start = Instant::now();
+    let compressed = zstd::encode_all(std::io::Cursor::new(data), level).unwrap();
+    let compress_time = start.elapsed();
+
+    let start = Instant::now();
+    let decompressed = zstd::decode_all(std::io::Cursor::new(&compressed)).unwrap();
+    let decompress_time = start.elapsed();
+
+    let ratio = (compressed.len() as f64) / (data.len() as f64) * 100.0;
+    let compress_mbps = (data.len() as f64 / 1_048_576.0) / compress_time.as_secs_f64();
+    let decompress_mbps = (data.len() as f64 / 1_048_576.0) / decompress_time.as_secs_f64();
+
+    assert_eq!(decompressed, data);
+
+    println!("=== Zstd L{} ({}) ===", level, name);
+    println!("  Compressed:  {} bytes ({:.1}%)", compressed.len(), ratio);
+    println!("  Reduction:   {:.1}%", 100.0 - ratio);
+    println!("  Compress:    {:.1} ms ({:.1} MB/s)", compress_time.as_secs_f64() * 1000.0, compress_mbps);
+    println!("  Decompress:  {:.1} ms ({:.1} MB/s)", decompress_time.as_secs_f64() * 1000.0, decompress_mbps);
+    println!();
+}
+
+fn main() {
+    println!("╔══════════════════════════════════════════════════════════╗");
+    println!("║     ROXIFY BWT-ANS COMPRESSION BENCHMARK               ║");
+    println!("╚══════════════════════════════════════════════════════════╝\n");
+
+    let text_1k: Vec<u8> = "Hello World! This is a test of the roxify compression engine. ".repeat(16).into_bytes();
+    bench_roundtrip("Text 1KB", &text_1k);
+    bench_zstd("Text 1KB", &text_1k, 3);
+    bench_zstd("Text 1KB", &text_1k, 19);
+
+    let text_100k: Vec<u8> = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. ".repeat(1200).into_bytes();
+    bench_roundtrip("Text 100KB", &text_100k);
+    bench_zstd("Text 100KB", &text_100k, 3);
+    bench_zstd("Text 100KB", &text_100k, 19);
+
+    let text_1m: Vec<u8> = {
+        let mut data = Vec::with_capacity(1_048_576);
+        let phrases = [
+            b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. ".as_slice(),
+            b"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ".as_slice(),
+            b"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. ".as_slice(),
+            b"Duis aute irure dolor in reprehenderit in voluptate velit esse. ".as_slice(),
+            b"Excepteur sint occaecat cupidatat non proident, sunt in culpa. ".as_slice(),
+        ];
+        let mut i = 0;
+        while data.len() < 1_048_576 {
+            data.extend_from_slice(phrases[i % phrases.len()]);
+            i += 1;
+        }
+        data.truncate(1_048_576);
+        data
+    };
+    bench_roundtrip("Text 1MB", &text_1m);
+    bench_zstd("Text 1MB", &text_1m, 3);
+    bench_zstd("Text 1MB", &text_1m, 19);
+
+    let json_data: Vec<u8> = {
+        let mut data = String::with_capacity(512_000);
+        data.push('[');
+        for i in 0..5000 {
+            if i > 0 { data.push(','); }
+            data.push_str(&format!(
+                r#"{{"id":{},"name":"user_{}","email":"user{}@example.com","active":{},"score":{:.2},"tags":["tag1","tag2","tag3"]}}"#,
+                i, i, i, i % 2 == 0, (i as f64) * 1.337
+            ));
+        }
+        data.push(']');
+        data.into_bytes()
+    };
+    bench_roundtrip("JSON 500KB", &json_data);
+    bench_zstd("JSON 500KB", &json_data, 3);
+    bench_zstd("JSON 500KB", &json_data, 19);
+
+    let random_data: Vec<u8> = {
+        let mut data = vec![0u8; 100_000];
+        let mut state = 12345u64;
+        for b in data.iter_mut() {
+            state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
+            *b = (state >> 33) as u8;
+        }
+        data
+    };
+    bench_roundtrip("Random 100KB", &random_data);
+    bench_zstd("Random 100KB", &random_data, 3);
+
+    let binary_data: Vec<u8> = {
+        let mut data = Vec::with_capacity(256_000);
+        for i in 0..256_000u32 {
+            match i % 7 {
+                0 => data.push(0),
+                1 => data.push(0xFF),
+                2 => data.push((i & 0xFF) as u8),
+                3 => data.push(((i >> 8) & 0xFF) as u8),
+                4 => data.push(b'A' + (i % 26) as u8),
+                5 => data.push(0x20),
+                _ => data.push((i.wrapping_mul(37) & 0xFF) as u8),
+            }
+        }
+        data
+    };
+    bench_roundtrip("Binary 256KB", &binary_data);
+    bench_zstd("Binary 256KB", &binary_data, 3);
+    bench_zstd("Binary 256KB", &binary_data, 19);
+
+    println!("All round-trip tests PASSED!");
+}

package/native/bwt.rs

@@ -0,0 +1,56 @@
+use anyhow::Result;
+use libsais::bwt::Bwt;
+use libsais::typestate::OwnedBuffer;
+use libsais::BwtConstruction;
+use rayon::prelude::*;
+
+pub struct BwtResult {
+    pub transformed: Vec<u8>,
+    pub primary_index: u32,
+}
+
+pub fn bwt_encode(data: &[u8]) -> Result<BwtResult> {
+    let n = data.len();
+    if n == 0 {
+        return Ok(BwtResult { transformed: Vec::new(), primary_index: 0 });
+    }
+
+    let bwt_result = BwtConstruction::for_text(data)
+        .with_owned_temporary_array_buffer32()
+        .single_threaded()
+        .run()
+        .map_err(|e| anyhow::anyhow!("libsais BWT: {:?}", e))?;
+
+    let primary_index = bwt_result.primary_index() as u32;
+    let transformed = bwt_result.bwt().to_vec();
+
+    Ok(BwtResult { transformed, primary_index })
+}
+
+pub fn bwt_decode(bwt_data: &[u8], primary_index: u32) -> Result<Vec<u8>> {
+    if bwt_data.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let bwt_obj: Bwt<'static, u8, OwnedBuffer> =
+        unsafe { Bwt::from_parts(bwt_data.to_vec(), primary_index as usize) };
+
+    let text = bwt_obj
+        .unbwt()
+        .with_owned_temporary_array_buffer32()
+        .single_threaded()
+        .run()
+        .map_err(|e| anyhow::anyhow!("libsais UnBWT: {:?}", e))?;
+
+    Ok(text.as_slice().to_vec())
+}
+
+pub fn bwt_encode_streaming(block_size: usize, data: &[u8]) -> Result<Vec<(BwtResult, usize)>> {
+    data.par_chunks(block_size)
+        .enumerate()
+        .map(|(i, chunk)| {
+            let result = bwt_encode(chunk)?;
+            Ok((result, i * block_size))
+        })
+        .collect()
+}

package/native/context_mixing.rs

@@ -0,0 +1,117 @@
+#[derive(Clone, Copy, Debug)]
+pub struct ProbabilityEstimate {
+    pub p0: u32,
+    pub p1: u32,
+    pub total: u32,
+}
+
+impl ProbabilityEstimate {
+    pub fn entropy_bits(&self) -> f32 {
+        if self.total == 0 {
+            return 0.0;
+        }
+        let p0 = (self.p0 as f32) / (self.total as f32);
+        let p1 = (self.p1 as f32) / (self.total as f32);
+
+        let mut bits = 0.0;
+        if p0 > 0.0 {
+            bits -= p0 * p0.log2();
+        }
+        if p1 > 0.0 {
+            bits -= p1 * p1.log2();
+        }
+        bits
+    }
+}
+
+pub struct ContextMixer {
+    contexts_order0: Vec<ProbabilityEstimate>,
+    contexts_order1: Vec<[ProbabilityEstimate; 256]>,
+    contexts_order2: Vec<[[ProbabilityEstimate; 256]; 256]>,
+}
+
+impl ContextMixer {
+    pub fn new() -> Self {
+        ContextMixer {
+            contexts_order0: vec![ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 1],
+            contexts_order1: vec![
+                [ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 256];
+                256
+            ],
+            contexts_order2: vec![
+                [[ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 256]; 256];
+                256
+            ],
+        }
+    }
+
+    pub fn predict_order0(&self) -> ProbabilityEstimate {
+        self.contexts_order0[0]
+    }
+
+    pub fn predict_order1(&self, context1: u8) -> ProbabilityEstimate {
+        self.contexts_order1[context1 as usize][0]
+    }
+
+    pub fn predict_order2(&self, context1: u8, context2: u8) -> ProbabilityEstimate {
+        self.contexts_order2[context1 as usize][context2 as usize][0]
+    }
+
+    pub fn update_order0(&mut self, bit: bool) {
+        let ctx = &mut self.contexts_order0[0];
+        if bit {
+            ctx.p1 += 1;
+        } else {
+            ctx.p0 += 1;
+        }
+        ctx.total += 1;
+    }
+
+    pub fn update_order1(&mut self, context1: u8, bit: bool) {
+        let ctx = &mut self.contexts_order1[context1 as usize][0];
+        if bit {
+            ctx.p1 += 1;
+        } else {
+            ctx.p0 += 1;
+        }
+        ctx.total += 1;
+    }
+
+    pub fn update_order2(&mut self, context1: u8, context2: u8, bit: bool) {
+        let ctx = &mut self.contexts_order2[context1 as usize][context2 as usize][0];
+        if bit {
+            ctx.p1 += 1;
+        } else {
+            ctx.p0 += 1;
+        }
+        ctx.total += 1;
+    }
+}
+
+pub fn analyze_entropy(data: &[u8]) -> f32 {
+    let mut freq = [0u32; 256];
+    for &byte in data {
+        freq[byte as usize] += 1;
+    }
+
+    let total = data.len() as f32;
+    if total == 0.0 {
+        return 0.0;
+    }
+
+    let inv_total = 1.0 / total;
+    let mut entropy = 0.0f32;
+    for &f in &freq {
+        if f > 0 {
+            let p = f as f32 * inv_total;
+            entropy -= p * p.log2();
+        }
+    }
+    entropy
+}
+
+pub fn estimate_compression_gain(original: &[u8], entropy_bits: f32) -> f64 {
+    let theoretical_min = (original.len() as f64) * (entropy_bits as f64) / 8.0;
+    let ratio = theoretical_min / (original.len() as f64);
+    (1.0 - ratio) * 100.0
+}