roxify 1.13.7 → 1.13.9

package/native/bench_hybrid.rs

@@ -1,145 +0,0 @@
-use std::time::Instant;
-
-mod rans_byte;
-mod bwt;
-mod mtf;
-mod context_mixing;
-mod pool;
-mod hybrid;
-
-fn bench_roundtrip(name: &str, data: &[u8]) {
-    let compressor = hybrid::HybridCompressor::new();
-
-    let start = Instant::now();
-    let (compressed, stats) = compressor.compress(data).unwrap();
-    let compress_time = start.elapsed();
-
-    let start = Instant::now();
-    let decompressed = compressor.decompress(&compressed).unwrap();
-    let decompress_time = start.elapsed();
-
-    let ratio = (compressed.len() as f64) / (data.len() as f64) * 100.0;
-    let compress_mbps = (data.len() as f64 / 1_048_576.0) / compress_time.as_secs_f64();
-    let decompress_mbps = (data.len() as f64 / 1_048_576.0) / decompress_time.as_secs_f64();
-
-    assert_eq!(decompressed, data, "ROUND-TRIP FAILED for {}", name);
-
-    println!("=== {} ===", name);
-    println!("  Input:       {} bytes", data.len());
-    println!("  Compressed:  {} bytes ({:.1}%)", compressed.len(), ratio);
-    println!("  Reduction:   {:.1}%", 100.0 - ratio);
-    println!("  Compress:    {:.1} ms ({:.1} MB/s)", compress_time.as_secs_f64() * 1000.0, compress_mbps);
-    println!("  Decompress:  {:.1} ms ({:.1} MB/s)", decompress_time.as_secs_f64() * 1000.0, decompress_mbps);
-    println!("  Entropy:     {:.2} bits/byte", stats.entropy_bits);
-    println!();
-}
-
-fn bench_zstd(name: &str, data: &[u8], level: i32) {
-    let start = Instant::now();
-    let compressed = zstd::encode_all(std::io::Cursor::new(data), level).unwrap();
-    let compress_time = start.elapsed();
-
-    let start = Instant::now();
-    let decompressed = zstd::decode_all(std::io::Cursor::new(&compressed)).unwrap();
-    let decompress_time = start.elapsed();
-
-    let ratio = (compressed.len() as f64) / (data.len() as f64) * 100.0;
-    let compress_mbps = (data.len() as f64 / 1_048_576.0) / compress_time.as_secs_f64();
-    let decompress_mbps = (data.len() as f64 / 1_048_576.0) / decompress_time.as_secs_f64();
-
-    assert_eq!(decompressed, data);
-
-    println!("=== Zstd L{} ({}) ===", level, name);
-    println!("  Compressed:  {} bytes ({:.1}%)", compressed.len(), ratio);
-    println!("  Reduction:   {:.1}%", 100.0 - ratio);
-    println!("  Compress:    {:.1} ms ({:.1} MB/s)", compress_time.as_secs_f64() * 1000.0, compress_mbps);
-    println!("  Decompress:  {:.1} ms ({:.1} MB/s)", decompress_time.as_secs_f64() * 1000.0, decompress_mbps);
-    println!();
-}
-
-fn main() {
-    println!("╔══════════════════════════════════════════════════════════╗");
-    println!("║     ROXIFY BWT-ANS COMPRESSION BENCHMARK               ║");
-    println!("╚══════════════════════════════════════════════════════════╝\n");
-
-    let text_1k: Vec<u8> = "Hello World! This is a test of the roxify compression engine. ".repeat(16).into_bytes();
-    bench_roundtrip("Text 1KB", &text_1k);
-    bench_zstd("Text 1KB", &text_1k, 3);
-    bench_zstd("Text 1KB", &text_1k, 19);
-
-    let text_100k: Vec<u8> = "The quick brown fox jumps over the lazy dog. Pack my box with five dozen liquor jugs. ".repeat(1200).into_bytes();
-    bench_roundtrip("Text 100KB", &text_100k);
-    bench_zstd("Text 100KB", &text_100k, 3);
-    bench_zstd("Text 100KB", &text_100k, 19);
-
-    let text_1m: Vec<u8> = {
-        let mut data = Vec::with_capacity(1_048_576);
-        let phrases = [
-            b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. ".as_slice(),
-            b"Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ".as_slice(),
-            b"Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris. ".as_slice(),
-            b"Duis aute irure dolor in reprehenderit in voluptate velit esse. ".as_slice(),
-            b"Excepteur sint occaecat cupidatat non proident, sunt in culpa. ".as_slice(),
-        ];
-        let mut i = 0;
-        while data.len() < 1_048_576 {
-            data.extend_from_slice(phrases[i % phrases.len()]);
-            i += 1;
-        }
-        data.truncate(1_048_576);
-        data
-    };
-    bench_roundtrip("Text 1MB", &text_1m);
-    bench_zstd("Text 1MB", &text_1m, 3);
-    bench_zstd("Text 1MB", &text_1m, 19);
-
-    let json_data: Vec<u8> = {
-        let mut data = String::with_capacity(512_000);
-        data.push('[');
-        for i in 0..5000 {
-            if i > 0 { data.push(','); }
-            data.push_str(&format!(
-                r#"{{"id":{},"name":"user_{}","email":"user{}@example.com","active":{},"score":{:.2},"tags":["tag1","tag2","tag3"]}}"#,
-                i, i, i, i % 2 == 0, (i as f64) * 1.337
-            ));
-        }
-        data.push(']');
-        data.into_bytes()
-    };
-    bench_roundtrip("JSON 500KB", &json_data);
-    bench_zstd("JSON 500KB", &json_data, 3);
-    bench_zstd("JSON 500KB", &json_data, 19);
-
-    let random_data: Vec<u8> = {
-        let mut data = vec![0u8; 100_000];
-        let mut state = 12345u64;
-        for b in data.iter_mut() {
-            state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
-            *b = (state >> 33) as u8;
-        }
-        data
-    };
-    bench_roundtrip("Random 100KB", &random_data);
-    bench_zstd("Random 100KB", &random_data, 3);
-
-    let binary_data: Vec<u8> = {
-        let mut data = Vec::with_capacity(256_000);
-        for i in 0..256_000u32 {
-            match i % 7 {
-                0 => data.push(0),
-                1 => data.push(0xFF),
-                2 => data.push((i & 0xFF) as u8),
-                3 => data.push(((i >> 8) & 0xFF) as u8),
-                4 => data.push(b'A' + (i % 26) as u8),
-                5 => data.push(0x20),
-                _ => data.push((i.wrapping_mul(37) & 0xFF) as u8),
-            }
-        }
-        data
-    };
-    bench_roundtrip("Binary 256KB", &binary_data);
-    bench_zstd("Binary 256KB", &binary_data, 3);
-    bench_zstd("Binary 256KB", &binary_data, 19);
-
-    println!("All round-trip tests PASSED!");
-}

package/native/bwt.rs

@@ -1,56 +0,0 @@
-use anyhow::Result;
-use libsais::bwt::Bwt;
-use libsais::typestate::OwnedBuffer;
-use libsais::BwtConstruction;
-use rayon::prelude::*;
-
-pub struct BwtResult {
-    pub transformed: Vec<u8>,
-    pub primary_index: u32,
-}
-
-pub fn bwt_encode(data: &[u8]) -> Result<BwtResult> {
-    let n = data.len();
-    if n == 0 {
-        return Ok(BwtResult { transformed: Vec::new(), primary_index: 0 });
-    }
-
-    let bwt_result = BwtConstruction::for_text(data)
-        .with_owned_temporary_array_buffer32()
-        .single_threaded()
-        .run()
-        .map_err(|e| anyhow::anyhow!("libsais BWT: {:?}", e))?;
-
-    let primary_index = bwt_result.primary_index() as u32;
-    let transformed = bwt_result.bwt().to_vec();
-
-    Ok(BwtResult { transformed, primary_index })
-}
-
-pub fn bwt_decode(bwt_data: &[u8], primary_index: u32) -> Result<Vec<u8>> {
-    if bwt_data.is_empty() {
-        return Ok(Vec::new());
-    }
-
-    let bwt_obj: Bwt<'static, u8, OwnedBuffer> =
-        unsafe { Bwt::from_parts(bwt_data.to_vec(), primary_index as usize) };
-
-    let text = bwt_obj
-        .unbwt()
-        .with_owned_temporary_array_buffer32()
-        .single_threaded()
-        .run()
-        .map_err(|e| anyhow::anyhow!("libsais UnBWT: {:?}", e))?;
-
-    Ok(text.as_slice().to_vec())
-}
-
-pub fn bwt_encode_streaming(block_size: usize, data: &[u8]) -> Result<Vec<(BwtResult, usize)>> {
-    data.par_chunks(block_size)
-        .enumerate()
-        .map(|(i, chunk)| {
-            let result = bwt_encode(chunk)?;
-            Ok((result, i * block_size))
-        })
-        .collect()
-}

package/native/context_mixing.rs

@@ -1,117 +0,0 @@
-#[derive(Clone, Copy, Debug)]
-pub struct ProbabilityEstimate {
-    pub p0: u32,
-    pub p1: u32,
-    pub total: u32,
-}
-
-impl ProbabilityEstimate {
-    pub fn entropy_bits(&self) -> f32 {
-        if self.total == 0 {
-            return 0.0;
-        }
-        let p0 = (self.p0 as f32) / (self.total as f32);
-        let p1 = (self.p1 as f32) / (self.total as f32);
-
-        let mut bits = 0.0;
-        if p0 > 0.0 {
-            bits -= p0 * p0.log2();
-        }
-        if p1 > 0.0 {
-            bits -= p1 * p1.log2();
-        }
-        bits
-    }
-}
-
-pub struct ContextMixer {
-    contexts_order0: Vec<ProbabilityEstimate>,
-    contexts_order1: Vec<[ProbabilityEstimate; 256]>,
-    contexts_order2: Vec<[[ProbabilityEstimate; 256]; 256]>,
-}
-
-impl ContextMixer {
-    pub fn new() -> Self {
-        ContextMixer {
-            contexts_order0: vec![ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 1],
-            contexts_order1: vec![
-                [ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 256];
-                256
-            ],
-            contexts_order2: vec![
-                [[ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 256]; 256];
-                256
-            ],
-        }
-    }
-
-    pub fn predict_order0(&self) -> ProbabilityEstimate {
-        self.contexts_order0[0]
-    }
-
-    pub fn predict_order1(&self, context1: u8) -> ProbabilityEstimate {
-        self.contexts_order1[context1 as usize][0]
-    }
-
-    pub fn predict_order2(&self, context1: u8, context2: u8) -> ProbabilityEstimate {
-        self.contexts_order2[context1 as usize][context2 as usize][0]
-    }
-
-    pub fn update_order0(&mut self, bit: bool) {
-        let ctx = &mut self.contexts_order0[0];
-        if bit {
-            ctx.p1 += 1;
-        } else {
-            ctx.p0 += 1;
-        }
-        ctx.total += 1;
-    }
-
-    pub fn update_order1(&mut self, context1: u8, bit: bool) {
-        let ctx = &mut self.contexts_order1[context1 as usize][0];
-        if bit {
-            ctx.p1 += 1;
-        } else {
-            ctx.p0 += 1;
-        }
-        ctx.total += 1;
-    }
-
-    pub fn update_order2(&mut self, context1: u8, context2: u8, bit: bool) {
-        let ctx = &mut self.contexts_order2[context1 as usize][context2 as usize][0];
-        if bit {
-            ctx.p1 += 1;
-        } else {
-            ctx.p0 += 1;
-        }
-        ctx.total += 1;
-    }
-}
-
-pub fn analyze_entropy(data: &[u8]) -> f32 {
-    let mut freq = [0u32; 256];
-    for &byte in data {
-        freq[byte as usize] += 1;
-    }
-
-    let total = data.len() as f32;
-    if total == 0.0 {
-        return 0.0;
-    }
-
-    let inv_total = 1.0 / total;
-    let mut entropy = 0.0f32;
-    for &f in &freq {
-        if f > 0 {
-            let p = f as f32 * inv_total;
-            entropy -= p * p.log2();
-        }
-    }
-    entropy
-}
-
-pub fn estimate_compression_gain(original: &[u8], entropy_bits: f32) -> f64 {
-    let theoretical_min = (original.len() as f64) * (entropy_bits as f64) / 8.0;
-    let ratio = theoretical_min / (original.len() as f64);
-    (1.0 - ratio) * 100.0
-}

package/native/core.rs

@@ -1,378 +0,0 @@
-use rayon::prelude::*;
-use std::sync::Arc;
-use std::path::PathBuf;
-use anyhow::Result;
-
-pub struct PlainScanResult {
-    pub marker_positions: Vec<u32>,
-    pub magic_positions: Vec<u32>,
-}
-
-pub fn scan_pixels_bytes(buf: &[u8], channels: usize, marker_bytes: Option<&[u8]>) -> PlainScanResult {
-    let magic = b"ROX1";
-
-    let magic_positions: Vec<u32> = if buf.len() >= 4 {
-        (0..(buf.len() - 3))
-            .into_par_iter()
-            .filter_map(|i| if &buf[i..i + 4] == magic { Some(i as u32) } else { None })
-            .collect()
-    } else {
-        Vec::new()
-    };
-
-    let markers: Vec<[u8; 3]> = match marker_bytes {
-        Some(bytes) if !bytes.is_empty() => {
-            if bytes.len() % 3 != 0 {
-                return PlainScanResult { marker_positions: Vec::new(), magic_positions };
-            }
-            bytes.chunks(3).map(|c| [c[0], c[1], c[2]]).collect()
-        }
-        _ => Vec::new(),
-    };
-
-    let marker_positions = if markers.is_empty() {
-        Vec::new()
-    } else {
-        let markers = Arc::new(markers);
-        let ch = channels as usize;
-        if ch < 3 || buf.len() < 3 {
-            Vec::new()
-        } else {
-            let pixel_count = buf.len() / ch;
-            (0..pixel_count)
-                .into_par_iter()
-                .filter_map(|i| {
-                    let base = i * ch;
-                    if base + 3 > buf.len() {
-                        return None;
-                    }
-                    for m in markers.iter() {
-                        if buf[base] == m[0] && buf[base + 1] == m[1] && buf[base + 2] == m[2] {
-                            return Some(i as u32);
-                        }
-                    }
-                    None
-                })
-                .collect()
-        }
-    };
-
-    PlainScanResult { marker_positions, magic_positions }
-}
-
-pub fn crc32_bytes(buf: &[u8]) -> u32 {
-    // parallelize checksum on large buffers, since crc32fast::hash is single-threaded
-    const PAR_THRESHOLD: usize = 4 * 1024 * 1024; // 4 MiB
-    if buf.len() < PAR_THRESHOLD {
-        crc32fast::hash(buf)
-    } else {
-        // compute per-chunk hasher in parallel then combine
-        let chunk = PAR_THRESHOLD;
-        let combined = buf
-            .par_chunks(chunk)
-            .map(|chunk| {
-                let mut h = crc32fast::Hasher::new();
-                h.update(chunk);
-                h
-            })
-            .reduce(|| crc32fast::Hasher::new(), |mut a, b| {
-                a.combine(&b);
-                a
-            });
-        combined.finalize()
-    }
-}
-
-pub fn adler32_bytes(buf: &[u8]) -> u32 {
-    let mut hasher = simd_adler32::Adler32::new();
-    hasher.write(buf);
-    hasher.finish()
-}
-
-pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {
-    let len = buf.len();
-    if len == 0 {
-        return Vec::new();
-    }
-    let mut out = vec![0u8; len];
-    out[0] = buf[0];
-    for i in 1..len {
-        out[i] = buf[i].wrapping_sub(buf[i - 1]);
-    }
-    out
-}
-
-pub fn delta_decode_bytes(buf: &[u8]) -> Vec<u8> {
-    let len = buf.len();
-    if len == 0 {
-        return Vec::new();
-    }
-    let mut out = vec![0u8; len];
-    out[0] = buf[0];
-    for i in 1..len {
-        out[i] = out[i - 1].wrapping_add(buf[i]);
-    }
-    out
-}
-
-
-
-pub fn train_zstd_dictionary(sample_paths: &[PathBuf], dict_size: usize) -> Result<Vec<u8>> {
-    // load all sample files contiguously
-    let mut samples = Vec::new();
-    let mut lengths = Vec::new();
-    for path in sample_paths {
-        let data = std::fs::read(path)?;
-        lengths.push(data.len());
-        samples.extend_from_slice(&data);
-    }
-    let dict = zstd::dict::from_continuous(&samples, &lengths, dict_size)?;
-    Ok(dict)
-}
-
-/// Compress a slice with optional zstd dictionary.
-///
-/// When `dict` is `Some`, the dictionary is passed to the encoder (same
-/// dict required for decompression).  Pass `None` for normal compression.
-///
-/// For large buffers (>50 MiB) without a dictionary, multiple chunk sizes
-/// are benchmarked on a sample and the best is selected automatically.
-pub fn zstd_compress_bytes(buf: &[u8], level: i32, dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
-    zstd_compress_with_prefix(buf, level, dict, &[])
-}
-
-fn compute_entropy_sample(buf: &[u8]) -> f32 {
-    let sample_size = buf.len().min(16384);
-    if sample_size == 0 {
-        return 4.0;
-    }
-    let sample = &buf[..sample_size];
-    let mut freq = [0u32; 256];
-    for &b in sample {
-        freq[b as usize] += 1;
-    }
-    let len = sample.len() as f32;
-    let mut ent: f32 = 0.0;
-    for &c in &freq {
-        if c > 0 {
-            let p = c as f32 / len;
-            ent -= p * p.log2();
-        }
-    }
-    ent
-}
-
-fn compute_adaptive_level(buf: &[u8], requested_level: i32, total_len: usize) -> i32 {
-    let entropy = compute_entropy_sample(buf);
-
-    let size_cap = match total_len {
-        s if s > 2 * 1024 * 1024 * 1024 => 1,
-        s if s > 1024 * 1024 * 1024 => match entropy {
-            e if e < 3.0 => 3,
-            _ => 1,
-        },
-        s if s > 256 * 1024 * 1024 => match entropy {
-            e if e < 3.0 => 6,
-            e if e < 5.0 => 3,
-            _ => 1,
-        },
-        s if s > 64 * 1024 * 1024 => match entropy {
-            e if e < 3.0 => 12,
-            e if e < 5.0 => 6,
-            e if e < 7.0 => 3,
-            _ => 1,
-        },
-        s if s > 16 * 1024 * 1024 => match entropy {
-            e if e < 3.0 => 15,
-            e if e < 5.0 => 9,
-            e if e < 7.0 => 6,
-            e if e < 7.5 => 3,
-            _ => 1,
-        },
-        s if s > 1024 * 1024 => match entropy {
-            e if e < 3.0 => 19,
-            e if e < 5.0 => 12,
-            e if e < 6.5 => 6,
-            e if e < 7.5 => 3,
-            _ => 1,
-        },
-        s if s > 64 * 1024 => match entropy {
-            e if e < 4.0 => 9,
-            e if e < 6.0 => 6,
-            e if e < 7.5 => 3,
-            _ => 1,
-        },
-        s if s > 4096 => match entropy {
-            e if e < 5.0 => 6,
-            e if e < 7.0 => 3,
-            _ => 1,
-        },
-        _ => match entropy {
-            e if e < 6.0 => 3,
-            _ => 1,
-        },
-    };
-
-    requested_level.min(size_cap)
-}
-
-pub fn zstd_compress_with_prefix(buf: &[u8], level: i32, dict: Option<&[u8]>, prefix: &[u8]) -> std::result::Result<Vec<u8>, String> {
-    use std::io::Write;
-
-    let actual_level = level.min(22).max(1);
-    let total_len = prefix.len() + buf.len();
-    let adaptive_level = compute_adaptive_level(buf, actual_level, total_len);
-
-    let estimated_output = if total_len < 1024 {
-        total_len
-    } else {
-        total_len * 3 / 4
-    };
-
-    if dict.is_none() && total_len < 4 * 1024 * 1024 {
-        if prefix.is_empty() {
-            return zstd::bulk::compress(buf, adaptive_level)
-                .map_err(|e| format!("zstd bulk compress error: {}", e));
-        }
-        let mut combined = Vec::with_capacity(total_len);
-        combined.extend_from_slice(prefix);
-        combined.extend_from_slice(buf);
-        return zstd::bulk::compress(&combined, adaptive_level)
-            .map_err(|e| format!("zstd bulk compress error: {}", e));
-    }
-
-    let mut encoder = if let Some(d) = dict {
-        zstd::stream::Encoder::with_dictionary(Vec::with_capacity(estimated_output), adaptive_level, d)
-            .map_err(|e| format!("zstd encoder init error: {}", e))?
-    } else {
-        zstd::stream::Encoder::new(Vec::with_capacity(estimated_output), adaptive_level)
-            .map_err(|e| format!("zstd encoder init error: {}", e))?
-    };
-
-    let threads = num_cpus::get() as u32;
-    if threads > 1 {
-        let max_threads = if adaptive_level >= 20 { threads.min(4) } else { threads };
-        let _ = encoder.multithread(max_threads);
-    }
-
-    if total_len > 256 * 1024 && adaptive_level >= 3 {
-        let _ = encoder.long_distance_matching(true);
-    }
-    if total_len > 256 * 1024 {
-        let wlog = if total_len > 1024 * 1024 * 1024 { 30 }
-            else if total_len > 512 * 1024 * 1024 { 29 }
-            else if total_len > 64 * 1024 * 1024 { 28 }
-            else if total_len > 8 * 1024 * 1024 { 27 }
-            else { 26 };
-        let _ = encoder.window_log(wlog);
-    }
-
-    let _ = encoder.set_pledged_src_size(Some(total_len as u64));
-
-    if !prefix.is_empty() {
-        encoder.write_all(prefix).map_err(|e| format!("zstd write prefix error: {}", e))?;
-    }
-
-    let chunk_size = if total_len > 256 * 1024 * 1024 { 16 * 1024 * 1024 }
-        else if total_len > 64 * 1024 * 1024 { 8 * 1024 * 1024 }
-        else { buf.len() };
-
-    for chunk in buf.chunks(chunk_size) {
-        encoder.write_all(chunk).map_err(|e| format!("zstd write error: {}", e))?;
-    }
-
-    encoder.finish().map_err(|e| format!("zstd finish error: {}", e))
-}
-
-pub fn zstd_decompress_bytes(buf: &[u8], dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
-    use std::io::Read;
-    let estimated = buf.len().saturating_mul(3).max(4096);
-    let mut out = Vec::with_capacity(estimated);
-    if let Some(d) = dict {
-        let mut decoder = zstd::stream::Decoder::with_dictionary(std::io::Cursor::new(buf), d)
-            .map_err(|e| format!("zstd decoder init error: {}", e))?;
-        decoder.window_log_max(31).map_err(|e| format!("zstd window_log_max error: {}", e))?;
-        decoder.read_to_end(&mut out).map_err(|e| format!("zstd decompress error: {}", e))?;
-    } else {
-        let mut decoder = zstd::stream::Decoder::new(std::io::Cursor::new(buf))
-            .map_err(|e| format!("zstd decoder init error: {}", e))?;
-        decoder.window_log_max(31).map_err(|e| format!("zstd window_log_max error: {}", e))?;
-        decoder.read_to_end(&mut out).map_err(|e| format!("zstd decompress error: {}", e))?;
-    }
-    Ok(out)
-}
-
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_scan_magic() {
-        let data = b"xxxxROX1yyyyROX1".to_vec();
-        let res = scan_pixels_bytes(&data, 3, None);
-        assert_eq!(res.magic_positions.len(), 2);
-    }
-
-    #[test]
-    fn test_markers() {
-        let pixels = vec![1u8,2,3, 4,5,6, 1,2,3];
-        let markers_vec = vec![1u8,2,3];
-        let res = scan_pixels_bytes(&pixels, 3, Some(&markers_vec));
-        assert_eq!(res.marker_positions, vec![0,2]);
-    }
-
-    #[test]
-    fn test_train_dictionary() {
-        use std::fs::{write, create_dir_all};
-        let td = std::env::temp_dir().join("rox_dict_test");
-        let _ = create_dir_all(&td);
-        let f1 = td.join("a.bin");
-        let f2 = td.join("b.bin");
-        // produce 1 MiB of repeated data per file
-        let big = vec![0xABu8; 1024 * 1024];
-        write(&f1, &big).unwrap();
-        write(&f2, &big).unwrap();
-        // choose dictionary size 16 KiB (far below total sample size ≈2 MiB)
-        match train_zstd_dictionary(&[f1.clone(), f2.clone()], 16 * 1024) {
-            Ok(dict) => {
-                assert!(dict.len() <= 16 * 1024);
-                assert!(!dict.is_empty());
-            }
-            Err(e) => {
-                // dictionary training may fail due to insufficient or unsuitable samples;
-                // ensure error string is nonempty to catch panics
-                assert!(!e.to_string().is_empty());
-            }
-        }
-    }
-
-    #[test]
-    fn test_delta_roundtrip() {
-        let data = vec![10u8, 20, 30, 40, 250];
-        let enc = delta_encode_bytes(&data);
-        let dec = delta_decode_bytes(&enc);
-        assert_eq!(dec, data);
-    }
-
-    #[test]
-    fn test_crc_adler() {
-        let data = b"hello".to_vec();
-        assert_eq!(crc32_bytes(&data), crc32fast::hash(&data));
-        assert_eq!(adler32_bytes(&data), adler32_bytes(&data));
-
-        // also test large buffer triggers parallel branch
-        let big = vec![0xAAu8; 5 * 1024 * 1024];
-        assert_eq!(crc32_bytes(&big), crc32fast::hash(&big));
-    }
-
-    #[test]
-    fn test_zstd_dict_roundtrip() {
-        let data = b"this is some test data that repeats. ".repeat(1000);
-        // simple dictionary containing a substring
-        let dict = b"test data";
-        let compressed = zstd_compress_bytes(&data, 3, Some(dict)).expect("compress");
-        let decompressed = zstd_decompress_bytes(&compressed, Some(dict)).expect("decompress");
-        assert_eq!(decompressed, data);
-    }
-}