roxify 1.10.1 → 1.11.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.10.1"
+version = "1.11.0"
 edition = "2021"
 publish = false
 

package/native/context_mixing.rs

@@ -1,5 +1,3 @@
-use rayon::prelude::*;
-
 #[derive(Clone, Copy, Debug)]
 pub struct ProbabilityEstimate {
     pub p0: u32,
@@ -91,26 +89,25 @@ impl ContextMixer {
 }
 
 pub fn analyze_entropy(data: &[u8]) -> f32 {
-    let freq: Vec<u32> = {
-        let mut f = vec![0u32; 256];
-        for &byte in data {
-            f[byte as usize] += 1;
-        }
-        f
-    };
+    let mut freq = [0u32; 256];
+    for &byte in data {
+        freq[byte as usize] += 1;
+    }
 
-    let total: u32 = freq.iter().sum();
-    if total == 0 {
+    let total = data.len() as f32;
+    if total == 0.0 {
         return 0.0;
     }
 
-    freq.par_iter()
-        .filter(|&&f| f > 0)
-        .map(|&f| {
-            let p = (f as f32) / (total as f32);
-            -p * p.log2()
-        })
-        .sum()
+    let inv_total = 1.0 / total;
+    let mut entropy = 0.0f32;
+    for &f in &freq {
+        if f > 0 {
+            let p = f as f32 * inv_total;
+            entropy -= p * p.log2();
+        }
+    }
+    entropy
 }
 
 pub fn estimate_compression_gain(original: &[u8], entropy_bits: f32) -> f64 {

package/native/hybrid.rs

@@ -5,7 +5,14 @@ use crate::mtf::{mtf_encode, mtf_decode, rle0_encode, rle0_decode};
 use crate::rans_byte::{SymbolStats, rans_encode_block, rans_decode_block};
 use crate::context_mixing::analyze_entropy;
 
-const BLOCK_SIZE: usize = 1024 * 1024;
+const BLOCK_SIZE: usize = 256 * 1024;
+
+const BLOCK_FLAG_BWT: u8 = 0;
+const BLOCK_FLAG_ZSTD: u8 = 1;
+const BLOCK_FLAG_STORE: u8 = 2;
+
+const ENTROPY_THRESHOLD_STORE: f32 = 7.95;
+const ENTROPY_THRESHOLD_ZSTD: f32 = 7.5;
 
 #[derive(Clone, Debug)]
 pub struct CompressionStats {
@@ -29,19 +36,24 @@ impl HybridCompressor {
 
     pub fn compress(&self, data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
         let original_size = data.len() as u64;
-        let entropy = analyze_entropy(data);
 
         let blocks: Vec<&[u8]> = data.chunks(self.block_size).collect();
         let blocks_count = blocks.len();
 
         let compressed_blocks: Vec<Vec<u8>> = blocks
-            .into_iter()
+            .par_iter()
             .map(|block| compress_block(block))
             .collect::<Result<Vec<_>, _>>()?;
 
+        let entropy = if data.len() > 4096 {
+            analyze_entropy(&data[..4096.min(data.len())])
+        } else {
+            analyze_entropy(data)
+        };
+
         let total_compressed: usize = compressed_blocks.iter().map(|b| b.len() + 4).sum();
         let mut result = Vec::with_capacity(16 + total_compressed);
-        result.extend_from_slice(b"RBW1");
+        result.extend_from_slice(b"RBW2");
         result.extend_from_slice(&(blocks_count as u32).to_le_bytes());
         result.extend_from_slice(&original_size.to_le_bytes());
 
@@ -67,7 +79,9 @@ impl HybridCompressor {
             return Err(anyhow::anyhow!("Invalid compressed data"));
         }
 
-        if &data[0..4] != b"RBW1" {
+        let magic = &data[0..4];
+        let v2 = magic == b"RBW2";
+        if magic != b"RBW1" && !v2 {
             return Err(anyhow::anyhow!("Invalid magic"));
         }
 
@@ -78,7 +92,7 @@ impl HybridCompressor {
         ]) as usize;
 
         let mut pos = 16;
-        let mut block_ranges: Vec<(usize, usize)> = Vec::with_capacity(blocks_count);
+        let mut block_slices: Vec<&[u8]> = Vec::with_capacity(blocks_count);
 
         for _ in 0..blocks_count {
             if pos + 4 > data.len() {
@@ -91,15 +105,18 @@ impl HybridCompressor {
             if pos + block_size > data.len() {
                 return Err(anyhow::anyhow!("Truncated block data"));
             }
-            block_ranges.push((pos, block_size));
+            block_slices.push(&data[pos..pos + block_size]);
             pos += block_size;
         }
 
-        let decompressed_blocks: Vec<Vec<u8>> = block_ranges
-            .into_iter()
-            .map(|(start, size)| {
-                let block_data = &data[start..start + size];
-                decompress_block(block_data)
+        let decompressed_blocks: Vec<Vec<u8>> = block_slices
+            .par_iter()
+            .map(|block_data| {
+                if v2 {
+                    decompress_block_v2(block_data)
+                } else {
+                    decompress_block_v1(block_data)
+                }
             })
             .collect::<Result<Vec<_>, _>>()?;
 
@@ -121,31 +138,121 @@ impl HybridCompressor {
 
 fn compress_block(block: &[u8]) -> Result<Vec<u8>> {
     if block.is_empty() {
-        return Ok(Vec::new());
+        return Ok(vec![BLOCK_FLAG_STORE]);
+    }
+
+    let entropy = analyze_entropy(block);
+
+    if entropy >= ENTROPY_THRESHOLD_STORE {
+        let mut result = Vec::with_capacity(1 + block.len());
+        result.push(BLOCK_FLAG_STORE);
+        result.extend_from_slice(block);
+        return Ok(result);
+    }
+
+    if entropy >= ENTROPY_THRESHOLD_ZSTD {
+        let compressed = zstd::encode_all(block, 1)?;
+        if compressed.len() < block.len() {
+            let mut result = Vec::with_capacity(1 + 4 + compressed.len());
+            result.push(BLOCK_FLAG_ZSTD);
+            result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+            result.extend_from_slice(&compressed);
+            return Ok(result);
+        }
+        let mut result = Vec::with_capacity(1 + block.len());
+        result.push(BLOCK_FLAG_STORE);
+        result.extend_from_slice(block);
+        return Ok(result);
     }
 
     let bwt = bwt_encode(block)?;
     let mtf_data = mtf_encode(&bwt.transformed);
     let rle_data = rle0_encode(&mtf_data);
-
     let stats = SymbolStats::from_data(&rle_data);
     let encoded = rans_encode_block(&rle_data, &stats);
-
     let stats_bytes = stats.serialize();
-    let rle_len = rle_data.len() as u32;
-    let orig_len = block.len() as u32;
 
-    let mut result = Vec::with_capacity(4 + 4 + 4 + stats_bytes.len() + encoded.len());
-    result.extend_from_slice(&bwt.primary_index.to_le_bytes());
-    result.extend_from_slice(&orig_len.to_le_bytes());
-    result.extend_from_slice(&rle_len.to_le_bytes());
-    result.extend_from_slice(&stats_bytes);
-    result.extend_from_slice(&encoded);
+    let bwt_total = 1 + 4 + 4 + 4 + stats_bytes.len() + encoded.len();
+
+    if bwt_total < block.len() {
+        let zstd_compressed = zstd::encode_all(block, 3)?;
+        let zstd_total = 1 + 4 + zstd_compressed.len();
 
+        if zstd_total < bwt_total {
+            let mut result = Vec::with_capacity(zstd_total);
+            result.push(BLOCK_FLAG_ZSTD);
+            result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+            result.extend_from_slice(&zstd_compressed);
+            return Ok(result);
+        }
+
+        let mut result = Vec::with_capacity(bwt_total);
+        result.push(BLOCK_FLAG_BWT);
+        result.extend_from_slice(&bwt.primary_index.to_le_bytes());
+        result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+        result.extend_from_slice(&(rle_data.len() as u32).to_le_bytes());
+        result.extend_from_slice(&stats_bytes);
+        result.extend_from_slice(&encoded);
+        return Ok(result);
+    }
+
+    let zstd_compressed = zstd::encode_all(block, 3)?;
+    if 1 + 4 + zstd_compressed.len() < block.len() {
+        let mut result = Vec::with_capacity(1 + 4 + zstd_compressed.len());
+        result.push(BLOCK_FLAG_ZSTD);
+        result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+        result.extend_from_slice(&zstd_compressed);
+        return Ok(result);
+    }
+
+    let mut result = Vec::with_capacity(1 + block.len());
+    result.push(BLOCK_FLAG_STORE);
+    result.extend_from_slice(block);
     Ok(result)
 }
 
-fn decompress_block(block: &[u8]) -> Result<Vec<u8>> {
+fn decompress_block_v2(block: &[u8]) -> Result<Vec<u8>> {
+    if block.is_empty() {
+        return Err(anyhow::anyhow!("Empty block"));
+    }
+
+    match block[0] {
+        BLOCK_FLAG_STORE => Ok(block[1..].to_vec()),
+        BLOCK_FLAG_ZSTD => {
+            if block.len() < 5 {
+                return Err(anyhow::anyhow!("Truncated zstd block"));
+            }
+            let orig_len = u32::from_le_bytes([block[1], block[2], block[3], block[4]]) as usize;
+            let mut decoded = zstd::decode_all(&block[5..])?;
+            decoded.truncate(orig_len);
+            Ok(decoded)
+        }
+        BLOCK_FLAG_BWT => {
+            if block.len() < 13 {
+                return Err(anyhow::anyhow!("Truncated BWT block"));
+            }
+            let primary_index = u32::from_le_bytes([block[1], block[2], block[3], block[4]]);
+            let orig_len = u32::from_le_bytes([block[5], block[6], block[7], block[8]]) as usize;
+            let rle_len = u32::from_le_bytes([block[9], block[10], block[11], block[12]]) as usize;
+
+            let (stats, stats_size) = SymbolStats::deserialize(&block[13..])?;
+            let encoded = &block[13 + stats_size..];
+
+            let rle_data = rans_decode_block(encoded, &stats, rle_len)?;
+            let mtf_data = rle0_decode(&rle_data);
+            let bwt_data = mtf_decode(&mtf_data);
+            let original = bwt_decode(&bwt_data, primary_index)?;
+
+            if original.len() != orig_len {
+                return Err(anyhow::anyhow!("Size mismatch"));
+            }
+            Ok(original)
+        }
+        _ => Err(anyhow::anyhow!("Unknown block type: {}", block[0])),
+    }
+}
+
+fn decompress_block_v1(block: &[u8]) -> Result<Vec<u8>> {
     if block.len() < 12 {
         return Err(anyhow::anyhow!("Block too small"));
     }
@@ -163,22 +270,18 @@ fn decompress_block(block: &[u8]) -> Result<Vec<u8>> {
     let original = bwt_decode(&bwt_data, primary_index)?;
 
     if original.len() != orig_len {
-        return Err(anyhow::anyhow!(
-            "Size mismatch: expected {}, got {}",
-            orig_len,
-            original.len()
-        ));
+        return Err(anyhow::anyhow!("Size mismatch"));
     }
 
     Ok(original)
 }
 
 pub fn compress_high_performance(data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
-    let compressor = HybridCompressor::new(false, 4);
+    let compressor = HybridCompressor::new(false, 0);
     compressor.compress(data)
 }
 
 pub fn decompress_high_performance(data: &[u8]) -> Result<Vec<u8>> {
-    let compressor = HybridCompressor::new(false, 4);
+    let compressor = HybridCompressor::new(false, 0);
     compressor.decompress(data)
 }

package/native/rans_byte.rs

@@ -106,10 +106,10 @@ impl SymbolStats {
     }
 }
 
+#[inline(always)]
 fn rans_enc_put(state: &mut u32, buf: &mut Vec<u8>, start: u32, freq: u32) {
-    let x = *state;
     let x_max = ((RANS_BYTE_L >> PROB_BITS) << 8) * freq;
-    let mut x = x;
+    let mut x = *state;
     while x >= x_max {
         buf.push((x & 0xFF) as u8);
         x >>= 8;
@@ -117,7 +117,22 @@ fn rans_enc_put(state: &mut u32, buf: &mut Vec<u8>, start: u32, freq: u32) {
     *state = ((x / freq) << PROB_BITS) + (x % freq) + start;
 }
 
-fn rans_dec_init(data: &[u8], pos: &mut usize) -> u32 {
+#[inline(always)]
+fn rans_dec_renorm(state: &mut u32, data: &[u8], pos: &mut usize) {
+    while *state < RANS_BYTE_L && *pos < data.len() {
+        *state = (*state << 8) | (data[*pos] as u32);
+        *pos += 1;
+    }
+}
+
+fn write_state(out: &mut Vec<u8>, state: u32) {
+    out.push((state >> 24) as u8);
+    out.push(((state >> 16) & 0xFF) as u8);
+    out.push(((state >> 8) & 0xFF) as u8);
+    out.push((state & 0xFF) as u8);
+}
+
+fn read_state(data: &[u8], pos: &mut usize) -> u32 {
     let s = (data[*pos] as u32) << 24
         | (data[*pos + 1] as u32) << 16
         | (data[*pos + 2] as u32) << 8
@@ -126,18 +141,48 @@ fn rans_dec_init(data: &[u8], pos: &mut usize) -> u32 {
     s
 }
 
-fn rans_dec_renorm(state: &mut u32, data: &[u8], pos: &mut usize) {
-    while *state < RANS_BYTE_L && *pos < data.len() {
-        *state = (*state << 8) | (data[*pos] as u32);
-        *pos += 1;
-    }
-}
-
 pub fn rans_encode_block(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
     if data.is_empty() {
         return Vec::new();
     }
 
+    if data.len() < 8 {
+        return rans_encode_single(data, stats);
+    }
+
+    let mut s0: u32 = RANS_BYTE_L;
+    let mut s1: u32 = RANS_BYTE_L;
+    let mut rev_bytes: Vec<u8> = Vec::with_capacity(data.len() + 32);
+
+    let len = data.len();
+    let even_start = if len % 2 == 0 { len } else { len - 1 };
+
+    if len % 2 != 0 {
+        let sym = data[len - 1] as usize;
+        rans_enc_put(&mut s1, &mut rev_bytes, stats.cum_freqs[sym], stats.freqs[sym]);
+    }
+
+    let mut i = even_start;
+    while i >= 2 {
+        i -= 2;
+        let sym1 = data[i + 1] as usize;
+        rans_enc_put(&mut s1, &mut rev_bytes, stats.cum_freqs[sym1], stats.freqs[sym1]);
+        let sym0 = data[i] as usize;
+        rans_enc_put(&mut s0, &mut rev_bytes, stats.cum_freqs[sym0], stats.freqs[sym0]);
+    }
+
+    let mut output = Vec::with_capacity(9 + rev_bytes.len());
+    output.push(1);
+    write_state(&mut output, s0);
+    write_state(&mut output, s1);
+
+    for &b in rev_bytes.iter().rev() {
+        output.push(b);
+    }
+    output
+}
+
+fn rans_encode_single(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
     let mut state: u32 = RANS_BYTE_L;
     let mut rev_bytes: Vec<u8> = Vec::with_capacity(data.len() + 16);
 
@@ -146,11 +191,9 @@ pub fn rans_encode_block(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
         rans_enc_put(&mut state, &mut rev_bytes, stats.cum_freqs[s], stats.freqs[s]);
     }
 
-    let mut output = Vec::with_capacity(4 + rev_bytes.len());
-    output.push((state >> 24) as u8);
-    output.push(((state >> 16) & 0xFF) as u8);
-    output.push(((state >> 8) & 0xFF) as u8);
-    output.push((state & 0xFF) as u8);
+    let mut output = Vec::with_capacity(5 + rev_bytes.len());
+    output.push(0);
+    write_state(&mut output, state);
 
     for &b in rev_bytes.iter().rev() {
         output.push(b);
@@ -159,19 +202,30 @@ pub fn rans_encode_block(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
 }
 
 pub fn rans_decode_block(encoded: &[u8], stats: &SymbolStats, output_len: usize) -> Result<Vec<u8>> {
-    if encoded.len() < 4 {
+    if encoded.is_empty() {
         return Err(anyhow::anyhow!("Data too short"));
     }
 
-    let mut cum2sym = vec![0u8; PROB_SCALE as usize];
+    let mut cum2sym = [0u8; PROB_SCALE as usize];
     for s in 0..256usize {
-        for slot in (stats.cum_freqs[s] as usize)..(stats.cum_freqs[s + 1] as usize) {
-            cum2sym[slot] = s as u8;
+        let start = stats.cum_freqs[s] as usize;
+        let end = stats.cum_freqs[s + 1] as usize;
+        if end > start {
+            cum2sym[start..end].fill(s as u8);
         }
     }
 
-    let mut pos = 0usize;
-    let mut state = rans_dec_init(encoded, &mut pos);
+    let mode = encoded[0];
+    let mut pos = 1usize;
+
+    if mode == 1 && output_len >= 8 {
+        return rans_decode_interleaved(encoded, &cum2sym, stats, output_len, &mut pos);
+    }
+
+    if pos + 4 > encoded.len() {
+        return Err(anyhow::anyhow!("Data too short"));
+    }
+    let mut state = read_state(encoded, &mut pos);
     let mut output = Vec::with_capacity(output_len);
 
     for _ in 0..output_len {
@@ -188,3 +242,45 @@ pub fn rans_decode_block(encoded: &[u8], stats: &SymbolStats, output_len: usize)
 
     Ok(output)
 }
+
+fn rans_decode_interleaved(
+    encoded: &[u8],
+    cum2sym: &[u8; PROB_SCALE as usize],
+    stats: &SymbolStats,
+    output_len: usize,
+    pos: &mut usize,
+) -> Result<Vec<u8>> {
+    if *pos + 8 > encoded.len() {
+        return Err(anyhow::anyhow!("Data too short for interleaved"));
+    }
+    let mut s0 = read_state(encoded, pos);
+    let mut s1 = read_state(encoded, pos);
+    let mut output = Vec::with_capacity(output_len);
+
+    let pairs = output_len / 2;
+    for _ in 0..pairs {
+        let slot0 = s0 & (PROB_SCALE - 1);
+        let sym0 = cum2sym[slot0 as usize];
+        output.push(sym0);
+        let freq0 = stats.freqs[sym0 as usize];
+        let start0 = stats.cum_freqs[sym0 as usize];
+        s0 = freq0 * (s0 >> PROB_BITS) + slot0 - start0;
+        rans_dec_renorm(&mut s0, encoded, pos);
+
+        let slot1 = s1 & (PROB_SCALE - 1);
+        let sym1 = cum2sym[slot1 as usize];
+        output.push(sym1);
+        let freq1 = stats.freqs[sym1 as usize];
+        let start1 = stats.cum_freqs[sym1 as usize];
+        s1 = freq1 * (s1 >> PROB_BITS) + slot1 - start1;
+        rans_dec_renorm(&mut s1, encoded, pos);
+    }
+
+    if output_len % 2 != 0 {
+        let slot = s1 & (PROB_SCALE - 1);
+        let sym = cum2sym[slot as usize];
+        output.push(sym);
+    }
+
+    Ok(output)
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "roxify",
-  "version": "1.10.1",
+  "version": "1.11.0",
   "type": "module",
   "description": "Ultra-lightweight PNG steganography with native Rust acceleration. Encode binary data into PNG images with zstd compression.",
   "main": "dist/index.js",