roxify 1.10.0 → 1.11.0

package/native/hybrid.rs

@@ -1,12 +1,18 @@
 use anyhow::Result;
 use rayon::prelude::*;
-use crate::bwt::bwt_encode;
+use crate::bwt::{bwt_encode, bwt_decode};
+use crate::mtf::{mtf_encode, mtf_decode, rle0_encode, rle0_decode};
+use crate::rans_byte::{SymbolStats, rans_encode_block, rans_decode_block};
 use crate::context_mixing::analyze_entropy;
-use crate::rans::{build_symbols_from_frequencies, RansEncoder};
-use crate::pool::BufferPool;
-use std::sync::Arc;
 
-const BLOCK_SIZE: usize = 16 * 1024 * 1024;
+const BLOCK_SIZE: usize = 256 * 1024;
+
+const BLOCK_FLAG_BWT: u8 = 0;
+const BLOCK_FLAG_ZSTD: u8 = 1;
+const BLOCK_FLAG_STORE: u8 = 2;
+
+const ENTROPY_THRESHOLD_STORE: f32 = 7.95;
+const ENTROPY_THRESHOLD_ZSTD: f32 = 7.5;
 
 #[derive(Clone, Debug)]
 pub struct CompressionStats {
@@ -18,34 +24,38 @@ pub struct CompressionStats {
 }
 
 pub struct HybridCompressor {
-    pool: Arc<BufferPool>,
-    enable_gpu: bool,
     block_size: usize,
 }
 
 impl HybridCompressor {
-    pub fn new(enable_gpu: bool, pool_size: usize) -> Self {
+    pub fn new(_enable_gpu: bool, _pool_size: usize) -> Self {
         HybridCompressor {
-            pool: Arc::new(BufferPool::new(pool_size, BLOCK_SIZE)),
-            enable_gpu,
             block_size: BLOCK_SIZE,
         }
     }
 
     pub fn compress(&self, data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
         let original_size = data.len() as u64;
-        let entropy = analyze_entropy(data);
 
         let blocks: Vec<&[u8]> = data.chunks(self.block_size).collect();
         let blocks_count = blocks.len();
 
         let compressed_blocks: Vec<Vec<u8>> = blocks
-            .into_par_iter()
-            .map(|block| self.compress_block(block))
+            .par_iter()
+            .map(|block| compress_block(block))
             .collect::<Result<Vec<_>, _>>()?;
 
-        let mut result = Vec::with_capacity(original_size as usize / 2);
+        let entropy = if data.len() > 4096 {
+            analyze_entropy(&data[..4096.min(data.len())])
+        } else {
+            analyze_entropy(data)
+        };
+
+        let total_compressed: usize = compressed_blocks.iter().map(|b| b.len() + 4).sum();
+        let mut result = Vec::with_capacity(16 + total_compressed);
+        result.extend_from_slice(b"RBW2");
         result.extend_from_slice(&(blocks_count as u32).to_le_bytes());
+        result.extend_from_slice(&original_size.to_le_bytes());
 
         for block in &compressed_blocks {
             result.extend_from_slice(&(block.len() as u32).to_le_bytes());
@@ -64,83 +74,58 @@ impl HybridCompressor {
         }))
     }
 
-    fn compress_block(&self, block: &[u8]) -> Result<Vec<u8>> {
-        if block.is_empty() {
-            return Ok(Vec::new());
+    pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>> {
+        if data.len() < 16 {
+            return Err(anyhow::anyhow!("Invalid compressed data"));
         }
 
-        let bwt = bwt_encode(block)?;
-        let bwt_data = &bwt.transformed;
-
-        let mut freqs = vec![0u32; 256];
-        for &byte in bwt_data {
-            freqs[byte as usize] += 1;
+        let magic = &data[0..4];
+        let v2 = magic == b"RBW2";
+        if magic != b"RBW1" && !v2 {
+            return Err(anyhow::anyhow!("Invalid magic"));
         }
 
-        let symbols = build_symbols_from_frequencies(&freqs);
-        let mut encoder = RansEncoder::new(symbols);
-
-        for &byte in bwt_data {
-            for bit_idx in 0..8 {
-                let bit = (byte >> bit_idx) & 1 == 1;
-                let symbol_idx = if bit { 1 } else { 0 };
-                let _ = encoder.encode(symbol_idx);
-            }
-        }
+        let blocks_count = u32::from_le_bytes([data[4], data[5], data[6], data[7]]) as usize;
+        let original_size = u64::from_le_bytes([
+            data[8], data[9], data[10], data[11],
+            data[12], data[13], data[14], data[15],
+        ]) as usize;
 
-        let encoded = encoder.finish();
-        let mut result = Vec::with_capacity(4 + encoded.len());
-        result.extend_from_slice(&bwt.primary_index.to_le_bytes());
-        result.extend_from_slice(&encoded);
-
-        Ok(result)
-    }
-
-    pub fn decompress(&self, data: &[u8]) -> Result<Vec<u8>> {
-        if data.len() < 4 {
-            return Err(anyhow::anyhow!("Invalid compressed data"));
-        }
-
-        let blocks_count = u32::from_le_bytes([data[0], data[1], data[2], data[3]]) as usize;
-        let mut pos = 4;
-        let mut result = Vec::new();
+        let mut pos = 16;
+        let mut block_slices: Vec<&[u8]> = Vec::with_capacity(blocks_count);
 
         for _ in 0..blocks_count {
             if pos + 4 > data.len() {
                 return Err(anyhow::anyhow!("Truncated block header"));
             }
-
             let block_size = u32::from_le_bytes([
-                data[pos],
-                data[pos + 1],
-                data[pos + 2],
-                data[pos + 3],
+                data[pos], data[pos + 1], data[pos + 2], data[pos + 3],
             ]) as usize;
             pos += 4;
-
             if pos + block_size > data.len() {
                 return Err(anyhow::anyhow!("Truncated block data"));
             }
-
-            let block_data = &data[pos..pos + block_size];
+            block_slices.push(&data[pos..pos + block_size]);
             pos += block_size;
-
-            let decompressed = self.decompress_block(block_data)?;
-            result.extend_from_slice(&decompressed);
         }
 
-        Ok(result)
-    }
+        let decompressed_blocks: Vec<Vec<u8>> = block_slices
+            .par_iter()
+            .map(|block_data| {
+                if v2 {
+                    decompress_block_v2(block_data)
+                } else {
+                    decompress_block_v1(block_data)
+                }
+            })
+            .collect::<Result<Vec<_>, _>>()?;
 
-    fn decompress_block(&self, block: &[u8]) -> Result<Vec<u8>> {
-        if block.len() < 4 {
-            return Ok(Vec::new());
+        let mut result = Vec::with_capacity(original_size);
+        for block in decompressed_blocks {
+            result.extend_from_slice(&block);
         }
 
-        let _primary_index = u32::from_le_bytes([block[0], block[1], block[2], block[3]]);
-        let _encoded_data = &block[4..];
-
-        Ok(Vec::new())
+        Ok(result)
     }
 
     pub fn estimate_gain(&self, data: &[u8]) -> f64 {
@@ -151,12 +136,152 @@ impl HybridCompressor {
     }
 }
 
+fn compress_block(block: &[u8]) -> Result<Vec<u8>> {
+    if block.is_empty() {
+        return Ok(vec![BLOCK_FLAG_STORE]);
+    }
+
+    let entropy = analyze_entropy(block);
+
+    if entropy >= ENTROPY_THRESHOLD_STORE {
+        let mut result = Vec::with_capacity(1 + block.len());
+        result.push(BLOCK_FLAG_STORE);
+        result.extend_from_slice(block);
+        return Ok(result);
+    }
+
+    if entropy >= ENTROPY_THRESHOLD_ZSTD {
+        let compressed = zstd::encode_all(block, 1)?;
+        if compressed.len() < block.len() {
+            let mut result = Vec::with_capacity(1 + 4 + compressed.len());
+            result.push(BLOCK_FLAG_ZSTD);
+            result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+            result.extend_from_slice(&compressed);
+            return Ok(result);
+        }
+        let mut result = Vec::with_capacity(1 + block.len());
+        result.push(BLOCK_FLAG_STORE);
+        result.extend_from_slice(block);
+        return Ok(result);
+    }
+
+    let bwt = bwt_encode(block)?;
+    let mtf_data = mtf_encode(&bwt.transformed);
+    let rle_data = rle0_encode(&mtf_data);
+    let stats = SymbolStats::from_data(&rle_data);
+    let encoded = rans_encode_block(&rle_data, &stats);
+    let stats_bytes = stats.serialize();
+
+    let bwt_total = 1 + 4 + 4 + 4 + stats_bytes.len() + encoded.len();
+
+    if bwt_total < block.len() {
+        let zstd_compressed = zstd::encode_all(block, 3)?;
+        let zstd_total = 1 + 4 + zstd_compressed.len();
+
+        if zstd_total < bwt_total {
+            let mut result = Vec::with_capacity(zstd_total);
+            result.push(BLOCK_FLAG_ZSTD);
+            result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+            result.extend_from_slice(&zstd_compressed);
+            return Ok(result);
+        }
+
+        let mut result = Vec::with_capacity(bwt_total);
+        result.push(BLOCK_FLAG_BWT);
+        result.extend_from_slice(&bwt.primary_index.to_le_bytes());
+        result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+        result.extend_from_slice(&(rle_data.len() as u32).to_le_bytes());
+        result.extend_from_slice(&stats_bytes);
+        result.extend_from_slice(&encoded);
+        return Ok(result);
+    }
+
+    let zstd_compressed = zstd::encode_all(block, 3)?;
+    if 1 + 4 + zstd_compressed.len() < block.len() {
+        let mut result = Vec::with_capacity(1 + 4 + zstd_compressed.len());
+        result.push(BLOCK_FLAG_ZSTD);
+        result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+        result.extend_from_slice(&zstd_compressed);
+        return Ok(result);
+    }
+
+    let mut result = Vec::with_capacity(1 + block.len());
+    result.push(BLOCK_FLAG_STORE);
+    result.extend_from_slice(block);
+    Ok(result)
+}
+
+fn decompress_block_v2(block: &[u8]) -> Result<Vec<u8>> {
+    if block.is_empty() {
+        return Err(anyhow::anyhow!("Empty block"));
+    }
+
+    match block[0] {
+        BLOCK_FLAG_STORE => Ok(block[1..].to_vec()),
+        BLOCK_FLAG_ZSTD => {
+            if block.len() < 5 {
+                return Err(anyhow::anyhow!("Truncated zstd block"));
+            }
+            let orig_len = u32::from_le_bytes([block[1], block[2], block[3], block[4]]) as usize;
+            let mut decoded = zstd::decode_all(&block[5..])?;
+            decoded.truncate(orig_len);
+            Ok(decoded)
+        }
+        BLOCK_FLAG_BWT => {
+            if block.len() < 13 {
+                return Err(anyhow::anyhow!("Truncated BWT block"));
+            }
+            let primary_index = u32::from_le_bytes([block[1], block[2], block[3], block[4]]);
+            let orig_len = u32::from_le_bytes([block[5], block[6], block[7], block[8]]) as usize;
+            let rle_len = u32::from_le_bytes([block[9], block[10], block[11], block[12]]) as usize;
+
+            let (stats, stats_size) = SymbolStats::deserialize(&block[13..])?;
+            let encoded = &block[13 + stats_size..];
+
+            let rle_data = rans_decode_block(encoded, &stats, rle_len)?;
+            let mtf_data = rle0_decode(&rle_data);
+            let bwt_data = mtf_decode(&mtf_data);
+            let original = bwt_decode(&bwt_data, primary_index)?;
+
+            if original.len() != orig_len {
+                return Err(anyhow::anyhow!("Size mismatch"));
+            }
+            Ok(original)
+        }
+        _ => Err(anyhow::anyhow!("Unknown block type: {}", block[0])),
+    }
+}
+
+fn decompress_block_v1(block: &[u8]) -> Result<Vec<u8>> {
+    if block.len() < 12 {
+        return Err(anyhow::anyhow!("Block too small"));
+    }
+
+    let primary_index = u32::from_le_bytes([block[0], block[1], block[2], block[3]]);
+    let orig_len = u32::from_le_bytes([block[4], block[5], block[6], block[7]]) as usize;
+    let rle_len = u32::from_le_bytes([block[8], block[9], block[10], block[11]]) as usize;
+
+    let (stats, stats_size) = SymbolStats::deserialize(&block[12..])?;
+    let encoded = &block[12 + stats_size..];
+
+    let rle_data = rans_decode_block(encoded, &stats, rle_len)?;
+    let mtf_data = rle0_decode(&rle_data);
+    let bwt_data = mtf_decode(&mtf_data);
+    let original = bwt_decode(&bwt_data, primary_index)?;
+
+    if original.len() != orig_len {
+        return Err(anyhow::anyhow!("Size mismatch"));
+    }
+
+    Ok(original)
+}
+
 pub fn compress_high_performance(data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
-    let compressor = HybridCompressor::new(true, 4);
+    let compressor = HybridCompressor::new(false, 0);
     compressor.compress(data)
 }
 
 pub fn decompress_high_performance(data: &[u8]) -> Result<Vec<u8>> {
-    let compressor = HybridCompressor::new(true, 4);
+    let compressor = HybridCompressor::new(false, 0);
     compressor.decompress(data)
 }

package/native/lib.rs

@@ -9,7 +9,9 @@ mod core;
 #[cfg(feature = "gpu")]
 mod gpu;
 mod rans;
+mod rans_byte;
 mod bwt;
+mod mtf;
 mod context_mixing;
 mod pool;
 mod hybrid;
@@ -150,17 +152,18 @@ pub fn entropy_estimate(buffer: Buffer) -> f32 {
 
 #[cfg(not(test))]
 #[napi]
-pub fn hybrid_compress(buffer: Buffer) -> Result<Vec<u8>> {
+pub fn hybrid_compress(buffer: Buffer) -> Result<Buffer> {
     match hybrid::compress_high_performance(&buffer) {
-        Ok((compressed, _stats)) => Ok(compressed),
+        Ok((compressed, _stats)) => Ok(Buffer::from(compressed)),
         Err(e) => Err(Error::from_reason(e.to_string())),
     }
 }
 
 #[cfg(not(test))]
 #[napi]
-pub fn hybrid_decompress(buffer: Buffer) -> Result<Vec<u8>> {
+pub fn hybrid_decompress(buffer: Buffer) -> Result<Buffer> {
     hybrid::decompress_high_performance(&buffer)
+        .map(Buffer::from)
         .map_err(|e| Error::from_reason(e.to_string()))
 }
 

package/native/mtf.rs

@@ -0,0 +1,106 @@
+pub fn mtf_encode(data: &[u8]) -> Vec<u8> {
+    let mut table = [0u8; 256];
+    for i in 0..256 {
+        table[i] = i as u8;
+    }
+    let mut output = Vec::with_capacity(data.len());
+
+    for &byte in data {
+        let pos = unsafe { table.iter().position(|&b| b == byte).unwrap_unchecked() };
+        output.push(pos as u8);
+        if pos > 0 {
+            let val = table[pos];
+            table.copy_within(0..pos, 1);
+            table[0] = val;
+        }
+    }
+
+    output
+}
+
+pub fn mtf_decode(data: &[u8]) -> Vec<u8> {
+    let mut table = [0u8; 256];
+    for i in 0..256 {
+        table[i] = i as u8;
+    }
+    let mut output = Vec::with_capacity(data.len());
+
+    for &idx in data {
+        let pos = idx as usize;
+        let byte = table[pos];
+        output.push(byte);
+        if pos > 0 {
+            let val = table[pos];
+            table.copy_within(0..pos, 1);
+            table[0] = val;
+        }
+    }
+
+    output
+}
+
+pub fn rle0_encode(data: &[u8]) -> Vec<u8> {
+    let mut output = Vec::with_capacity(data.len());
+    let mut i = 0;
+
+    while i < data.len() {
+        if data[i] == 0 {
+            let mut run = 0u32;
+            while i < data.len() && data[i] == 0 {
+                run += 1;
+                i += 1;
+            }
+            output.push(0);
+            if run <= 127 {
+                output.push(run as u8);
+            } else if run <= 16383 {
+                output.push(0x80 | ((run >> 8) as u8));
+                output.push((run & 0xFF) as u8);
+            } else {
+                output.push(0xC0 | ((run >> 16) as u8));
+                output.push(((run >> 8) & 0xFF) as u8);
+                output.push((run & 0xFF) as u8);
+            }
+        } else {
+            output.push(data[i]);
+            i += 1;
+        }
+    }
+
+    output
+}
+
+pub fn rle0_decode(data: &[u8]) -> Vec<u8> {
+    let mut output = Vec::with_capacity(data.len() * 2);
+    let mut i = 0;
+
+    while i < data.len() {
+        if data[i] == 0 {
+            i += 1;
+            if i >= data.len() {
+                break;
+            }
+            let run;
+            if data[i] & 0xC0 == 0xC0 {
+                let hi = (data[i] & 0x3F) as u32;
+                run = (hi << 16) | ((data[i + 1] as u32) << 8) | (data[i + 2] as u32);
+                i += 3;
+            } else if data[i] & 0x80 != 0 {
+                let hi = (data[i] & 0x7F) as u32;
+                run = (hi << 8) | (data[i + 1] as u32);
+                i += 2;
+            } else {
+                run = data[i] as u32;
+                i += 1;
+            }
+            for _ in 0..run {
+                output.push(0);
+            }
+        } else {
+            output.push(data[i]);
+            i += 1;
+        }
+    }
+
+    output
+}