roxify 1.10.1 → 1.11.1

package/native/hybrid.rs

@@ -7,6 +7,13 @@ use crate::context_mixing::analyze_entropy;
 
 const BLOCK_SIZE: usize = 1024 * 1024;
 
+const BLOCK_FLAG_BWT: u8 = 0;
+const BLOCK_FLAG_ZSTD: u8 = 1;
+const BLOCK_FLAG_STORE: u8 = 2;
+
+const ENTROPY_THRESHOLD_STORE: f32 = 7.95;
+const ENTROPY_THRESHOLD_ZSTD: f32 = 7.5;
+
 #[derive(Clone, Debug)]
 pub struct CompressionStats {
     pub original_size: u64,
@@ -29,19 +36,24 @@ impl HybridCompressor {
 
     pub fn compress(&self, data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
         let original_size = data.len() as u64;
-        let entropy = analyze_entropy(data);
 
         let blocks: Vec<&[u8]> = data.chunks(self.block_size).collect();
         let blocks_count = blocks.len();
 
         let compressed_blocks: Vec<Vec<u8>> = blocks
-            .into_iter()
+            .par_iter()
             .map(|block| compress_block(block))
             .collect::<Result<Vec<_>, _>>()?;
 
+        let entropy = if data.len() > 4096 {
+            analyze_entropy(&data[..4096.min(data.len())])
+        } else {
+            analyze_entropy(data)
+        };
+
         let total_compressed: usize = compressed_blocks.iter().map(|b| b.len() + 4).sum();
         let mut result = Vec::with_capacity(16 + total_compressed);
-        result.extend_from_slice(b"RBW1");
+        result.extend_from_slice(b"RBW2");
         result.extend_from_slice(&(blocks_count as u32).to_le_bytes());
         result.extend_from_slice(&original_size.to_le_bytes());
 
@@ -67,7 +79,9 @@ impl HybridCompressor {
             return Err(anyhow::anyhow!("Invalid compressed data"));
         }
 
-        if &data[0..4] != b"RBW1" {
+        let magic = &data[0..4];
+        let v2 = magic == b"RBW2";
+        if magic != b"RBW1" && !v2 {
             return Err(anyhow::anyhow!("Invalid magic"));
         }
 
@@ -78,7 +92,7 @@ impl HybridCompressor {
         ]) as usize;
 
         let mut pos = 16;
-        let mut block_ranges: Vec<(usize, usize)> = Vec::with_capacity(blocks_count);
+        let mut block_slices: Vec<&[u8]> = Vec::with_capacity(blocks_count);
 
         for _ in 0..blocks_count {
             if pos + 4 > data.len() {
@@ -91,15 +105,18 @@ impl HybridCompressor {
             if pos + block_size > data.len() {
                 return Err(anyhow::anyhow!("Truncated block data"));
             }
-            block_ranges.push((pos, block_size));
+            block_slices.push(&data[pos..pos + block_size]);
             pos += block_size;
         }
 
-        let decompressed_blocks: Vec<Vec<u8>> = block_ranges
-            .into_iter()
-            .map(|(start, size)| {
-                let block_data = &data[start..start + size];
-                decompress_block(block_data)
+        let decompressed_blocks: Vec<Vec<u8>> = block_slices
+            .par_iter()
+            .map(|block_data| {
+                if v2 {
+                    decompress_block_v2(block_data)
+                } else {
+                    decompress_block_v1(block_data)
+                }
             })
             .collect::<Result<Vec<_>, _>>()?;
 
@@ -121,31 +138,121 @@ impl HybridCompressor {
 
 fn compress_block(block: &[u8]) -> Result<Vec<u8>> {
     if block.is_empty() {
-        return Ok(Vec::new());
+        return Ok(vec![BLOCK_FLAG_STORE]);
+    }
+
+    let entropy = analyze_entropy(block);
+
+    if entropy >= ENTROPY_THRESHOLD_STORE {
+        let mut result = Vec::with_capacity(1 + block.len());
+        result.push(BLOCK_FLAG_STORE);
+        result.extend_from_slice(block);
+        return Ok(result);
+    }
+
+    if entropy >= ENTROPY_THRESHOLD_ZSTD {
+        let compressed = zstd::encode_all(block, 1)?;
+        if compressed.len() < block.len() {
+            let mut result = Vec::with_capacity(1 + 4 + compressed.len());
+            result.push(BLOCK_FLAG_ZSTD);
+            result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+            result.extend_from_slice(&compressed);
+            return Ok(result);
+        }
+        let mut result = Vec::with_capacity(1 + block.len());
+        result.push(BLOCK_FLAG_STORE);
+        result.extend_from_slice(block);
+        return Ok(result);
     }
 
     let bwt = bwt_encode(block)?;
     let mtf_data = mtf_encode(&bwt.transformed);
     let rle_data = rle0_encode(&mtf_data);
-
     let stats = SymbolStats::from_data(&rle_data);
     let encoded = rans_encode_block(&rle_data, &stats);
-
     let stats_bytes = stats.serialize();
-    let rle_len = rle_data.len() as u32;
-    let orig_len = block.len() as u32;
 
-    let mut result = Vec::with_capacity(4 + 4 + 4 + stats_bytes.len() + encoded.len());
-    result.extend_from_slice(&bwt.primary_index.to_le_bytes());
-    result.extend_from_slice(&orig_len.to_le_bytes());
-    result.extend_from_slice(&rle_len.to_le_bytes());
-    result.extend_from_slice(&stats_bytes);
-    result.extend_from_slice(&encoded);
+    let bwt_total = 1 + 4 + 4 + 4 + stats_bytes.len() + encoded.len();
+
+    if bwt_total < block.len() {
+        let zstd_compressed = zstd::encode_all(block, 3)?;
+        let zstd_total = 1 + 4 + zstd_compressed.len();
 
+        if zstd_total < bwt_total {
+            let mut result = Vec::with_capacity(zstd_total);
+            result.push(BLOCK_FLAG_ZSTD);
+            result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+            result.extend_from_slice(&zstd_compressed);
+            return Ok(result);
+        }
+
+        let mut result = Vec::with_capacity(bwt_total);
+        result.push(BLOCK_FLAG_BWT);
+        result.extend_from_slice(&bwt.primary_index.to_le_bytes());
+        result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+        result.extend_from_slice(&(rle_data.len() as u32).to_le_bytes());
+        result.extend_from_slice(&stats_bytes);
+        result.extend_from_slice(&encoded);
+        return Ok(result);
+    }
+
+    let zstd_compressed = zstd::encode_all(block, 3)?;
+    if 1 + 4 + zstd_compressed.len() < block.len() {
+        let mut result = Vec::with_capacity(1 + 4 + zstd_compressed.len());
+        result.push(BLOCK_FLAG_ZSTD);
+        result.extend_from_slice(&(block.len() as u32).to_le_bytes());
+        result.extend_from_slice(&zstd_compressed);
+        return Ok(result);
+    }
+
+    let mut result = Vec::with_capacity(1 + block.len());
+    result.push(BLOCK_FLAG_STORE);
+    result.extend_from_slice(block);
     Ok(result)
 }
 
-fn decompress_block(block: &[u8]) -> Result<Vec<u8>> {
+fn decompress_block_v2(block: &[u8]) -> Result<Vec<u8>> {
+    if block.is_empty() {
+        return Err(anyhow::anyhow!("Empty block"));
+    }
+
+    match block[0] {
+        BLOCK_FLAG_STORE => Ok(block[1..].to_vec()),
+        BLOCK_FLAG_ZSTD => {
+            if block.len() < 5 {
+                return Err(anyhow::anyhow!("Truncated zstd block"));
+            }
+            let orig_len = u32::from_le_bytes([block[1], block[2], block[3], block[4]]) as usize;
+            let mut decoded = zstd::decode_all(&block[5..])?;
+            decoded.truncate(orig_len);
+            Ok(decoded)
+        }
+        BLOCK_FLAG_BWT => {
+            if block.len() < 13 {
+                return Err(anyhow::anyhow!("Truncated BWT block"));
+            }
+            let primary_index = u32::from_le_bytes([block[1], block[2], block[3], block[4]]);
+            let orig_len = u32::from_le_bytes([block[5], block[6], block[7], block[8]]) as usize;
+            let rle_len = u32::from_le_bytes([block[9], block[10], block[11], block[12]]) as usize;
+
+            let (stats, stats_size) = SymbolStats::deserialize(&block[13..])?;
+            let encoded = &block[13 + stats_size..];
+
+            let rle_data = rans_decode_block(encoded, &stats, rle_len)?;
+            let mtf_data = rle0_decode(&rle_data);
+            let bwt_data = mtf_decode(&mtf_data);
+            let original = bwt_decode(&bwt_data, primary_index)?;
+
+            if original.len() != orig_len {
+                return Err(anyhow::anyhow!("Size mismatch"));
+            }
+            Ok(original)
+        }
+        _ => Err(anyhow::anyhow!("Unknown block type: {}", block[0])),
+    }
+}
+
+fn decompress_block_v1(block: &[u8]) -> Result<Vec<u8>> {
     if block.len() < 12 {
         return Err(anyhow::anyhow!("Block too small"));
     }
@@ -163,22 +270,18 @@ fn decompress_block(block: &[u8]) -> Result<Vec<u8>> {
     let original = bwt_decode(&bwt_data, primary_index)?;
 
     if original.len() != orig_len {
-        return Err(anyhow::anyhow!(
-            "Size mismatch: expected {}, got {}",
-            orig_len,
-            original.len()
-        ));
+        return Err(anyhow::anyhow!("Size mismatch"));
     }
 
     Ok(original)
 }
 
 pub fn compress_high_performance(data: &[u8]) -> Result<(Vec<u8>, CompressionStats)> {
-    let compressor = HybridCompressor::new(false, 4);
+    let compressor = HybridCompressor::new(false, 0);
     compressor.compress(data)
 }
 
 pub fn decompress_high_performance(data: &[u8]) -> Result<Vec<u8>> {
-    let compressor = HybridCompressor::new(false, 4);
+    let compressor = HybridCompressor::new(false, 0);
     compressor.decompress(data)
 }

package/native/lib.rs

@@ -24,6 +24,7 @@ mod audio;
 mod progress;
 mod reconstitution;
 mod archive;
+mod streaming;
 
 pub use core::*;
 #[cfg(feature = "gpu")]

package/native/main.rs

@@ -14,6 +14,7 @@ mod png_utils;
 mod audio;
 mod reconstitution;
 mod archive;
+mod streaming;
 
 use crate::encoder::ImageFormat;
 use std::path::PathBuf;
@@ -104,15 +105,28 @@ enum Commands {
 }
 
 fn read_all(path: &PathBuf) -> anyhow::Result<Vec<u8>> {
-    let mut f = File::open(path)?;
-    let mut buf = Vec::new();
-    f.read_to_end(&mut buf)?;
-    Ok(buf)
+    let metadata = std::fs::metadata(path)?;
+    let size = metadata.len() as usize;
+
+    if size > 256 * 1024 * 1024 {
+        let file = File::open(path)?;
+        let mmap = unsafe { memmap2::Mmap::map(&file)? };
+        Ok(mmap.to_vec())
+    } else {
+        let mut f = File::open(path)?;
+        let mut buf = Vec::with_capacity(size);
+        f.read_to_end(&mut buf)?;
+        Ok(buf)
+    }
 }
 
 fn write_all(path: &PathBuf, data: &[u8]) -> anyhow::Result<()> {
-    let mut f = File::create(path)?;
-    f.write_all(data)?;
+    let f = File::create(path)?;
+    let buf_size = if data.len() > 64 * 1024 * 1024 { 16 * 1024 * 1024 }
+        else { (8 * 1024 * 1024).min(data.len().max(8192)) };
+    let mut writer = std::io::BufWriter::with_capacity(buf_size, f);
+    writer.write_all(data)?;
+    writer.flush()?;
     Ok(())
 }
 
@@ -144,14 +158,12 @@ fn main() -> anyhow::Result<()> {
         Commands::Encode { input, output, level, passphrase, encrypt, name, dict } => {
             let is_dir = input.is_dir();
             let (payload, file_list_json) = if is_dir {
-                let tar_data = archive::tar_pack_directory(&input)
+                let result = archive::tar_pack_directory_with_list(&input)
                     .map_err(|e| anyhow::anyhow!(e))?;
-                let list = archive::tar_file_list(&tar_data)
-                    .map_err(|e| anyhow::anyhow!(e))?;
-                let json_list: Vec<serde_json::Value> = list.iter()
+                let json_list: Vec<serde_json::Value> = result.file_list.iter()
                     .map(|(name, size)| serde_json::json!({"name": name, "size": size}))
                     .collect();
-                (tar_data, Some(serde_json::to_string(&json_list)?))
+                (result.data, Some(serde_json::to_string(&json_list)?))
             } else {
                 let pack_result = packer::pack_path_with_metadata(&input)?;
                 (pack_result.data, pack_result.file_list_json)
@@ -165,31 +177,45 @@ fn main() -> anyhow::Result<()> {
                 None => None,
             };
 
-            let png = if let Some(ref pass) = passphrase {
-                encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+            let use_streaming = payload.len() > 64 * 1024 * 1024;
+
+            if use_streaming {
+                streaming::encode_to_png_file(
                     &payload,
+                    &output,
                     level,
-                    Some(pass),
+                    passphrase.as_deref(),
                     Some(&encrypt),
-                    ImageFormat::Png,
                     file_name,
                     file_list_json.as_deref(),
                     dict_bytes.as_deref(),
-                )?
+                )?;
             } else {
-                encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
-                    &payload,
-                    level,
-                    None,
-                    None,
-                    ImageFormat::Png,
-                    file_name,
-                    file_list_json.as_deref(),
-                    dict_bytes.as_deref(),
-                )?
-            };
-
-            write_all(&output, &png)?;
+                let png = if let Some(ref pass) = passphrase {
+                    encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+                        &payload,
+                        level,
+                        Some(pass),
+                        Some(&encrypt),
+                        ImageFormat::Png,
+                        file_name,
+                        file_list_json.as_deref(),
+                        dict_bytes.as_deref(),
+                    )?
+                } else {
+                    encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+                        &payload,
+                        level,
+                        None,
+                        None,
+                        ImageFormat::Png,
+                        file_name,
+                        file_list_json.as_deref(),
+                        dict_bytes.as_deref(),
+                    )?
+                };
+                write_all(&output, &png)?;
+            }
 
             if file_list_json.is_some() {
                 if is_dir {

package/native/rans_byte.rs

@@ -106,10 +106,10 @@ impl SymbolStats {
     }
 }
 
+#[inline(always)]
 fn rans_enc_put(state: &mut u32, buf: &mut Vec<u8>, start: u32, freq: u32) {
-    let x = *state;
     let x_max = ((RANS_BYTE_L >> PROB_BITS) << 8) * freq;
-    let mut x = x;
+    let mut x = *state;
     while x >= x_max {
         buf.push((x & 0xFF) as u8);
         x >>= 8;
@@ -117,7 +117,22 @@ fn rans_enc_put(state: &mut u32, buf: &mut Vec<u8>, start: u32, freq: u32) {
     *state = ((x / freq) << PROB_BITS) + (x % freq) + start;
 }
 
-fn rans_dec_init(data: &[u8], pos: &mut usize) -> u32 {
+#[inline(always)]
+fn rans_dec_renorm(state: &mut u32, data: &[u8], pos: &mut usize) {
+    while *state < RANS_BYTE_L && *pos < data.len() {
+        *state = (*state << 8) | (data[*pos] as u32);
+        *pos += 1;
+    }
+}
+
+fn write_state(out: &mut Vec<u8>, state: u32) {
+    out.push((state >> 24) as u8);
+    out.push(((state >> 16) & 0xFF) as u8);
+    out.push(((state >> 8) & 0xFF) as u8);
+    out.push((state & 0xFF) as u8);
+}
+
+fn read_state(data: &[u8], pos: &mut usize) -> u32 {
     let s = (data[*pos] as u32) << 24
         | (data[*pos + 1] as u32) << 16
         | (data[*pos + 2] as u32) << 8
@@ -126,18 +141,48 @@ fn rans_dec_init(data: &[u8], pos: &mut usize) -> u32 {
     s
 }
 
-fn rans_dec_renorm(state: &mut u32, data: &[u8], pos: &mut usize) {
-    while *state < RANS_BYTE_L && *pos < data.len() {
-        *state = (*state << 8) | (data[*pos] as u32);
-        *pos += 1;
-    }
-}
-
 pub fn rans_encode_block(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
     if data.is_empty() {
         return Vec::new();
     }
 
+    if data.len() < 8 {
+        return rans_encode_single(data, stats);
+    }
+
+    let mut s0: u32 = RANS_BYTE_L;
+    let mut s1: u32 = RANS_BYTE_L;
+    let mut rev_bytes: Vec<u8> = Vec::with_capacity(data.len() + 32);
+
+    let len = data.len();
+    let even_start = if len % 2 == 0 { len } else { len - 1 };
+
+    if len % 2 != 0 {
+        let sym = data[len - 1] as usize;
+        rans_enc_put(&mut s1, &mut rev_bytes, stats.cum_freqs[sym], stats.freqs[sym]);
+    }
+
+    let mut i = even_start;
+    while i >= 2 {
+        i -= 2;
+        let sym1 = data[i + 1] as usize;
+        rans_enc_put(&mut s1, &mut rev_bytes, stats.cum_freqs[sym1], stats.freqs[sym1]);
+        let sym0 = data[i] as usize;
+        rans_enc_put(&mut s0, &mut rev_bytes, stats.cum_freqs[sym0], stats.freqs[sym0]);
+    }
+
+    let mut output = Vec::with_capacity(9 + rev_bytes.len());
+    output.push(1);
+    write_state(&mut output, s0);
+    write_state(&mut output, s1);
+
+    for &b in rev_bytes.iter().rev() {
+        output.push(b);
+    }
+    output
+}
+
+fn rans_encode_single(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
     let mut state: u32 = RANS_BYTE_L;
     let mut rev_bytes: Vec<u8> = Vec::with_capacity(data.len() + 16);
 
@@ -146,11 +191,9 @@ pub fn rans_encode_block(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
         rans_enc_put(&mut state, &mut rev_bytes, stats.cum_freqs[s], stats.freqs[s]);
     }
 
-    let mut output = Vec::with_capacity(4 + rev_bytes.len());
-    output.push((state >> 24) as u8);
-    output.push(((state >> 16) & 0xFF) as u8);
-    output.push(((state >> 8) & 0xFF) as u8);
-    output.push((state & 0xFF) as u8);
+    let mut output = Vec::with_capacity(5 + rev_bytes.len());
+    output.push(0);
+    write_state(&mut output, state);
 
     for &b in rev_bytes.iter().rev() {
         output.push(b);
@@ -159,19 +202,30 @@ pub fn rans_encode_block(data: &[u8], stats: &SymbolStats) -> Vec<u8> {
 }
 
 pub fn rans_decode_block(encoded: &[u8], stats: &SymbolStats, output_len: usize) -> Result<Vec<u8>> {
-    if encoded.len() < 4 {
+    if encoded.is_empty() {
         return Err(anyhow::anyhow!("Data too short"));
     }
 
-    let mut cum2sym = vec![0u8; PROB_SCALE as usize];
+    let mut cum2sym = [0u8; PROB_SCALE as usize];
     for s in 0..256usize {
-        for slot in (stats.cum_freqs[s] as usize)..(stats.cum_freqs[s + 1] as usize) {
-            cum2sym[slot] = s as u8;
+        let start = stats.cum_freqs[s] as usize;
+        let end = stats.cum_freqs[s + 1] as usize;
+        if end > start {
+            cum2sym[start..end].fill(s as u8);
         }
     }
 
-    let mut pos = 0usize;
-    let mut state = rans_dec_init(encoded, &mut pos);
+    let mode = encoded[0];
+    let mut pos = 1usize;
+
+    if mode == 1 && output_len >= 8 {
+        return rans_decode_interleaved(encoded, &cum2sym, stats, output_len, &mut pos);
+    }
+
+    if pos + 4 > encoded.len() {
+        return Err(anyhow::anyhow!("Data too short"));
+    }
+    let mut state = read_state(encoded, &mut pos);
     let mut output = Vec::with_capacity(output_len);
 
     for _ in 0..output_len {
@@ -188,3 +242,45 @@ pub fn rans_decode_block(encoded: &[u8], stats: &SymbolStats, output_len: usize)
 
     Ok(output)
 }
+
+fn rans_decode_interleaved(
+    encoded: &[u8],
+    cum2sym: &[u8; PROB_SCALE as usize],
+    stats: &SymbolStats,
+    output_len: usize,
+    pos: &mut usize,
+) -> Result<Vec<u8>> {
+    if *pos + 8 > encoded.len() {
+        return Err(anyhow::anyhow!("Data too short for interleaved"));
+    }
+    let mut s0 = read_state(encoded, pos);
+    let mut s1 = read_state(encoded, pos);
+    let mut output = Vec::with_capacity(output_len);
+
+    let pairs = output_len / 2;
+    for _ in 0..pairs {
+        let slot0 = s0 & (PROB_SCALE - 1);
+        let sym0 = cum2sym[slot0 as usize];
+        output.push(sym0);
+        let freq0 = stats.freqs[sym0 as usize];
+        let start0 = stats.cum_freqs[sym0 as usize];
+        s0 = freq0 * (s0 >> PROB_BITS) + slot0 - start0;
+        rans_dec_renorm(&mut s0, encoded, pos);
+
+        let slot1 = s1 & (PROB_SCALE - 1);
+        let sym1 = cum2sym[slot1 as usize];
+        output.push(sym1);
+        let freq1 = stats.freqs[sym1 as usize];
+        let start1 = stats.cum_freqs[sym1 as usize];
+        s1 = freq1 * (s1 >> PROB_BITS) + slot1 - start1;
+        rans_dec_renorm(&mut s1, encoded, pos);
+    }
+
+    if output_len % 2 != 0 {
+        let slot = s1 & (PROB_SCALE - 1);
+        let sym = cum2sym[slot as usize];
+        output.push(sym);
+    }
+
+    Ok(output)
+}