roxify 1.10.1 → 1.11.1

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.10.1"
+version = "1.11.1"
 edition = "2021"
 publish = false
 

package/native/archive.rs

@@ -4,7 +4,12 @@ use rayon::prelude::*;
 use tar::{Archive, Builder, Header};
 use walkdir::WalkDir;
 
-pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+pub struct TarPackResult {
+    pub data: Vec<u8>,
+    pub file_list: Vec<(String, u64)>,
+}
+
+pub fn tar_pack_directory_with_list(dir_path: &Path) -> Result<TarPackResult, String> {
     let base = dir_path;
 
     let entries: Vec<_> = WalkDir::new(dir_path)
@@ -27,6 +32,10 @@ pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
         })
         .collect();
 
+    let file_list: Vec<(String, u64)> = file_data.iter()
+        .map(|(name, data)| (name.clone(), data.len() as u64))
+        .collect();
+
     let total_estimate: usize = file_data.iter().map(|(n, d)| 512 + d.len() + 512 + n.len()).sum();
     let mut buf = Vec::with_capacity(total_estimate + 1024);
     {
@@ -42,20 +51,42 @@ pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
         }
         builder.finish().map_err(|e| format!("tar finish: {}", e))?;
     }
-    Ok(buf)
+    Ok(TarPackResult { data: buf, file_list })
+}
+
+pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+    tar_pack_directory_with_list(dir_path).map(|r| r.data)
+}
+
+pub fn tar_file_list_fast(tar_data: &[u8]) -> Vec<(String, u64)> {
+    let mut list = Vec::new();
+    let mut pos = 0;
+    while pos + 512 <= tar_data.len() {
+        let header = &tar_data[pos..pos + 512];
+        if header.iter().all(|&b| b == 0) {
+            break;
+        }
+        let name_end = header[..100].iter().position(|&b| b == 0).unwrap_or(100);
+        let name = String::from_utf8_lossy(&header[..name_end]).to_string();
+        let size_str = String::from_utf8_lossy(&header[124..136]);
+        let size = u64::from_str_radix(size_str.trim().trim_matches('\0'), 8).unwrap_or(0);
+        if !name.is_empty() {
+            list.push((name, size));
+        }
+        let data_blocks = (size as usize + 511) / 512;
+        pos += 512 + data_blocks * 512;
+    }
+    list
 }
 
 pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, String> {
     let mut archive = Archive::new(Cursor::new(tar_data));
-    let mut written = Vec::new();
+    let mut entries_data: Vec<(std::path::PathBuf, Vec<u8>)> = Vec::new();
 
     let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
     for entry in entries {
         let mut entry = entry.map_err(|e| format!("tar entry: {}", e))?;
-        let path = entry
-            .path()
-            .map_err(|e| format!("tar entry path: {}", e))?
-            .to_path_buf();
+        let path = entry.path().map_err(|e| format!("tar entry path: {}", e))?.to_path_buf();
 
         let mut safe = std::path::PathBuf::new();
         for comp in path.components() {
@@ -63,23 +94,36 @@ pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, Str
                 safe.push(osstr);
             }
         }
-
         if safe.as_os_str().is_empty() {
             continue;
         }
 
-        let dest = output_dir.join(&safe);
-        if let Some(parent) = dest.parent() {
-            std::fs::create_dir_all(parent)
-                .map_err(|e| format!("mkdir {:?}: {}", parent, e))?;
-        }
+        let mut data = Vec::with_capacity(entry.size() as usize);
+        std::io::Read::read_to_end(&mut entry, &mut data)
+            .map_err(|e| format!("tar read {:?}: {}", safe, e))?;
+        entries_data.push((safe, data));
+    }
 
-        entry
-            .unpack(&dest)
-            .map_err(|e| format!("tar unpack {:?}: {}", dest, e))?;
-        written.push(safe.to_string_lossy().to_string());
+    let dirs: std::collections::HashSet<_> = entries_data.iter()
+        .filter_map(|(p, _)| {
+            let dest = output_dir.join(p);
+            dest.parent().map(|d| d.to_path_buf())
+        })
+        .collect();
+    for dir in &dirs {
+        std::fs::create_dir_all(dir).map_err(|e| format!("mkdir {:?}: {}", dir, e))?;
     }
 
+    let written: Vec<String> = entries_data.par_iter()
+        .filter_map(|(safe, data)| {
+            let dest = output_dir.join(safe);
+            match std::fs::write(&dest, data) {
+                Ok(_) => Some(safe.to_string_lossy().to_string()),
+                Err(_) => None,
+            }
+        })
+        .collect();
+
     Ok(written)
 }
 

package/native/context_mixing.rs

@@ -1,5 +1,3 @@
-use rayon::prelude::*;
-
 #[derive(Clone, Copy, Debug)]
 pub struct ProbabilityEstimate {
     pub p0: u32,
@@ -91,26 +89,25 @@ impl ContextMixer {
 }
 
 pub fn analyze_entropy(data: &[u8]) -> f32 {
-    let freq: Vec<u32> = {
-        let mut f = vec![0u32; 256];
-        for &byte in data {
-            f[byte as usize] += 1;
-        }
-        f
-    };
+    let mut freq = [0u32; 256];
+    for &byte in data {
+        freq[byte as usize] += 1;
+    }
 
-    let total: u32 = freq.iter().sum();
-    if total == 0 {
+    let total = data.len() as f32;
+    if total == 0.0 {
         return 0.0;
     }
 
-    freq.par_iter()
-        .filter(|&&f| f > 0)
-        .map(|&f| {
-            let p = (f as f32) / (total as f32);
-            -p * p.log2()
-        })
-        .sum()
+    let inv_total = 1.0 / total;
+    let mut entropy = 0.0f32;
+    for &f in &freq {
+        if f > 0 {
+            let p = f as f32 * inv_total;
+            entropy -= p * p.log2();
+        }
+    }
+    entropy
 }
 
 pub fn estimate_compression_gain(original: &[u8], entropy_bits: f32) -> f64 {

package/native/core.rs

@@ -85,15 +85,55 @@ pub fn crc32_bytes(buf: &[u8]) -> u32 {
 
 pub fn adler32_bytes(buf: &[u8]) -> u32 {
     const MOD: u32 = 65521;
+    const NMAX: usize = 5552;
+
+    if buf.len() > 4 * 1024 * 1024 {
+        return adler32_parallel(buf);
+    }
+
     let mut a: u32 = 1;
     let mut b: u32 = 0;
-    for &v in buf {
-        a = (a + v as u32) % MOD;
-        b = (b + a) % MOD;
+
+    for chunk in buf.chunks(NMAX) {
+        for &v in chunk {
+            a += v as u32;
+            b += a;
+        }
+        a %= MOD;
+        b %= MOD;
     }
+
     (b << 16) | a
 }
 
+fn adler32_parallel(buf: &[u8]) -> u32 {
+    use rayon::prelude::*;
+    const MOD: u32 = 65521;
+    const CHUNK: usize = 1024 * 1024;
+
+    let chunks: Vec<&[u8]> = buf.chunks(CHUNK).collect();
+    let partials: Vec<(u32, u32, usize)> = chunks.par_iter().map(|chunk| {
+        let mut a: u32 = 0;
+        let mut b: u32 = 0;
+        for &v in *chunk {
+            a += v as u32;
+            b += a;
+        }
+        a %= MOD;
+        b %= MOD;
+        (a, b, chunk.len())
+    }).collect();
+
+    let mut a: u64 = 1;
+    let mut b: u64 = 0;
+    for (pa, pb, len) in partials {
+        b = (b + pb as u64 + a * len as u64) % MOD as u64;
+        a = (a + pa as u64) % MOD as u64;
+    }
+
+    ((b as u32) << 16) | (a as u32)
+}
+
 pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {
     let len = buf.len();
     if len == 0 {
@@ -173,40 +213,85 @@ pub fn train_zstd_dictionary(sample_paths: &[PathBuf], dict_size: usize) -> Resu
 /// For large buffers (>50 MiB) without a dictionary, multiple chunk sizes
 /// are benchmarked on a sample and the best is selected automatically.
 pub fn zstd_compress_bytes(buf: &[u8], level: i32, dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
+    zstd_compress_with_prefix(buf, level, dict, &[])
+}
+
+pub fn zstd_compress_with_prefix(buf: &[u8], level: i32, dict: Option<&[u8]>, prefix: &[u8]) -> std::result::Result<Vec<u8>, String> {
     use std::io::Write;
 
     let actual_level = level.min(22).max(1);
+    let total_len = prefix.len() + buf.len();
+
+    let adaptive_level = if total_len > 2 * 1024 * 1024 * 1024 {
+        actual_level.min(1)
+    } else if total_len > 1024 * 1024 * 1024 {
+        actual_level.min(3)
+    } else if total_len > 256 * 1024 * 1024 {
+        actual_level.min(6)
+    } else if total_len > 64 * 1024 * 1024 {
+        actual_level.min(12)
+    } else {
+        actual_level
+    };
+
+    if dict.is_none() && total_len < 4 * 1024 * 1024 {
+        if prefix.is_empty() {
+            return zstd::bulk::compress(buf, adaptive_level)
+                .map_err(|e| format!("zstd bulk compress error: {}", e));
+        }
+        let mut combined = Vec::with_capacity(total_len);
+        combined.extend_from_slice(prefix);
+        combined.extend_from_slice(buf);
+        return zstd::bulk::compress(&combined, adaptive_level)
+            .map_err(|e| format!("zstd bulk compress error: {}", e));
+    }
+
     let mut encoder = if let Some(d) = dict {
-        zstd::stream::Encoder::with_dictionary(Vec::new(), actual_level, d)
+        zstd::stream::Encoder::with_dictionary(Vec::with_capacity(total_len / 2), adaptive_level, d)
             .map_err(|e| format!("zstd encoder init error: {}", e))?
     } else {
-        zstd::stream::Encoder::new(Vec::new(), actual_level)
+        zstd::stream::Encoder::new(Vec::with_capacity(total_len / 2), adaptive_level)
             .map_err(|e| format!("zstd encoder init error: {}", e))?
     };
 
     let threads = num_cpus::get() as u32;
     if threads > 1 {
-        let max_threads = if actual_level >= 20 { threads.min(4) } else { threads };
+        let max_threads = if adaptive_level >= 20 { threads.min(4) } else { threads };
         let _ = encoder.multithread(max_threads);
     }
 
-    if buf.len() > 1024 * 1024 {
+    if total_len > 256 * 1024 && adaptive_level >= 3 {
         let _ = encoder.long_distance_matching(true);
-        let wlog = if buf.len() > 512 * 1024 * 1024 { 28 }
-            else if buf.len() > 64 * 1024 * 1024 { 27 }
+    }
+    if total_len > 256 * 1024 {
+        let wlog = if total_len > 1024 * 1024 * 1024 { 30 }
+            else if total_len > 512 * 1024 * 1024 { 29 }
+            else if total_len > 64 * 1024 * 1024 { 28 }
+            else if total_len > 8 * 1024 * 1024 { 27 }
             else { 26 };
         let _ = encoder.window_log(wlog);
     }
 
-    let _ = encoder.set_pledged_src_size(Some(buf.len() as u64));
+    let _ = encoder.set_pledged_src_size(Some(total_len as u64));
+
+    if !prefix.is_empty() {
+        encoder.write_all(prefix).map_err(|e| format!("zstd write prefix error: {}", e))?;
+    }
+
+    let chunk_size = if total_len > 256 * 1024 * 1024 { 16 * 1024 * 1024 }
+        else if total_len > 64 * 1024 * 1024 { 8 * 1024 * 1024 }
+        else { buf.len() };
+
+    for chunk in buf.chunks(chunk_size) {
+        encoder.write_all(chunk).map_err(|e| format!("zstd write error: {}", e))?;
+    }
 
-    encoder.write_all(buf).map_err(|e| format!("zstd write error: {}", e))?;
     encoder.finish().map_err(|e| format!("zstd finish error: {}", e))
 }
 
 pub fn zstd_decompress_bytes(buf: &[u8], dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
     use std::io::Read;
-    let mut out = Vec::new();
+    let mut out = Vec::with_capacity(buf.len() * 2);
     if let Some(d) = dict {
         let mut decoder = zstd::stream::Decoder::with_dictionary(std::io::Cursor::new(buf), d)
             .map_err(|e| format!("zstd decoder init error: {}", e))?;

package/native/crypto.rs

@@ -10,7 +10,7 @@ use sha2::Sha256;
 const ENC_NONE: u8 = 0x00;
 const ENC_AES: u8 = 0x01;
 const ENC_XOR: u8 = 0x02;
-const PBKDF2_ITERS: u32 = 1_000_000;
+const PBKDF2_ITERS: u32 = 600_000;
 
 pub fn encrypt_xor(data: &[u8], passphrase: &str) -> Vec<u8> {
     let key = passphrase.as_bytes();

package/native/encoder.rs

@@ -2,7 +2,6 @@ use anyhow::Result;
 use std::process::{Command, Stdio};
 
 const MAGIC: &[u8] = b"ROX1";
-const ENC_NONE: u8 = 0x00;
 const PIXEL_MAGIC: &[u8] = b"PXL1";
 const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
 
@@ -112,9 +111,7 @@ pub fn encode_to_wav_with_encryption_name_and_filelist(
     file_list: Option<&str>,
 ) -> Result<Vec<u8>> {
     // Same compression + encryption pipeline as PNG
-    let payload_input = [MAGIC, data].concat();
-
-    let compressed = crate::core::zstd_compress_bytes(&payload_input, compression_level, None)
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, None, MAGIC)
         .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
 
     let encrypted = if let Some(pass) = passphrase {
@@ -180,9 +177,7 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
     file_list: Option<&str>,
     dict: Option<&[u8]>,
 ) -> Result<Vec<u8>> {
-    let payload_input = [MAGIC, data].concat();
-
-    let compressed = crate::core::zstd_compress_bytes(&payload_input, compression_level, dict)
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, dict, MAGIC)
         .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
 
     let encrypted = if let Some(pass) = passphrase {
@@ -194,65 +189,73 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
     } else {
         crate::crypto::no_encryption(&compressed)
     };
+    drop(compressed);
 
     let meta_pixel = build_meta_pixel_with_name_and_filelist(&encrypted, name, file_list)?;
-    let data_without_markers = [PIXEL_MAGIC, &meta_pixel].concat();
-
-    let padding_needed = (3 - (data_without_markers.len() % 3)) % 3;
-    let padded_data = if padding_needed > 0 {
-        [&data_without_markers[..], &vec![0u8; padding_needed]].concat()
-    } else {
-        data_without_markers
-    };
+    drop(encrypted);
 
-    let mut marker_bytes = Vec::with_capacity(12);
-    for m in &MARKER_START {
-        marker_bytes.extend_from_slice(&[m.0, m.1, m.2]);
-    }
-    marker_bytes.extend_from_slice(&[MARKER_ZSTD.0, MARKER_ZSTD.1, MARKER_ZSTD.2]);
+    let raw_payload_len = PIXEL_MAGIC.len() + meta_pixel.len();
+    let padding_needed = (3 - (raw_payload_len % 3)) % 3;
+    let padded_len = raw_payload_len + padding_needed;
 
-    let data_with_markers = [&marker_bytes[..], &padded_data[..]].concat();
+    let marker_start_len = 12;
+    let marker_end_len = 9;
 
-    let mut marker_end_bytes = Vec::with_capacity(9);
-    for m in &MARKER_END {
-        marker_end_bytes.extend_from_slice(&[m.0, m.1, m.2]);
-    }
-
-    let data_pixels = (data_with_markers.len() + 2) / 3;
+    let data_with_markers_len = marker_start_len + padded_len;
+    let data_pixels = (data_with_markers_len + 2) / 3;
     let end_marker_pixels = 3;
     let total_pixels = data_pixels + end_marker_pixels;
 
     let side = (total_pixels as f64).sqrt().ceil() as usize;
     let side = side.max(end_marker_pixels);
-
     let width = side;
     let height = side;
 
     let total_data_bytes = width * height * 3;
-    let mut full_data = vec![0u8; total_data_bytes];
+    let marker_end_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
 
-    let marker_start_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
+    let flat = build_flat_pixel_buffer(&meta_pixel, padding_needed, marker_end_pos, marker_end_len, total_data_bytes);
+    drop(meta_pixel);
 
-    let copy_len = data_with_markers.len().min(marker_start_pos);
-    full_data[..copy_len].copy_from_slice(&data_with_markers[..copy_len]);
+    let row_bytes = width * 3;
+    let idat_data = create_raw_deflate_from_rows(&flat, row_bytes, height);
+    drop(flat);
 
-    let end_len = marker_end_bytes.len().min(total_data_bytes - marker_start_pos);
-    full_data[marker_start_pos..marker_start_pos + end_len]
-        .copy_from_slice(&marker_end_bytes[..end_len]);
-
-    let stride = width * 3 + 1;
-    let mut scanlines = Vec::with_capacity(height * stride);
+    build_png(width, height, &idat_data, file_list)
+}
 
-    for row in 0..height {
-        scanlines.push(0u8);
-        let src_start = row * width * 3;
-        let src_end = (row + 1) * width * 3;
-        scanlines.extend_from_slice(&full_data[src_start..src_end]);
+fn build_flat_pixel_buffer(
+    meta_pixel: &[u8],
+    padding_needed: usize,
+    marker_end_pos: usize,
+    marker_end_len: usize,
+    total_data_bytes: usize,
+) -> Vec<u8> {
+    let header_len = 12 + PIXEL_MAGIC.len() + meta_pixel.len() + padding_needed;
+    let mut flat = vec![0u8; total_data_bytes];
+
+    let mut pos = 0;
+    for m in &MARKER_START {
+        flat[pos] = m.0; flat[pos + 1] = m.1; flat[pos + 2] = m.2;
+        pos += 3;
     }
+    flat[pos] = MARKER_ZSTD.0; flat[pos + 1] = MARKER_ZSTD.1; flat[pos + 2] = MARKER_ZSTD.2;
+    pos += 3;
 
-    let idat_data = create_raw_deflate(&scanlines);
+    flat[pos..pos + PIXEL_MAGIC.len()].copy_from_slice(PIXEL_MAGIC);
+    pos += PIXEL_MAGIC.len();
 
-    build_png(width, height, &idat_data, file_list)
+    flat[pos..pos + meta_pixel.len()].copy_from_slice(meta_pixel);
+    pos += meta_pixel.len();
+
+    if marker_end_pos + marker_end_len <= total_data_bytes {
+        for (i, m) in MARKER_END.iter().enumerate() {
+            let off = marker_end_pos + i * 3;
+            flat[off] = m.0; flat[off + 1] = m.1; flat[off + 2] = m.2;
+        }
+    }
+
+    flat
 }
 
 fn build_meta_pixel(payload: &[u8]) -> Result<Vec<u8>> {
@@ -323,33 +326,37 @@ fn write_chunk(out: &mut Vec<u8>, chunk_type: &[u8; 4], data: &[u8]) -> Result<(
     out.extend_from_slice(chunk_type);
     out.extend_from_slice(data);
 
-    let mut crc_data = Vec::with_capacity(chunk_type.len() + data.len());
-    crc_data.extend_from_slice(chunk_type);
-    crc_data.extend_from_slice(data);
-    let crc = crate::core::crc32_bytes(&crc_data);
+    let mut hasher = crc32fast::Hasher::new();
+    hasher.update(chunk_type);
+    hasher.update(data);
+    let crc = hasher.finalize();
 
     out.extend_from_slice(&crc.to_be_bytes());
     Ok(())
 }
 
 fn create_raw_deflate(data: &[u8]) -> Vec<u8> {
-    let mut result = Vec::with_capacity(data.len() + 6 + (data.len() / 65535 + 1) * 5);
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (data.len() + MAX_BLOCK - 1) / MAX_BLOCK;
+    let total_size = 2 + num_blocks * 5 + data.len() + 4;
+    let mut result = Vec::with_capacity(total_size);
 
     result.push(0x78);
     result.push(0x01);
 
     let mut offset = 0;
     while offset < data.len() {
-        let chunk_size = (data.len() - offset).min(65535);
+        let chunk_size = (data.len() - offset).min(MAX_BLOCK);
         let is_last = offset + chunk_size >= data.len();
 
-        result.push(if is_last { 0x01 } else { 0x00 });
-
-        result.push(chunk_size as u8);
-        result.push((chunk_size >> 8) as u8);
-        result.push(!chunk_size as u8);
-        result.push((!(chunk_size >> 8)) as u8);
-
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        result.extend_from_slice(&header);
         result.extend_from_slice(&data[offset..offset + chunk_size]);
         offset += chunk_size;
     }
@@ -360,6 +367,51 @@ fn create_raw_deflate(data: &[u8]) -> Vec<u8> {
     result
 }
 
+fn create_raw_deflate_from_rows(flat: &[u8], row_bytes: usize, height: usize) -> Vec<u8> {
+    let stride = row_bytes + 1;
+    let scanlines_total = height * stride;
+
+    let mut scanlines = vec![0u8; scanlines_total];
+    for row in 0..height {
+        let flat_start = row * row_bytes;
+        let flat_end = (flat_start + row_bytes).min(flat.len());
+        let copy_len = flat_end.saturating_sub(flat_start);
+        if copy_len > 0 {
+            let dst_start = row * stride + 1;
+            scanlines[dst_start..dst_start + copy_len].copy_from_slice(&flat[flat_start..flat_end]);
+        }
+    }
+
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (scanlines_total + MAX_BLOCK - 1) / MAX_BLOCK;
+    let total_size = 2 + num_blocks * 5 + scanlines_total + 4;
+    let mut result = Vec::with_capacity(total_size);
+
+    result.push(0x78);
+    result.push(0x01);
+
+    let mut offset = 0;
+    while offset < scanlines.len() {
+        let chunk_size = (scanlines.len() - offset).min(MAX_BLOCK);
+        let is_last = offset + chunk_size >= scanlines.len();
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        result.extend_from_slice(&header);
+        result.extend_from_slice(&scanlines[offset..offset + chunk_size]);
+        offset += chunk_size;
+    }
+
+    let adler = crate::core::adler32_bytes(&scanlines);
+    result.extend_from_slice(&adler.to_be_bytes());
+
+    result
+}
+
 fn predict_best_format(data: &[u8]) -> ImageFormat {
     if data.len() < 2048 {
         return ImageFormat::Png;