roxify 1.7.6 → 1.8.1

package/Cargo.toml

@@ -0,0 +1,82 @@
+[package]
+name = "roxify_native"
+version = "1.8.1"
+edition = "2021"
+publish = false
+
+[lib]
+name = "roxify_native"
+crate-type = ["cdylib"]
+path = "native/lib.rs"
+
+[[bin]]
+name = "roxify_native"
+path = "native/main.rs"
+
+[dev-dependencies]
+
+[dependencies]
+napi = "2"
+napi-derive = "2"
+rayon = "1.7"
+zstd = { version = "0.11", features = ["zstdmt"] }
+crc32fast = "1.3"
+num_cpus = "1.16"
+clap = { version = "4", features = ["derive"] }
+serde_json = "1.0"
+anyhow = "1.0"
+png = "0.18.0"
+image = { version = "0.25", default-features = false, features = ["png"] }
+# indicatif removed from default deps to reduce heavy build graph
+walkdir = "2.5.0"
+tar = "0.4"
+aes-gcm = "0.10"
+pbkdf2 = "0.12"
+rand = "0.8"
+sha2 = "0.10"
+mimalloc = "0.1"
+
+wgpu = { version = "0.19", optional = true }
+memmap2 = "0.9"
+bytemuck = { version = "1.14", features = ["derive"] }
+# tokio is optional now; enable via the 'async' feature
+tokio = { version = "1", features = ["sync", "rt"], optional = true }
+parking_lot = "0.12"
+pollster = { version = "0.3", optional = true }
+
+[features]
+# default is intentionally empty so the crate compiles fast for local checks.
+# Enable 'gpu' to pull in the WGPU and pollster dependencies (heavy).
+# Enable 'async' to include tokio runtime (optional).
+# Example: `cargo build -p roxify_native --features gpu`
+default = []
+
+gpu = ["wgpu", "pollster"]
+async = ["tokio"]
+full = ["gpu", "async"]
+
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+strip = true
+panic = "abort"
+
+[profile.release-size]
+inherits = "release"
+opt-level = "z"
+lto = true
+strip = true
+
+[profile.fastdev]
+# Fast development profile for minimal user CPU and fast compilation.
+# Lower optimization and high codegen units to parallelize compilation work,
+# enable incremental to speed up subsequent incremental builds.
+inherits = "release"
+opt-level = 1
+lto = false
+codegen-units = 16
+debug = false
+incremental = true
+panic = "abort"
+

package/dist/cli.js

@@ -6,7 +6,7 @@ import { DataFormatError, decodePngToBinary, encodeBinaryToPng, hasPassphraseInP
 import { packPathsGenerator, unpackBuffer } from './pack.js';
 import * as cliProgress from './stub-progress.js';
 import { encodeWithRustCLI, isRustBinaryAvailable, } from './utils/rust-cli-wrapper.js';
-const VERSION = '1.6.1';
+const VERSION = '1.8.0';
 function getDirectorySize(dirPath) {
     let totalSize = 0;
     try {

package/native/archive.rs

@@ -0,0 +1,176 @@
+use std::io::Cursor;
+use std::path::Path;
+use rayon::prelude::*;
+use tar::{Archive, Builder, Header};
+use walkdir::WalkDir;
+
+pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+    let base = dir_path;
+
+    let entries: Vec<_> = WalkDir::new(dir_path)
+        .follow_links(false)
+        .into_iter()
+        .filter_map(|e| e.ok())
+        .filter(|e| e.file_type().is_file())
+        .collect();
+
+    let file_data: Vec<(String, Vec<u8>)> = entries
+        .par_iter()
+        .filter_map(|entry| {
+            let full = entry.path();
+            let rel = full.strip_prefix(base).unwrap_or(full);
+            let rel_str = rel.to_string_lossy().replace('\\', "/");
+            match std::fs::read(full) {
+                Ok(data) => Some((rel_str, data)),
+                Err(_) => None,
+            }
+        })
+        .collect();
+
+    let total_estimate: usize = file_data.iter().map(|(n, d)| 512 + d.len() + 512 + n.len()).sum();
+    let mut buf = Vec::with_capacity(total_estimate + 1024);
+    {
+        let mut builder = Builder::new(&mut buf);
+        for (rel_str, data) in &file_data {
+            let mut header = Header::new_gnu();
+            header.set_size(data.len() as u64);
+            header.set_mode(0o644);
+            header.set_cksum();
+            builder
+                .append_data(&mut header, rel_str, &data[..])
+                .map_err(|e| format!("tar append {}: {}", rel_str, e))?;
+        }
+        builder.finish().map_err(|e| format!("tar finish: {}", e))?;
+    }
+    Ok(buf)
+}
+
+pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, String> {
+    let mut archive = Archive::new(Cursor::new(tar_data));
+    let mut written = Vec::new();
+
+    let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
+    for entry in entries {
+        let mut entry = entry.map_err(|e| format!("tar entry: {}", e))?;
+        let path = entry
+            .path()
+            .map_err(|e| format!("tar entry path: {}", e))?
+            .to_path_buf();
+
+        let mut safe = std::path::PathBuf::new();
+        for comp in path.components() {
+            if let std::path::Component::Normal(osstr) = comp {
+                safe.push(osstr);
+            }
+        }
+
+        if safe.as_os_str().is_empty() {
+            continue;
+        }
+
+        let dest = output_dir.join(&safe);
+        if let Some(parent) = dest.parent() {
+            std::fs::create_dir_all(parent)
+                .map_err(|e| format!("mkdir {:?}: {}", parent, e))?;
+        }
+
+        entry
+            .unpack(&dest)
+            .map_err(|e| format!("tar unpack {:?}: {}", dest, e))?;
+        written.push(safe.to_string_lossy().to_string());
+    }
+
+    Ok(written)
+}
+
+pub fn is_tar(data: &[u8]) -> bool {
+    if data.len() < 263 {
+        return false;
+    }
+    &data[257..262] == b"ustar"
+}
+
+pub fn tar_file_list(tar_data: &[u8]) -> Result<Vec<(String, u64)>, String> {
+    let mut archive = Archive::new(Cursor::new(tar_data));
+    let mut list = Vec::new();
+    let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
+    for entry in entries {
+        let entry = entry.map_err(|e| format!("tar entry: {}", e))?;
+        let path = entry
+            .path()
+            .map_err(|e| format!("tar path: {}", e))?
+            .to_string_lossy()
+            .to_string();
+        let size = entry.size();
+        list.push((path, size));
+    }
+    Ok(list)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+
+    #[test]
+    fn test_tar_roundtrip() {
+        let tmp = std::env::temp_dir().join("rox_tar_test");
+        let _ = fs::remove_dir_all(&tmp);
+        fs::create_dir_all(tmp.join("sub")).unwrap();
+        fs::write(tmp.join("hello.txt"), b"Hello TAR").unwrap();
+        fs::write(tmp.join("sub/nested.txt"), b"Nested!").unwrap();
+
+        let tar_data = tar_pack_directory(&tmp).unwrap();
+        assert!(is_tar(&tar_data));
+
+        let list = tar_file_list(&tar_data).unwrap();
+        assert_eq!(list.len(), 2);
+
+        let out = std::env::temp_dir().join("rox_tar_test_out");
+        let _ = fs::remove_dir_all(&out);
+        fs::create_dir_all(&out).unwrap();
+
+        let written = tar_unpack(&tar_data, &out).unwrap();
+        assert_eq!(written.len(), 2);
+        assert_eq!(fs::read_to_string(out.join("hello.txt")).unwrap(), "Hello TAR");
+        assert_eq!(fs::read_to_string(out.join("sub/nested.txt")).unwrap(), "Nested!");
+
+        let _ = fs::remove_dir_all(&tmp);
+        let _ = fs::remove_dir_all(&out);
+    }
+
+    #[test]
+    fn test_tar_zstd_roundtrip() {
+        use std::io::Write;
+
+        let tmp = std::env::temp_dir().join("rox_tar_zstd_test");
+        let _ = fs::remove_dir_all(&tmp);
+        fs::create_dir_all(tmp.join("a/b")).unwrap();
+        fs::write(tmp.join("root.txt"), b"root file content").unwrap();
+        fs::write(tmp.join("a/mid.txt"), b"mid level").unwrap();
+        fs::write(tmp.join("a/b/deep.txt"), b"deep nested file").unwrap();
+
+        let tar_data = tar_pack_directory(&tmp).unwrap();
+        assert!(is_tar(&tar_data));
+
+        let mut encoder = zstd::stream::Encoder::new(Vec::new(), 3).unwrap();
+        encoder.write_all(&tar_data).unwrap();
+        let compressed = encoder.finish().unwrap();
+
+        let decompressed = crate::core::zstd_decompress_bytes(&compressed, None).unwrap();
+        assert!(is_tar(&decompressed));
+
+        let out = std::env::temp_dir().join("rox_tar_zstd_test_out");
+        let _ = fs::remove_dir_all(&out);
+        fs::create_dir_all(&out).unwrap();
+
+        let written = tar_unpack(&decompressed, &out).unwrap();
+        assert_eq!(written.len(), 3);
+        assert_eq!(fs::read_to_string(out.join("root.txt")).unwrap(), "root file content");
+        assert_eq!(fs::read_to_string(out.join("a/mid.txt")).unwrap(), "mid level");
+        assert_eq!(fs::read_to_string(out.join("a/b/deep.txt")).unwrap(), "deep nested file");
+
+        let _ = fs::remove_dir_all(&tmp);
+        let _ = fs::remove_dir_all(&out);
+    }
+}

package/native/audio.rs

@@ -0,0 +1,151 @@
+/// WAV container for binary data.
+///
+/// Encodes raw bytes as 8-bit unsigned PCM mono samples (44100 Hz).
+/// Header is exactly 44 bytes. Total overhead: 44 bytes.
+///
+/// Compared to PNG (stored deflate): PNG overhead grows with data size
+/// (zlib framing, filter bytes, chunk CRCs). WAV overhead is constant.
+
+const WAV_HEADER_SIZE: usize = 44;
+const SAMPLE_RATE: u32 = 44100;
+const BITS_PER_SAMPLE: u16 = 8;
+const NUM_CHANNELS: u16 = 1;
+
+/// Pack raw bytes into a WAV file (8-bit PCM, mono, 44100 Hz).
+/// The bytes are stored directly as unsigned PCM samples.
+/// Returns the complete WAV file as a Vec<u8>.
+pub fn bytes_to_wav(data: &[u8]) -> Vec<u8> {
+    let data_size = data.len() as u32;
+    let file_size = WAV_HEADER_SIZE as u32 - 8 + data_size; // RIFF chunk size
+
+    let byte_rate = SAMPLE_RATE * NUM_CHANNELS as u32 * (BITS_PER_SAMPLE as u32 / 8);
+    let block_align = NUM_CHANNELS * (BITS_PER_SAMPLE / 8);
+
+    let mut wav = Vec::with_capacity(WAV_HEADER_SIZE + data.len());
+
+    // RIFF header
+    wav.extend_from_slice(b"RIFF");
+    wav.extend_from_slice(&file_size.to_le_bytes());
+    wav.extend_from_slice(b"WAVE");
+
+    // fmt sub-chunk
+    wav.extend_from_slice(b"fmt ");
+    wav.extend_from_slice(&16u32.to_le_bytes()); // sub-chunk size (PCM = 16)
+    wav.extend_from_slice(&1u16.to_le_bytes());  // audio format (1 = PCM)
+    wav.extend_from_slice(&NUM_CHANNELS.to_le_bytes());
+    wav.extend_from_slice(&SAMPLE_RATE.to_le_bytes());
+    wav.extend_from_slice(&byte_rate.to_le_bytes());
+    wav.extend_from_slice(&block_align.to_le_bytes());
+    wav.extend_from_slice(&BITS_PER_SAMPLE.to_le_bytes());
+
+    // data sub-chunk
+    wav.extend_from_slice(b"data");
+    wav.extend_from_slice(&data_size.to_le_bytes());
+    wav.extend_from_slice(data);
+
+    wav
+}
+
+/// Extract raw bytes from a WAV file.
+/// Returns the PCM data (the original bytes) or an error.
+pub fn wav_to_bytes(wav: &[u8]) -> Result<Vec<u8>, String> {
+    if wav.len() < WAV_HEADER_SIZE {
+        return Err("WAV data too short".to_string());
+    }
+
+    // Validate RIFF header
+    if &wav[0..4] != b"RIFF" {
+        return Err("Not a RIFF file".to_string());
+    }
+    if &wav[8..12] != b"WAVE" {
+        return Err("Not a WAVE file".to_string());
+    }
+
+    // Find the "data" sub-chunk (skip fmt and any other chunks)
+    let mut offset = 12; // past "RIFF" + size + "WAVE"
+    loop {
+        if offset + 8 > wav.len() {
+            return Err("data chunk not found".to_string());
+        }
+        let chunk_id = &wav[offset..offset + 4];
+        let chunk_size = u32::from_le_bytes([
+            wav[offset + 4],
+            wav[offset + 5],
+            wav[offset + 6],
+            wav[offset + 7],
+        ]) as usize;
+
+        if chunk_id == b"data" {
+            let data_start = offset + 8;
+            let data_end = data_start + chunk_size;
+            if data_end > wav.len() {
+                // Allow truncation: return what we have
+                return Ok(wav[data_start..].to_vec());
+            }
+            return Ok(wav[data_start..data_end].to_vec());
+        }
+
+        // Skip this chunk (+ padding byte if odd size)
+        offset += 8 + chunk_size;
+        if chunk_size % 2 != 0 {
+            offset += 1; // RIFF chunks are word-aligned
+        }
+    }
+}
+
+/// Check if a buffer starts with a RIFF/WAVE header.
+pub fn is_wav(buf: &[u8]) -> bool {
+    buf.len() >= 12 && &buf[0..4] == b"RIFF" && &buf[8..12] == b"WAVE"
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_wav_roundtrip() {
+        let data = b"Hello, World! This is roxify audio container test data.";
+        let wav = bytes_to_wav(data);
+
+        // Check header
+        assert_eq!(&wav[0..4], b"RIFF");
+        assert_eq!(&wav[8..12], b"WAVE");
+        assert_eq!(wav.len(), 44 + data.len());
+
+        // Roundtrip
+        let recovered = wav_to_bytes(&wav).expect("decode should succeed");
+        assert_eq!(recovered, data);
+    }
+
+    #[test]
+    fn test_wav_empty() {
+        let data: &[u8] = b"";
+        let wav = bytes_to_wav(data);
+        assert_eq!(wav.len(), 44);
+        let recovered = wav_to_bytes(&wav).expect("decode empty");
+        assert!(recovered.is_empty());
+    }
+
+    #[test]
+    fn test_wav_large() {
+        let data = vec![0xAB_u8; 1024 * 1024]; // 1 MB
+        let wav = bytes_to_wav(&data);
+        assert_eq!(wav.len(), 44 + 1024 * 1024);
+        let recovered = wav_to_bytes(&wav).expect("decode large");
+        assert_eq!(recovered, data);
+    }
+
+    #[test]
+    fn test_is_wav() {
+        let wav = bytes_to_wav(b"test");
+        assert!(is_wav(&wav));
+        assert!(!is_wav(b"not a wav"));
+        assert!(!is_wav(b"RIFF1234XXXX")); // RIFF but not WAVE
+    }
+
+    #[test]
+    fn test_invalid_wav() {
+        assert!(wav_to_bytes(b"short").is_err());
+        assert!(wav_to_bytes(b"NOT a RIFF file!").is_err());
+    }
+}

package/native/bwt.rs

@@ -0,0 +1,100 @@
+use anyhow::Result;
+use rayon::prelude::*;
+
+pub struct BwtResult {
+    pub transformed: Vec<u8>,
+    pub primary_index: u32,
+}
+
+pub fn bwt_encode(data: &[u8]) -> Result<BwtResult> {
+    if data.is_empty() {
+        return Ok(BwtResult {
+            transformed: Vec::new(),
+            primary_index: 0,
+        });
+    }
+
+    let n = data.len();
+    let mut rotations: Vec<usize> = (0..n).collect();
+
+    rotations.par_sort_by(|&a, &b| {
+        for i in 0..n {
+            let ca = data[(a + i) % n];
+            let cb = data[(b + i) % n];
+            if ca != cb {
+                return ca.cmp(&cb);
+            }
+        }
+        std::cmp::Ordering::Equal
+    });
+
+    let mut transformed = Vec::with_capacity(n);
+    let mut primary_index = 0u32;
+
+    for (idx, &rot) in rotations.iter().enumerate() {
+        if rot == 0 {
+            primary_index = idx as u32;
+        }
+        transformed.push(data[(rot + n - 1) % n]);
+    }
+
+    Ok(BwtResult {
+        transformed,
+        primary_index,
+    })
+}
+
+pub fn bwt_decode(data: &[u8], primary_index: u32) -> Result<Vec<u8>> {
+    if data.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let n = data.len();
+    let primary_idx = primary_index as usize;
+
+    if primary_idx >= n {
+        return Err(anyhow::anyhow!("Invalid primary index"));
+    }
+
+    let mut counts = vec![0usize; 256];
+    for &byte in data {
+        counts[byte as usize] += 1;
+    }
+
+    let mut cumsum = vec![0usize; 256];
+    let mut sum = 0;
+    for i in 0..256 {
+        cumsum[i] = sum;
+        sum += counts[i];
+    }
+
+    let mut next = vec![0usize; n];
+    let mut counts = vec![0usize; 256];
+
+    for i in 0..n {
+        let byte = data[i] as usize;
+        let pos = cumsum[byte] + counts[byte];
+        next[pos] = i;
+        counts[byte] += 1;
+    }
+
+    let mut result = Vec::with_capacity(n);
+    let mut idx = primary_idx;
+
+    for _ in 0..n {
+        result.push(data[idx]);
+        idx = next[idx];
+    }
+
+    Ok(result)
+}
+
+pub fn bwt_encode_streaming(block_size: usize, data: &[u8]) -> Result<Vec<(BwtResult, usize)>> {
+    data.par_chunks(block_size)
+        .enumerate()
+        .map(|(i, chunk)| {
+            let result = bwt_encode(chunk)?;
+            Ok((result, i * block_size))
+        })
+        .collect()
+}

package/native/context_mixing.rs

@@ -0,0 +1,120 @@
+use rayon::prelude::*;
+
+#[derive(Clone, Copy, Debug)]
+pub struct ProbabilityEstimate {
+    pub p0: u32,
+    pub p1: u32,
+    pub total: u32,
+}
+
+impl ProbabilityEstimate {
+    pub fn entropy_bits(&self) -> f32 {
+        if self.total == 0 {
+            return 0.0;
+        }
+        let p0 = (self.p0 as f32) / (self.total as f32);
+        let p1 = (self.p1 as f32) / (self.total as f32);
+
+        let mut bits = 0.0;
+        if p0 > 0.0 {
+            bits -= p0 * p0.log2();
+        }
+        if p1 > 0.0 {
+            bits -= p1 * p1.log2();
+        }
+        bits
+    }
+}
+
+pub struct ContextMixer {
+    contexts_order0: Vec<ProbabilityEstimate>,
+    contexts_order1: Vec<[ProbabilityEstimate; 256]>,
+    contexts_order2: Vec<[[ProbabilityEstimate; 256]; 256]>,
+}
+
+impl ContextMixer {
+    pub fn new() -> Self {
+        ContextMixer {
+            contexts_order0: vec![ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 1],
+            contexts_order1: vec![
+                [ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 256];
+                256
+            ],
+            contexts_order2: vec![
+                [[ProbabilityEstimate { p0: 1, p1: 1, total: 2 }; 256]; 256];
+                256
+            ],
+        }
+    }
+
+    pub fn predict_order0(&self) -> ProbabilityEstimate {
+        self.contexts_order0[0]
+    }
+
+    pub fn predict_order1(&self, context1: u8) -> ProbabilityEstimate {
+        self.contexts_order1[context1 as usize][0]
+    }
+
+    pub fn predict_order2(&self, context1: u8, context2: u8) -> ProbabilityEstimate {
+        self.contexts_order2[context1 as usize][context2 as usize][0]
+    }
+
+    pub fn update_order0(&mut self, bit: bool) {
+        let ctx = &mut self.contexts_order0[0];
+        if bit {
+            ctx.p1 += 1;
+        } else {
+            ctx.p0 += 1;
+        }
+        ctx.total += 1;
+    }
+
+    pub fn update_order1(&mut self, context1: u8, bit: bool) {
+        let ctx = &mut self.contexts_order1[context1 as usize][0];
+        if bit {
+            ctx.p1 += 1;
+        } else {
+            ctx.p0 += 1;
+        }
+        ctx.total += 1;
+    }
+
+    pub fn update_order2(&mut self, context1: u8, context2: u8, bit: bool) {
+        let ctx = &mut self.contexts_order2[context1 as usize][context2 as usize][0];
+        if bit {
+            ctx.p1 += 1;
+        } else {
+            ctx.p0 += 1;
+        }
+        ctx.total += 1;
+    }
+}
+
+pub fn analyze_entropy(data: &[u8]) -> f32 {
+    let freq: Vec<u32> = {
+        let mut f = vec![0u32; 256];
+        for &byte in data {
+            f[byte as usize] += 1;
+        }
+        f
+    };
+
+    let total: u32 = freq.iter().sum();
+    if total == 0 {
+        return 0.0;
+    }
+
+    freq.par_iter()
+        .filter(|&&f| f > 0)
+        .map(|&f| {
+            let p = (f as f32) / (total as f32);
+            -p * p.log2()
+        })
+        .sum()
+}
+
+pub fn estimate_compression_gain(original: &[u8], entropy_bits: f32) -> f64 {
+    let theoretical_min = (original.len() as f64) * (entropy_bits as f64) / 8.0;
+    let ratio = theoretical_min / (original.len() as f64);
+    (1.0 - ratio) * 100.0
+}