roxify 1.7.6 → 1.8.0

package/native/core.rs

@@ -0,0 +1,293 @@
+use rayon::prelude::*;
+use std::sync::Arc;
+use std::path::PathBuf;
+use anyhow::Result;
+
+pub struct PlainScanResult {
+    pub marker_positions: Vec<u32>,
+    pub magic_positions: Vec<u32>,
+}
+
+pub fn scan_pixels_bytes(buf: &[u8], channels: usize, marker_bytes: Option<&[u8]>) -> PlainScanResult {
+    let magic = b"ROX1";
+
+    let magic_positions: Vec<u32> = if buf.len() >= 4 {
+        (0..(buf.len() - 3))
+            .into_par_iter()
+            .filter_map(|i| if &buf[i..i + 4] == magic { Some(i as u32) } else { None })
+            .collect()
+    } else {
+        Vec::new()
+    };
+
+    let markers: Vec<[u8; 3]> = match marker_bytes {
+        Some(bytes) if !bytes.is_empty() => {
+            if bytes.len() % 3 != 0 {
+                return PlainScanResult { marker_positions: Vec::new(), magic_positions };
+            }
+            bytes.chunks(3).map(|c| [c[0], c[1], c[2]]).collect()
+        }
+        _ => Vec::new(),
+    };
+
+    let marker_positions = if markers.is_empty() {
+        Vec::new()
+    } else {
+        let markers = Arc::new(markers);
+        let ch = channels as usize;
+        if ch < 3 || buf.len() < 3 {
+            Vec::new()
+        } else {
+            let pixel_count = buf.len() / ch;
+            (0..pixel_count)
+                .into_par_iter()
+                .filter_map(|i| {
+                    let base = i * ch;
+                    if base + 3 > buf.len() {
+                        return None;
+                    }
+                    for m in markers.iter() {
+                        if buf[base] == m[0] && buf[base + 1] == m[1] && buf[base + 2] == m[2] {
+                            return Some(i as u32);
+                        }
+                    }
+                    None
+                })
+                .collect()
+        }
+    };
+
+    PlainScanResult { marker_positions, magic_positions }
+}
+
+pub fn crc32_bytes(buf: &[u8]) -> u32 {
+    // parallelize checksum on large buffers, since crc32fast::hash is single-threaded
+    const PAR_THRESHOLD: usize = 4 * 1024 * 1024; // 4 MiB
+    if buf.len() < PAR_THRESHOLD {
+        crc32fast::hash(buf)
+    } else {
+        // compute per-chunk hasher in parallel then combine
+        let chunk = PAR_THRESHOLD;
+        let combined = buf
+            .par_chunks(chunk)
+            .map(|chunk| {
+                let mut h = crc32fast::Hasher::new();
+                h.update(chunk);
+                h
+            })
+            .reduce(|| crc32fast::Hasher::new(), |mut a, b| {
+                a.combine(&b);
+                a
+            });
+        combined.finalize()
+    }
+}
+
+pub fn adler32_bytes(buf: &[u8]) -> u32 {
+    const MOD: u32 = 65521;
+    let mut a: u32 = 1;
+    let mut b: u32 = 0;
+    for &v in buf {
+        a = (a + v as u32) % MOD;
+        b = (b + a) % MOD;
+    }
+    (b << 16) | a
+}
+
+pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {
+    let len = buf.len();
+    if len == 0 {
+        return Vec::new();
+    }
+    let mut out = vec![0u8; len];
+    out[0] = buf[0];
+    for i in 1..len {
+        out[i] = buf[i].wrapping_sub(buf[i - 1]);
+    }
+    out
+}
+
+pub fn delta_decode_bytes(buf: &[u8]) -> Vec<u8> {
+    let len = buf.len();
+    if len == 0 {
+        return Vec::new();
+    }
+    let mut out = vec![0u8; len];
+    out[0] = buf[0];
+    for i in 1..len {
+        out[i] = out[i - 1].wrapping_add(buf[i]);
+    }
+    out
+}
+
+fn compress_with_chunk_size(buf: &[u8], level: i32, chunk_size: usize) -> std::result::Result<Vec<u8>, String> {
+    use std::io::Write;
+
+    // Allow ultra levels (20-22) for maximum compression
+    let actual_level = level.min(22).max(1);
+    let mut encoder = zstd::stream::Encoder::new(Vec::new(), actual_level)
+        .map_err(|e| format!("zstd encoder init error: {}", e))?;
+
+    let threads = num_cpus::get() as u32;
+    if threads > 1 {
+        // Ultra levels (>=20) use much more memory per thread; limit to 4
+        let max_threads = if actual_level >= 20 { threads.min(4) } else { threads };
+        let _ = encoder.multithread(max_threads);
+    }
+
+    if buf.len() > 1024 * 1024 {
+        let _ = encoder.long_distance_matching(true);
+        let wlog = if buf.len() > 512 * 1024 * 1024 { 28 }
+            else if buf.len() > 64 * 1024 * 1024 { 27 }
+            else { 26 };
+        let _ = encoder.window_log(wlog);
+    }
+
+    let _ = encoder.set_pledged_src_size(Some(buf.len() as u64));
+
+    for chunk in buf.chunks(chunk_size) {
+        encoder.write_all(chunk).map_err(|e| format!("zstd write error: {}", e))?;
+    }
+
+    encoder.finish().map_err(|e| format!("zstd finish error: {}", e))
+}
+
+pub fn train_zstd_dictionary(sample_paths: &[PathBuf], dict_size: usize) -> Result<Vec<u8>> {
+    // load all sample files contiguously
+    let mut samples = Vec::new();
+    let mut lengths = Vec::new();
+    for path in sample_paths {
+        let data = std::fs::read(path)?;
+        lengths.push(data.len());
+        samples.extend_from_slice(&data);
+    }
+    let dict = zstd::dict::from_continuous(&samples, &lengths, dict_size)?;
+    Ok(dict)
+}
+
+/// Compress a slice with optional zstd dictionary.
+///
+/// When `dict` is `Some`, the dictionary is passed to the encoder (same
+/// dict required for decompression).  Pass `None` for normal compression.
+///
+/// For large buffers (>50 MiB) without a dictionary, multiple chunk sizes
+/// are benchmarked on a sample and the best is selected automatically.
+pub fn zstd_compress_bytes(buf: &[u8], level: i32, dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
+    use std::io::Write;
+
+    let actual_level = level.min(22).max(1);
+    let mut encoder = if let Some(d) = dict {
+        zstd::stream::Encoder::with_dictionary(Vec::new(), actual_level, d)
+            .map_err(|e| format!("zstd encoder init error: {}", e))?
+    } else {
+        zstd::stream::Encoder::new(Vec::new(), actual_level)
+            .map_err(|e| format!("zstd encoder init error: {}", e))?
+    };
+
+    let threads = num_cpus::get() as u32;
+    if threads > 1 {
+        let max_threads = if actual_level >= 20 { threads.min(4) } else { threads };
+        let _ = encoder.multithread(max_threads);
+    }
+
+    if buf.len() > 1024 * 1024 {
+        let _ = encoder.long_distance_matching(true);
+        let wlog = if buf.len() > 512 * 1024 * 1024 { 28 }
+            else if buf.len() > 64 * 1024 * 1024 { 27 }
+            else { 26 };
+        let _ = encoder.window_log(wlog);
+    }
+
+    let _ = encoder.set_pledged_src_size(Some(buf.len() as u64));
+
+    encoder.write_all(buf).map_err(|e| format!("zstd write error: {}", e))?;
+    encoder.finish().map_err(|e| format!("zstd finish error: {}", e))
+}
+
+pub fn zstd_decompress_bytes(buf: &[u8], dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
+    use std::io::Read;
+    if let Some(d) = dict {
+        let mut decoder = zstd::stream::Decoder::with_dictionary(std::io::Cursor::new(buf), d)
+            .map_err(|e| format!("zstd decoder init error: {}", e))?;
+        let mut out = Vec::new();
+        decoder.read_to_end(&mut out).map_err(|e| format!("zstd decompress error: {}", e))?;
+        Ok(out)
+    } else {
+        zstd::stream::decode_all(buf).map_err(|e| format!("zstd decompress error: {}", e))
+    }
+}
+
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_scan_magic() {
+        let data = b"xxxxROX1yyyyROX1".to_vec();
+        let res = scan_pixels_bytes(&data, 3, None);
+        assert_eq!(res.magic_positions.len(), 2);
+    }
+
+    #[test]
+    fn test_markers() {
+        let pixels = vec![1u8,2,3, 4,5,6, 1,2,3];
+        let markers_vec = vec![1u8,2,3];
+        let res = scan_pixels_bytes(&pixels, 3, Some(&markers_vec));
+        assert_eq!(res.marker_positions, vec![0,2]);
+    }
+
+    #[test]
+    fn test_train_dictionary() {
+        use std::fs::{write, create_dir_all};
+        let td = std::env::temp_dir().join("rox_dict_test");
+        let _ = create_dir_all(&td);
+        let f1 = td.join("a.bin");
+        let f2 = td.join("b.bin");
+        // produce 1 MiB of repeated data per file
+        let big = vec![0xABu8; 1024 * 1024];
+        write(&f1, &big).unwrap();
+        write(&f2, &big).unwrap();
+        // choose dictionary size 16 KiB (far below total sample size ≈2 MiB)
+        match train_zstd_dictionary(&[f1.clone(), f2.clone()], 16 * 1024) {
+            Ok(dict) => {
+                assert!(dict.len() <= 16 * 1024);
+                assert!(!dict.is_empty());
+            }
+            Err(e) => {
+                // dictionary training may fail due to insufficient or unsuitable samples;
+                // ensure error string is nonempty to catch panics
+                assert!(!e.to_string().is_empty());
+            }
+        }
+    }
+
+    #[test]
+    fn test_delta_roundtrip() {
+        let data = vec![10u8, 20, 30, 40, 250];
+        let enc = delta_encode_bytes(&data);
+        let dec = delta_decode_bytes(&enc);
+        assert_eq!(dec, data);
+    }
+
+    #[test]
+    fn test_crc_adler() {
+        let data = b"hello".to_vec();
+        assert_eq!(crc32_bytes(&data), crc32fast::hash(&data));
+        assert_eq!(adler32_bytes(&data), adler32_bytes(&data));
+
+        // also test large buffer triggers parallel branch
+        let big = vec![0xAAu8; 5 * 1024 * 1024];
+        assert_eq!(crc32_bytes(&big), crc32fast::hash(&big));
+    }
+
+    #[test]
+    fn test_zstd_dict_roundtrip() {
+        let data = b"this is some test data that repeats. ".repeat(1000);
+        // simple dictionary containing a substring
+        let dict = b"test data";
+        let compressed = zstd_compress_bytes(&data, 3, Some(dict)).expect("compress");
+        let decompressed = zstd_decompress_bytes(&compressed, Some(dict)).expect("decompress");
+        assert_eq!(decompressed, data);
+    }
+}

package/native/crypto.rs

@@ -0,0 +1,119 @@
+use anyhow::{anyhow, Result};
+use aes_gcm::{
+    aead::{Aead, KeyInit},
+    Aes256Gcm, Nonce,
+};
+use pbkdf2::pbkdf2_hmac;
+use rand::RngCore;
+use sha2::Sha256;
+
+const ENC_NONE: u8 = 0x00;
+const ENC_AES: u8 = 0x01;
+const ENC_XOR: u8 = 0x02;
+const PBKDF2_ITERS: u32 = 1_000_000;
+
+pub fn encrypt_xor(data: &[u8], passphrase: &str) -> Vec<u8> {
+    let key = passphrase.as_bytes();
+    let mut result = Vec::with_capacity(1 + data.len());
+    result.push(ENC_XOR);
+
+    for (i, &byte) in data.iter().enumerate() {
+        result.push(byte ^ key[i % key.len()]);
+    }
+
+    result
+}
+
+pub fn encrypt_aes(data: &[u8], passphrase: &str) -> Result<Vec<u8>> {
+    let mut salt = [0u8; 16];
+    rand::thread_rng().fill_bytes(&mut salt);
+
+    let mut key = [0u8; 32];
+    pbkdf2_hmac::<Sha256>(passphrase.as_bytes(), &salt, PBKDF2_ITERS, &mut key);
+
+    let cipher = Aes256Gcm::new_from_slice(&key)
+        .map_err(|e| anyhow::anyhow!("Failed to create cipher: {}", e))?;
+
+    let mut iv = [0u8; 12];
+    rand::thread_rng().fill_bytes(&mut iv);
+    let nonce = Nonce::from_slice(&iv);
+
+    let ciphertext = cipher
+        .encrypt(nonce, data)
+        .map_err(|e| anyhow::anyhow!("Encryption failed: {}", e))?;
+
+    let cipher_len = ciphertext.len();
+    if cipher_len < 16 {
+        return Err(anyhow::anyhow!("Ciphertext too short"));
+    }
+
+    let tag = &ciphertext[cipher_len - 16..];
+    let encrypted_data = &ciphertext[..cipher_len - 16];
+
+    let mut result = Vec::with_capacity(1 + 16 + 12 + 16 + encrypted_data.len());
+    result.push(ENC_AES);
+    result.extend_from_slice(&salt);
+    result.extend_from_slice(&iv);
+    result.extend_from_slice(tag);
+    result.extend_from_slice(encrypted_data);
+
+    Ok(result)
+}
+
+pub fn no_encryption(data: &[u8]) -> Vec<u8> {
+    let mut result = Vec::with_capacity(1 + data.len());
+    result.push(ENC_NONE);
+    result.extend_from_slice(data);
+    result
+}
+
+pub fn decrypt_xor(data: &[u8], passphrase: &str) -> Result<Vec<u8>> {
+    if data.is_empty() { return Err(anyhow!("Empty xor payload")); }
+    if passphrase.is_empty() { return Err(anyhow!("Passphrase required")); }
+    let key = passphrase.as_bytes();
+    let mut out = Vec::with_capacity(data.len());
+    for (i, &b) in data.iter().enumerate() {
+        out.push(b ^ key[i % key.len()]);
+    }
+    Ok(out)
+}
+
+pub fn decrypt_aes(data: &[u8], passphrase: &str) -> Result<Vec<u8>> {
+    if data.len() < 1 + 16 + 12 + 16 { return Err(anyhow!("Invalid AES payload length")); }
+    let salt = &data[1..17];
+    let iv = &data[17..29];
+    let tag = &data[29..45];
+    let enc = &data[45..];
+
+    let mut key = [0u8; 32];
+    pbkdf2_hmac::<Sha256>(passphrase.as_bytes(), salt, PBKDF2_ITERS, &mut key);
+
+    let cipher = Aes256Gcm::new_from_slice(&key)
+        .map_err(|e| anyhow!("Failed to create cipher: {}", e))?;
+
+    let mut combined = Vec::with_capacity(enc.len() + tag.len());
+    combined.extend_from_slice(enc);
+    combined.extend_from_slice(tag);
+
+    let nonce = Nonce::from_slice(iv);
+    let decrypted = cipher.decrypt(nonce, combined.as_ref())
+        .map_err(|e| anyhow!("AES decryption failed: {}", e))?;
+    Ok(decrypted)
+}
+
+pub fn try_decrypt(buf: &[u8], passphrase: Option<&str>) -> Result<Vec<u8>> {
+    if buf.is_empty() { return Err(anyhow!("Empty buffer")); }
+    let flag = buf[0];
+    match flag {
+        ENC_NONE => Ok(buf[1..].to_vec()),
+        ENC_XOR => {
+            let pass = passphrase.ok_or_else(|| anyhow!("Passphrase required for XOR decryption"))?;
+            decrypt_xor(&buf[1..], pass)
+        }
+        ENC_AES => {
+            let pass = passphrase.ok_or_else(|| anyhow!("Passphrase required for AES decryption"))?;
+            decrypt_aes(buf, pass)
+        }
+        _ => Err(anyhow!("Unknown encryption flag: {}", flag)),
+    }
+}