roxify 1.13.2 → 1.13.3

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.13.2"
+version = "1.13.3"
 edition = "2021"
 publish = false
 
@@ -51,6 +51,7 @@ pbkdf2 = "0.12"
 rand = "0.8"
 sha2 = "0.10"
 mimalloc = "0.1"
+simd-adler32 = "0.3"
 
 memmap2 = "0.9"
 bytemuck = { version = "1.14", features = ["derive"] }

package/README.md

@@ -70,8 +70,8 @@ All measurements taken on Linux x64 (Intel i7-6700K @ 4.0 GHz, 32 GB RAM). Roxif
 
 | Dataset | unzip | Roxify decode | Speedup |
 | --- | --- | --- | --- |
-| Test A (177 MB) | 2.4 s | 1.8 s | 1.3x |
-| Test B (1.4 GB) | 8.4 s | 2.2 s | 3.8x |
+| Test A (177 MB) | 2.4 s | 1.3 s | 1.8x |
+| Test B (1.4 GB) | 8.4 s | 2.9 s | 2.9x |
 
 Roxify produces a valid PNG image instead of a ZIP archive. On these real-world datasets it compresses 20-37% smaller than ZIP -9 while encoding 15x faster, thanks to multi-threaded Zstd with long-distance matching.
 
@@ -537,6 +537,8 @@ Input --> Detect Format --> Demodulate/Read Blocks --> De-interleave --> RS ECC
 | `image_utils.rs`    | Image resizing, pixel format conversion, metadata extraction           |
 | `png_utils.rs`      | Low-level PNG chunk read/write operations                              |
 | `progress.rs`       | Progress tracking for long-running compression/decompression           |
+| `streaming_encode.rs` | Streaming directory-to-PNG encoder with real-time progress           |
+| `streaming_decode.rs` | Streaming PNG-to-directory decoder with real-time progress           |
 
 ### TypeScript Modules
 

package/dist/cli.js

@@ -317,20 +317,17 @@ async function encodeCommand(args) {
             console.log(`Encoding to ${resolvedOutput} (Using native Rust encoder)\n`);
             const startTime = Date.now();
             const encodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
-            let barValue = 0;
             encodeBar.start(100, 0, { step: 'Encoding', elapsed: '0' });
-            const progressInterval = setInterval(() => {
-                barValue = Math.min(barValue + 1, 99);
+            const encryptType = parsed.encrypt === 'xor' ? 'xor' : 'aes';
+            const fileName = basename(inputPaths[0]);
+            await encodeWithRustCLI(inputPaths.length === 1 ? resolvedInputs[0] : resolvedInputs[0], resolvedOutput, 19, parsed.passphrase, encryptType, fileName, (current, total, step) => {
+                const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
                 const elapsed = Math.floor((Date.now() - startTime) / 1000);
-                encodeBar.update(barValue, {
-                    step: 'Encoding',
+                encodeBar.update(Math.min(pct, 99), {
+                    step: step || 'Encoding',
                     elapsed: String(elapsed),
                 });
-            }, 500);
-            const encryptType = parsed.encrypt === 'xor' ? 'xor' : 'aes';
-            const fileName = basename(inputPaths[0]);
-            await encodeWithRustCLI(inputPaths.length === 1 ? resolvedInputs[0] : resolvedInputs[0], resolvedOutput, 19, parsed.passphrase, encryptType, fileName);
-            clearInterval(progressInterval);
+            });
             const encodeTime = Date.now() - startTime;
             encodeBar.update(100, {
                 step: 'done',
@@ -597,17 +594,15 @@ async function decodeCommand(args) {
             console.log('Decoding... (Using native Rust decoder)\n');
             const startTime = Date.now();
             const decodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
-            let barValue = 0;
             decodeBar.start(100, 0, { step: 'Decoding', elapsed: '0' });
-            const progressInterval = setInterval(() => {
-                barValue = Math.min(barValue + 2, 99);
-                decodeBar.update(barValue, {
-                    step: 'Decoding',
-                    elapsed: String(Math.floor((Date.now() - startTime) / 1000)),
+            await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict, (current, total, step) => {
+                const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
+                const elapsed = Math.floor((Date.now() - startTime) / 1000);
+                decodeBar.update(Math.min(pct, 99), {
+                    step: step || 'Decoding',
+                    elapsed: String(elapsed),
                 });
-            }, 300);
-            await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict);
-            clearInterval(progressInterval);
+            });
             const decodeTime = Date.now() - startTime;
             decodeBar.update(100, { step: 'done', elapsed: String(Math.floor(decodeTime / 1000)) });
             decodeBar.stop();

package/dist/utils/rust-cli-wrapper.d.ts

@@ -1,7 +1,8 @@
 declare function findRustBinary(): string | null;
 export { findRustBinary };
 export declare function isRustBinaryAvailable(): boolean;
-export declare function encodeWithRustCLI(inputPath: string, outputPath: string, compressionLevel?: number, passphrase?: string, encryptType?: 'aes' | 'xor', name?: string): Promise<void>;
-export declare function decodeWithRustCLI(inputPath: string, outputPath: string, passphrase?: string, files?: string[], dict?: string): Promise<void>;
+export type ProgressCallback = (current: number, total: number, step: string) => void;
+export declare function encodeWithRustCLI(inputPath: string, outputPath: string, compressionLevel?: number, passphrase?: string, encryptType?: 'aes' | 'xor', name?: string, onProgress?: ProgressCallback): Promise<void>;
+export declare function decodeWithRustCLI(inputPath: string, outputPath: string, passphrase?: string, files?: string[], dict?: string, onProgress?: ProgressCallback): Promise<void>;
 export declare function listWithRustCLI(inputPath: string): Promise<string>;
 export declare function havepassphraseWithRustCLI(inputPath: string): Promise<string>;

package/dist/utils/rust-cli-wrapper.js

@@ -162,9 +162,12 @@ function spawnRustCLI(args, options) {
         let stdout = '';
         const runSpawn = (exePath) => {
             let proc;
+            const hasProgress = !!options?.onProgress;
             const stdio = options?.collectStdout
-                ? ['pipe', 'pipe', 'inherit']
-                : 'inherit';
+                ? ['pipe', 'pipe', hasProgress ? 'pipe' : 'inherit']
+                : hasProgress
+                    ? ['pipe', 'inherit', 'pipe']
+                    : 'inherit';
             try {
                 proc = spawn(exePath, args, { stdio });
             }
@@ -184,6 +187,23 @@ function spawnRustCLI(args, options) {
             if (options?.collectStdout && proc.stdout) {
                 proc.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
             }
+            if (hasProgress && proc.stderr) {
+                let stderrBuf = '';
+                proc.stderr.on('data', (chunk) => {
+                    stderrBuf += chunk.toString();
+                    const lines = stderrBuf.split('\n');
+                    stderrBuf = lines.pop() || '';
+                    for (const line of lines) {
+                        const match = line.match(/^PROGRESS:(\d+):(\d+):(.+)$/);
+                        if (match) {
+                            options.onProgress(Number(match[1]), Number(match[2]), match[3]);
+                        }
+                        else if (line.trim()) {
+                            process.stderr.write(line + '\n');
+                        }
+                    }
+                });
+            }
             proc.on('error', (err) => {
                 if (!triedExtract) {
                     triedExtract = true;
@@ -213,7 +233,7 @@ function spawnRustCLI(args, options) {
         runSpawn(cliPath);
     });
 }
-export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name) {
+export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name, onProgress) {
     const cliPath = findRustBinary();
     if (!cliPath)
         throw new Error('Rust CLI binary not found');
@@ -231,9 +251,9 @@ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel
         args.push('--encrypt', encryptType);
     }
     args.push(inputPath, outputPath);
-    await spawnRustCLI(args);
+    await spawnRustCLI(args, { onProgress });
 }
-export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict) {
+export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict, onProgress) {
     const args = ['decompress', inputPath, outputPath];
     if (passphrase)
         args.push('--passphrase', passphrase);
@@ -241,7 +261,7 @@ export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files
         args.push('--files', JSON.stringify(files));
     if (dict)
         args.push('--dict', dict);
-    await spawnRustCLI(args);
+    await spawnRustCLI(args, { onProgress });
 }
 export async function listWithRustCLI(inputPath) {
     return spawnRustCLI(['list', inputPath], { collectStdout: true });

package/native/core.rs

@@ -84,54 +84,9 @@ pub fn crc32_bytes(buf: &[u8]) -> u32 {
 }
 
 pub fn adler32_bytes(buf: &[u8]) -> u32 {
-    const MOD: u32 = 65521;
-    const NMAX: usize = 5552;
-
-    if buf.len() > 4 * 1024 * 1024 {
-        return adler32_parallel(buf);
-    }
-
-    let mut a: u32 = 1;
-    let mut b: u32 = 0;
-
-    for chunk in buf.chunks(NMAX) {
-        for &v in chunk {
-            a += v as u32;
-            b += a;
-        }
-        a %= MOD;
-        b %= MOD;
-    }
-
-    (b << 16) | a
-}
-
-fn adler32_parallel(buf: &[u8]) -> u32 {
-    use rayon::prelude::*;
-    const MOD: u32 = 65521;
-    const CHUNK: usize = 1024 * 1024;
-
-    let chunks: Vec<&[u8]> = buf.chunks(CHUNK).collect();
-    let partials: Vec<(u32, u32, usize)> = chunks.par_iter().map(|chunk| {
-        let mut a: u32 = 0;
-        let mut b: u32 = 0;
-        for &v in *chunk {
-            a += v as u32;
-            b += a;
-        }
-        a %= MOD;
-        b %= MOD;
-        (a, b, chunk.len())
-    }).collect();
-
-    let mut a: u64 = 1;
-    let mut b: u64 = 0;
-    for (pa, pb, len) in partials {
-        b = (b + pb as u64 + a * len as u64) % MOD as u64;
-        a = (a + pa as u64) % MOD as u64;
-    }
-
-    ((b as u32) << 16) | (a as u32)
+    let mut hasher = simd_adler32::Adler32::new();
+    hasher.write(buf);
+    hasher.finish()
 }
 
 pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {

package/native/crypto.rs

@@ -75,6 +75,11 @@ pub fn no_encryption(data: &[u8]) -> Vec<u8> {
     result
 }
 
+pub fn no_encryption_in_place(mut data: Vec<u8>) -> Vec<u8> {
+    data.insert(0, ENC_NONE);
+    data
+}
+
 pub fn decrypt_xor(data: &[u8], passphrase: &str) -> Result<Vec<u8>> {
     if data.is_empty() { return Err(anyhow!("Empty xor payload")); }
     if passphrase.is_empty() { return Err(anyhow!("Passphrase required")); }

package/native/encoder.rs

@@ -1,5 +1,4 @@
 use anyhow::Result;
-use std::process::{Command, Stdio};
 
 const MAGIC: &[u8] = b"ROX1";
 const PIXEL_MAGIC: &[u8] = b"PXL1";
@@ -12,54 +11,23 @@ const MARKER_ZSTD: (u8, u8, u8) = (0, 255, 0);
 #[derive(Debug, Clone, Copy)]
 pub enum ImageFormat {
     Png,
-    WebP,
-    JpegXL,
 }
 
 pub fn encode_to_png(data: &[u8], compression_level: i32) -> Result<Vec<u8>> {
-    let format = predict_best_format_raw(data);
-    encode_to_png_with_encryption_name_and_format_and_filelist(data, compression_level, None, None, format, None, None, None)
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, None, None, None, None, None)
 }
 
 
 pub fn encode_to_png_with_name(data: &[u8], compression_level: i32, name: Option<&str>) -> Result<Vec<u8>> {
-    let format = predict_best_format_raw(data);
-    encode_to_png_with_encryption_name_and_format_and_filelist(data, compression_level, None, None, format, name, None, None)
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, None, None, name, None, None)
 }
 
 pub fn encode_to_png_with_name_and_filelist(data: &[u8], compression_level: i32, name: Option<&str>, file_list: Option<&str>) -> Result<Vec<u8>> {
-    encode_to_png_with_encryption_name_and_format_and_filelist(data, compression_level, None, None, ImageFormat::Png, name, file_list, None)
-}
-
-fn predict_best_format_raw(data: &[u8]) -> ImageFormat {
-    if data.len() < 512 {
-        return ImageFormat::Png;
-    }
-
-    let sample_size = data.len().min(4096);
-    let sample = &data[..sample_size];
-
-    let entropy = calculate_shannon_entropy(sample);
-    let repetition_score = detect_repetition_patterns(sample);
-    let unique_bytes = count_unique_bytes(sample);
-    let unique_ratio = unique_bytes as f64 / 256.0;
-    let is_sequential = detect_sequential_pattern(sample);
-
-    if entropy > 7.8 {
-        ImageFormat::Png
-    } else if is_sequential || repetition_score > 0.15 {
-        ImageFormat::JpegXL
-    } else if unique_ratio < 0.4 && entropy < 6.5 {
-        ImageFormat::JpegXL
-    } else if entropy < 5.0 {
-        ImageFormat::JpegXL
-    } else {
-        ImageFormat::Png
-    }
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, None, None, name, file_list, None)
 }
 
 pub fn encode_to_png_raw(data: &[u8], compression_level: i32) -> Result<Vec<u8>> {
-    encode_to_png_with_encryption_name_and_format_and_filelist(data, compression_level, None, None, ImageFormat::Png, None, None, None)
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, None, None, None, None, None)
 }
 
 pub fn encode_to_png_with_encryption_and_name(
@@ -69,8 +37,7 @@ pub fn encode_to_png_with_encryption_and_name(
     encrypt_type: Option<&str>,
     name: Option<&str>,
 ) -> Result<Vec<u8>> {
-    let format = predict_best_format_raw(data);
-    encode_to_png_with_encryption_name_and_format_and_filelist(data, compression_level, passphrase, encrypt_type, format, name, None, None)
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, passphrase, encrypt_type, name, None, None)
 }
 
 pub fn encode_to_png_with_encryption_name_and_filelist(
@@ -81,7 +48,7 @@ pub fn encode_to_png_with_encryption_name_and_filelist(
     name: Option<&str>,
     file_list: Option<&str>,
 ) -> Result<Vec<u8>> {
-    encode_to_png_with_encryption_name_and_format_and_filelist(data, compression_level, passphrase, encrypt_type, ImageFormat::Png, name, file_list, None)
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, passphrase, encrypt_type, name, file_list, None)
 }
 
 // ─── WAV container encoding ─────────────────────────────────────────────────
@@ -121,15 +88,13 @@ pub fn encode_to_wav_with_encryption_name_and_filelist(
             _ => crate::crypto::encrypt_aes(&compressed, pass)?,
         }
     } else {
-        crate::crypto::no_encryption(&compressed)
+        crate::crypto::no_encryption_in_place(compressed)
     };
 
     let meta_pixel = build_meta_pixel_with_name_and_filelist(&encrypted, name, file_list)?;
 
-    // Prepend PIXEL_MAGIC so decoder can validate the payload
     let wav_payload = [PIXEL_MAGIC, &meta_pixel].concat();
 
-    // Wrap in WAV container (44 bytes overhead, constant)
     Ok(crate::audio::bytes_to_wav(&wav_payload))
 }
 
@@ -144,28 +109,12 @@ pub fn encode_to_png_with_encryption_name_and_format_and_filelist(
     compression_level: i32,
     passphrase: Option<&str>,
     encrypt_type: Option<&str>,
-    format: ImageFormat,
+    _format: ImageFormat,
     name: Option<&str>,
     file_list: Option<&str>,
     dict: Option<&[u8]>,
 ) -> Result<Vec<u8>> {
-    let png = encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, passphrase, encrypt_type, name, file_list, dict)?;
-
-    match format {
-        ImageFormat::Png => Ok(png),
-        ImageFormat::WebP => {
-            match optimize_to_webp(&png) {
-                Ok(optimized) => reconvert_to_png(&optimized, "webp").or_else(|_| Ok(png)),
-                Err(_) => Ok(png),
-            }
-        },
-        ImageFormat::JpegXL => {
-            match optimize_to_jxl(&png) {
-                Ok(optimized) => reconvert_to_png(&optimized, "jxl").or_else(|_| Ok(png)),
-                Err(_) => Ok(png),
-            }
-        },
-    }
+    encode_to_png_with_encryption_name_and_filelist_internal(data, compression_level, passphrase, encrypt_type, name, file_list, dict)
 }
 
 fn encode_to_png_with_encryption_name_and_filelist_internal(
@@ -187,9 +136,8 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
             _ => crate::crypto::encrypt_aes(&compressed, pass)?,
         }
     } else {
-        crate::crypto::no_encryption(&compressed)
+        crate::crypto::no_encryption_in_place(compressed)
     };
-    drop(compressed);
 
     let meta_pixel = build_meta_pixel_with_name_and_filelist(&encrypted, name, file_list)?;
     drop(encrypted);
@@ -369,51 +317,6 @@ fn create_raw_deflate_from_rows(flat: &[u8], row_bytes: usize, height: usize) ->
     result
 }
 
-fn detect_sequential_pattern(data: &[u8]) -> bool {
-    if data.len() < 256 {
-        return false;
-    }
-
-    let check_len = data.len().min(256);
-    let mut sequential = 0;
-
-    for i in 0..check_len - 1 {
-        let diff = (data[i + 1] as i16 - data[i] as i16).abs();
-        if diff <= 1 {
-            sequential += 1;
-        }
-    }
-
-    sequential as f64 / (check_len - 1) as f64 > 0.6
-}
-
-fn count_unique_bytes(data: &[u8]) -> usize {
-    let mut seen = [false; 256];
-    for &byte in data {
-        seen[byte as usize] = true;
-    }
-    seen.iter().filter(|&&x| x).count()
-}
-
-fn calculate_shannon_entropy(data: &[u8]) -> f64 {
-    let mut freq = [0u32; 256];
-    for &byte in data {
-        freq[byte as usize] += 1;
-    }
-
-    let len = data.len() as f64;
-    let mut entropy = 0.0;
-
-    for &count in &freq {
-        if count > 0 {
-            let p = count as f64 / len;
-            entropy -= p * p.log2();
-        }
-    }
-
-    entropy
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -512,124 +415,3 @@ mod tests {
         }
     }
 }
-
-fn detect_repetition_patterns(data: &[u8]) -> f64 {
-    if data.len() < 4 {
-        return 0.0;
-    }
-
-    let mut repetitions = 0;
-    let mut total_checks = 0;
-
-    for i in 0..data.len().min(1024) {
-        if i + 3 < data.len() {
-            let byte = data[i];
-            if data[i + 1] == byte && data[i + 2] == byte && data[i + 3] == byte {
-                repetitions += 1;
-            }
-            total_checks += 1;
-        }
-    }
-
-    if total_checks > 0 {
-        repetitions as f64 / total_checks as f64
-    } else {
-        0.0
-    }
-}
-
-fn optimize_to_webp(png_data: &[u8]) -> Result<Vec<u8>> {
-    use std::fs;
-
-    let tmp_dir = std::env::temp_dir();
-    let id = rand::random::<u64>();
-    let tmp_in = tmp_dir.join(format!("roxify_{}_in.png", id));
-    let tmp_out = tmp_dir.join(format!("roxify_{}_out.webp", id));
-
-    fs::write(&tmp_in, png_data)?;
-
-    let status = Command::new("cwebp")
-        .args(&["-lossless", &tmp_in.to_string_lossy(), "-o", &tmp_out.to_string_lossy()])
-        .stderr(Stdio::null())
-        .stdout(Stdio::null())
-        .status()?;
-
-    if status.success() {
-        let result = fs::read(&tmp_out)?;
-        let _ = fs::remove_file(&tmp_in);
-        let _ = fs::remove_file(&tmp_out);
-        Ok(result)
-    } else {
-        let _ = fs::remove_file(&tmp_in);
-        let _ = fs::remove_file(&tmp_out);
-        Err(anyhow::anyhow!("WebP conversion failed"))
-    }
-}
-
-fn optimize_to_jxl(png_data: &[u8]) -> Result<Vec<u8>> {
-    use std::fs;
-
-    let tmp_dir = std::env::temp_dir();
-    let id = rand::random::<u64>();
-    let tmp_in = tmp_dir.join(format!("roxify_{}_in.png", id));
-    let tmp_out = tmp_dir.join(format!("roxify_{}_out.jxl", id));
-
-    fs::write(&tmp_in, png_data)?;
-
-    let status = Command::new("cjxl")
-        .args(&[&tmp_in.to_string_lossy() as &str, &tmp_out.to_string_lossy() as &str, "-d", "0", "-e", "9"])
-        .stderr(Stdio::null())
-        .stdout(Stdio::null())
-        .status()?;
-
-    if status.success() {
-        let result = fs::read(&tmp_out)?;
-        let _ = fs::remove_file(&tmp_in);
-        let _ = fs::remove_file(&tmp_out);
-        Ok(result)
-    } else {
-        let _ = fs::remove_file(&tmp_in);
-        let _ = fs::remove_file(&tmp_out);
-        Err(anyhow::anyhow!("JXL conversion failed"))
-    }
-}
-
-fn reconvert_to_png(data: &[u8], original_format: &str) -> Result<Vec<u8>> {
-    use std::fs;
-
-    let tmp_dir = std::env::temp_dir();
-    let id = rand::random::<u64>();
-    let tmp_in = match original_format {
-        "webp" => tmp_dir.join(format!("roxify_{}_reconvert_in.webp", id)),
-        "jxl" => tmp_dir.join(format!("roxify_{}_reconvert_in.jxl", id)),
-        _ => return Err(anyhow::anyhow!("Unknown format")),
-    };
-    let tmp_out = tmp_dir.join(format!("roxify_{}_reconvert_out.png", id));
-
-    fs::write(&tmp_in, data)?;
-
-    let status = match original_format {
-        "webp" => Command::new("dwebp")
-            .args(&[&tmp_in.to_string_lossy() as &str, "-o", &tmp_out.to_string_lossy() as &str])
-            .stderr(Stdio::null())
-            .stdout(Stdio::null())
-            .status()?,
-        "jxl" => Command::new("djxl")
-            .args(&[&tmp_in.to_string_lossy() as &str, &tmp_out.to_string_lossy() as &str])
-            .stderr(Stdio::null())
-            .stdout(Stdio::null())
-            .status()?,
-        _ => return Err(anyhow::anyhow!("Unknown format")),
-    };
-
-    if status.success() {
-        let result = fs::read(&tmp_out)?;
-        let _ = fs::remove_file(&tmp_in);
-        let _ = fs::remove_file(&tmp_out);
-        Ok(result)
-    } else {
-        let _ = fs::remove_file(&tmp_in);
-        let _ = fs::remove_file(&tmp_out);
-        Err(anyhow::anyhow!("Reconversion to PNG failed"))
-    }
-}