roxify 1.13.3 → 1.13.4

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.13.3"
+version = "1.13.4"
 edition = "2021"
 publish = false
 
@@ -35,6 +35,7 @@ zstd = { version = "0.11", features = ["zstdmt"] }
 crc32fast = "1.3"
 num_cpus = "1.16"
 clap = { version = "4", features = ["derive"] }
+serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 anyhow = "1.0"
 png = "0.18.0"

package/README.md

@@ -57,27 +57,29 @@ The core compression and image-processing logic is written in Rust and exposed t
 
 ## Benchmarks
 
-All measurements taken on Linux x64 (Intel i7-6700K @ 4.0 GHz, 32 GB RAM). Roxify uses its native Rust CLI (`roxify_native`) with streaming Zstd L3 + multi-threaded + LDM + window_log(30). ZIP uses `zip -r -q -9` (maximum compression).
+All measurements below use Roxify native Rust CLI (`roxify_native`) with streaming directory packing, Zstd L3, multi-threading, long-distance matching, and `window_log(30)`.
 
-### Real-world directory encoding: Roxify vs ZIP
+### Cold-cache throughput on ext4
 
-| Dataset | Original | ZIP -9 | Roxify PNG | ZIP time | Roxify time | Speedup |
-| --- | --- | --- | --- | --- | --- | --- |
-| Test A (19 638 files, 177 MB) | 177 MB | 87.7 MB (49.6%) | 54.9 MB (31.0%) | 17.6 s | 1.2 s | 14.7x |
-| Test B (3 936 files, 1.4 GB) | 1.4 GB | 513 MB (36.7%) | 409 MB (29.2%) | 1 min 46 s | 6.7 s | 15.9x |
+Measured with targeted page-cache eviction (`POSIX_FADV_DONTNEED`) before both encode and decode. Raw manifest lives in `docs/COLD_BENCHMARK_2026-04-15.json`.
 
-### Decompression
+| Dataset | Files | Source | Output PNG | Encode | Encode throughput | Decode | Decode throughput |
+| --- | --- | --- | --- | --- | --- | --- | --- |
+| Glados-Disc | 19,645 | 208.18 MiB | 54.83 MiB | 2.883 s | 72.22 MiB/s | 0.954 s | 218.16 MiB/s |
+| Gmod | 3,936 | 1.36 GiB | 411.09 MiB | 6.127 s | 227.69 MiB/s | 5.850 s | 238.48 MiB/s |
 
-| Dataset | unzip | Roxify decode | Speedup |
-| --- | --- | --- | --- |
-| Test A (177 MB) | 2.4 s | 1.3 s | 1.8x |
-| Test B (1.4 GB) | 8.4 s | 2.9 s | 2.9x |
+### High-latency source filesystem encode
 
-Roxify produces a valid PNG image instead of a ZIP archive. On these real-world datasets it compresses 20-37% smaller than ZIP -9 while encoding 15x faster, thanks to multi-threaded Zstd with long-distance matching.
+Roxify 1.13.4 adds adaptive parallel preload for small files before feeding Zstd. This specifically targets metadata-heavy trees on slower filesystems such as NTFS, APFS, exFAT, and network-backed mounts.
+
+| Dataset | Source FS | Before 1.13.4 | Roxify 1.13.4 | Speedup |
+| --- | --- | --- | --- | --- |
+| Glados-Disc (19,645 files) | NTFS under Linux | 81.608 s | 2.189 s | 37.3x |
+| Gmod (3,936 files) | NTFS under Linux | 22.578 s | 4.517 s | 5.0x |
 
 ### Data integrity
 
-100% lossless roundtrip verified by byte-exact diff on all datasets. Start and end markers verified in every output PNG.
+All benchmark runs completed with byte-exact roundtrip validation. Decode output matched original logical source bytes on every dataset.
 
 ---
 

package/native/main.rs

@@ -144,6 +144,19 @@ fn parse_markers(v: &[String]) -> Option<Vec<u8>> {
 
 fn main() -> anyhow::Result<()> {
     let cli = Cli::parse();
+
+fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
+    if files.trim_start().starts_with('[') {
+        serde_json::from_str::<Vec<String>>(files)
+            .map_err(|e| anyhow::anyhow!("Invalid JSON for --files: {}", e))
+    } else {
+        Ok(files
+            .split(',')
+            .map(|file| file.trim().to_string())
+            .filter(|file| !file.is_empty())
+            .collect())
+    }
+}
     match cli.command {
         Commands::TrainDict { samples, size, output } => {
             let dict = core::train_zstd_dictionary(&samples, size)?;
@@ -169,7 +182,7 @@ fn main() -> anyhow::Result<()> {
                         eprintln!("PROGRESS:{}:{}:{}", current, total, step);
                     })),
                 )?;
-                println!("(TAR archive, rXFL chunk embedded)");
+                println!("(directory payload, rXFL chunk embedded)");
                 return Ok(());
             }
 
@@ -234,7 +247,7 @@ fn main() -> anyhow::Result<()> {
             if file_list_json.is_some() {
                 eprintln!("PROGRESS:100:100:done");
                 if is_dir {
-                    println!("(TAR archive, rXFL chunk embedded)");
+                    println!("(directory payload, rXFL chunk embedded)");
                 } else {
                     println!("(rXFL chunk embedded)");
                 }
@@ -343,6 +356,11 @@ fn main() -> anyhow::Result<()> {
                         && sig == [137, 80, 78, 71, 13, 10, 26, 10]
                 });
 
+            let requested_files = match files.as_deref() {
+                Some(files_str) => Some(parse_requested_files(files_str)?),
+                None => None,
+            };
+
             if is_png_file && files.is_none() && dict.is_none() {
                 let out_dir = output.clone().unwrap_or_else(|| PathBuf::from("out.raw"));
                 match streaming_decode::streaming_decode_to_dir_encrypted_with_progress(
@@ -355,7 +373,7 @@ fn main() -> anyhow::Result<()> {
                 ) {
                     Ok(written) => {
                         eprintln!("PROGRESS:100:100:done");
-                        println!("Unpacked {} files (TAR)", written.len());
+                        println!("Unpacked {} files", written.len());
                         return Ok(());
                     }
                     Err(e) => {
@@ -364,25 +382,31 @@ fn main() -> anyhow::Result<()> {
                 }
             }
 
+            if is_png_file && requested_files.is_some() && dict.is_none() {
+                let out_dir = output.clone().unwrap_or_else(|| PathBuf::from("."));
+                std::fs::create_dir_all(&out_dir).map_err(|e| anyhow::anyhow!("Cannot create output directory {:?}: {}", out_dir, e))?;
+                let written = streaming_decode::streaming_decode_selected_to_dir_encrypted_with_progress(
+                    &input,
+                    &out_dir,
+                    requested_files.as_deref(),
+                    passphrase.as_deref(),
+                    Some(Box::new(|current, total, step| {
+                        eprintln!("PROGRESS:{}:{}:{}", current, total, step);
+                    })),
+                ).map_err(|e| anyhow::anyhow!(e))?;
+                eprintln!("PROGRESS:100:100:done");
+                println!("Unpacked {} files", written.len());
+                return Ok(());
+            }
+
             let buf = read_all(&input)?;
             eprintln!("PROGRESS:20:100:decompressing");
             let dict_bytes: Option<Vec<u8>> = match dict {
                 Some(path) => Some(read_all(&path)?),
                 None => None,
             };
-                        if let Some(files_str) = files {
-                                let file_list: Option<Vec<String>> = if files_str.trim_start().starts_with('[') {
-                    match serde_json::from_str::<Vec<String>>(&files_str) {
-                        Ok(v) => Some(v),
-                        Err(e) => {
-                            eprintln!("Invalid JSON for --files: {}", e);
-                            std::process::exit(1);
-                        }
-                    }
-                } else {
-                    let list = files_str.split(',').map(|s| s.trim().to_string()).filter(|s| !s.is_empty()).collect::<Vec<_>>();
-                    Some(list)
-                };
+                        if requested_files.is_some() {
+                                let file_list = requested_files;
 
                                 let is_png = buf.len() >= 8 && &buf[0..8] == &[137, 80, 78, 71, 13, 10, 26, 10];
 
@@ -427,7 +451,9 @@ fn main() -> anyhow::Result<()> {
                 std::fs::create_dir_all(&out_dir).map_err(|e| anyhow::anyhow!("Cannot create output directory {:?}: {}", out_dir, e))?;
                 let files_slice = file_list.as_ref().map(|v| v.as_slice());
 
-                                let written = packer::unpack_stream_to_dir(&mut reader, &out_dir, files_slice).map_err(|e| anyhow::anyhow!(e))?;
+                                let written = packer::unpack_stream_to_dir(&mut reader, &out_dir, files_slice, Some(&|current, total, step| {
+                                    eprintln!("PROGRESS:{}:{}:{}", current, total, step);
+                                }), 0).map_err(|e| anyhow::anyhow!(e))?;
                 eprintln!("PROGRESS:100:100:done");
                 println!("Unpacked {} files", written.len());
             } else {
@@ -498,7 +524,7 @@ fn main() -> anyhow::Result<()> {
                         .map_err(|e| anyhow::anyhow!("mkdir {:?}: {}", out_dir, e))?;
                     let written = archive::tar_unpack(&out_bytes, &out_dir)
                         .map_err(|e| anyhow::anyhow!(e))?;
-                    println!("Unpacked {} files (TAR) to {:?}", written.len(), out_dir);
+                    println!("Unpacked {} files to {:?}", written.len(), out_dir);
                 } else if out_bytes.len() >= 4
                     && (u32::from_be_bytes(out_bytes[0..4].try_into().unwrap()) == 0x524f5850u32
                         || u32::from_be_bytes(out_bytes[0..4].try_into().unwrap()) == 0x524f5849u32)

package/native/packer.rs

@@ -255,13 +255,50 @@ fn unpack_entries_sequential(buf: &[u8], start: usize, out_dir: &Path, files_opt
     Ok(written)
 }
 
-pub fn unpack_stream_to_dir<R: std::io::Read>(reader: &mut R, out_dir: &Path, files_opt: Option<&[String]>) -> Result<Vec<String>> {
+fn unpack_progress_percent(total_expected: u64, bytes_processed: u64, file_count: usize, processed_files: usize) -> u64 {
+    if total_expected > 0 {
+        return 10 + (bytes_processed.saturating_mul(89) / total_expected).min(89);
+    }
+    if file_count > 0 {
+        return 10 + ((processed_files as u64).saturating_mul(89) / file_count as u64).min(89);
+    }
+    10
+}
+
+fn report_unpack_progress(
+    progress: Option<&(dyn Fn(u64, u64, &str) + Send)>,
+    total_expected: u64,
+    bytes_processed: u64,
+    file_count: usize,
+    processed_files: usize,
+    last_pct: &mut u64,
+) {
+    if let Some(cb) = progress {
+        let pct = unpack_progress_percent(total_expected, bytes_processed, file_count, processed_files);
+        if pct > *last_pct {
+            *last_pct = pct;
+            cb(pct, 100, "extracting");
+        }
+    }
+}
+
+pub fn unpack_stream_to_dir<R: std::io::Read>(
+    reader: &mut R,
+    out_dir: &Path,
+    files_opt: Option<&[String]>,
+    progress: Option<&(dyn Fn(u64, u64, &str) + Send)>,
+    total_expected: u64,
+) -> Result<Vec<String>> {
     let mut written = Vec::new();
     let mut buf: Vec<u8> = Vec::new();
     let mut pos: usize = 0;
     let mut temp = [0u8; 64 * 1024];
     let files_filter: Option<std::collections::HashSet<String>> = files_opt.map(|l| l.iter().map(|s| s.clone()).collect());
     let mut requested = files_filter.as_ref().map(|s| s.len()).unwrap_or(usize::MAX);
+    let mut file_count = 0usize;
+    let mut processed_files = 0usize;
+    let mut bytes_processed = 0u64;
+    let mut last_pct = 10u64;
 
         let mut header_parsed = false;
     let debug = std::env::var("ROX_DEBUG").is_ok();
@@ -280,10 +317,10 @@ pub fn unpack_stream_to_dir<R: std::io::Read>(reader: &mut R, out_dir: &Path, fi
                 if debug { eprintln!("[rox debug] magic_header=0x{:08x}", magic_header); }
                 if magic_header == 0x524f5850u32 {
                                         pos += 4;
-                                        let _file_count = u32::from_be_bytes(buf[pos..pos+4].try_into().unwrap()) as usize;
+                                        file_count = u32::from_be_bytes(buf[pos..pos+4].try_into().unwrap()) as usize;
                     pos += 4;
                     header_parsed = true;
-                    if debug { eprintln!("[rox debug] header parsed, file_count={}", _file_count); }
+                    if debug { eprintln!("[rox debug] header parsed, file_count={}", file_count); }
                 } else if magic_header == 0x524f5831u32 {
                                         if debug { eprintln!("[rox debug] found ROX1 outer magic, skipping 4 bytes"); }
                     pos += 4;
@@ -310,6 +347,8 @@ pub fn unpack_stream_to_dir<R: std::io::Read>(reader: &mut R, out_dir: &Path, fi
             let content_start = pos + 2 + name_len + 8;
             let content_end = content_start + size;
             let content = &buf[content_start..content_end];
+            processed_files = processed_files.saturating_add(1);
+            bytes_processed = bytes_processed.saturating_add(size as u64);
 
                         let p = Path::new(&name);
             let mut safe = std::path::PathBuf::new();
@@ -328,10 +367,18 @@ pub fn unpack_stream_to_dir<R: std::io::Read>(reader: &mut R, out_dir: &Path, fi
                 written.push(safe.to_string_lossy().to_string());
                 if let Some(_set) = files_filter.as_ref() {
                     requested = requested.saturating_sub(1);
-                    if requested == 0 { return Ok(written); }
+                    report_unpack_progress(progress, total_expected, bytes_processed, file_count, processed_files, &mut last_pct);
+                    if requested == 0 {
+                        if let Some(cb) = progress {
+                            cb(99, 100, "finishing");
+                        }
+                        return Ok(written);
+                    }
                 }
             }
 
+            report_unpack_progress(progress, total_expected, bytes_processed, file_count, processed_files, &mut last_pct);
+
             pos = content_end;                         if pos > 0 {
                 buf.drain(0..pos);
                 pos = 0;
@@ -344,6 +391,10 @@ pub fn unpack_stream_to_dir<R: std::io::Read>(reader: &mut R, out_dir: &Path, fi
         }
     }
 
+    if let Some(cb) = progress {
+        cb(99, 100, "finishing");
+    }
+
     Ok(written)
 }
 
@@ -390,7 +441,7 @@ mod stream_tests {
         let tmpdir = std::env::temp_dir().join(format!("rox_unpack_test_{}", ms));
         let _ = std::fs::create_dir_all(&tmpdir);
 
-        let out = unpack_stream_to_dir(&mut dec2, &tmpdir, None)?;
+        let out = unpack_stream_to_dir(&mut dec2, &tmpdir, None, None, 0)?;
 
                 assert_eq!(out.len(), 2);
         assert!(tmpdir.join("file1.txt").exists());
@@ -432,7 +483,7 @@ mod stream_tests {
         let tmpdir = std::env::temp_dir().join(format!("rox_unpack_png_test_{}", ms));
         let _ = std::fs::create_dir_all(&tmpdir);
 
-        let out = unpack_stream_to_dir(&mut dec, &tmpdir, None)?;
+        let out = unpack_stream_to_dir(&mut dec, &tmpdir, None, None, 0)?;
 
         assert_eq!(out.len(), 2);
         assert!(tmpdir.join("file1.txt").exists());

package/native/streaming_decode.rs

@@ -4,13 +4,14 @@ use cipher::{KeyIvInit, StreamCipher};
 
 const PIXEL_MAGIC: &[u8] = b"PXL1";
 const MARKER_BYTES: usize = 12;
+const PACK_MAGIC: [u8; 4] = 0x524f5850u32.to_be_bytes();
 
 type Aes256Ctr = ctr::Ctr64BE<aes::Aes256>;
 
 pub type DecodeProgressCallback = Box<dyn Fn(u64, u64, &str) + Send>;
 
 pub fn streaming_decode_to_dir(png_path: &Path, out_dir: &Path) -> Result<Vec<String>, String> {
-    streaming_decode_to_dir_encrypted_with_progress(png_path, out_dir, None, None)
+    streaming_decode_selected_to_dir_encrypted_with_progress(png_path, out_dir, None, None, None)
 }
 
 pub fn streaming_decode_to_dir_encrypted(
@@ -18,7 +19,7 @@ pub fn streaming_decode_to_dir_encrypted(
     out_dir: &Path,
     passphrase: Option<&str>,
 ) -> Result<Vec<String>, String> {
-    streaming_decode_to_dir_encrypted_with_progress(png_path, out_dir, passphrase, None)
+    streaming_decode_selected_to_dir_encrypted_with_progress(png_path, out_dir, None, passphrase, None)
 }
 
 pub fn streaming_decode_to_dir_encrypted_with_progress(
@@ -26,6 +27,16 @@ pub fn streaming_decode_to_dir_encrypted_with_progress(
     out_dir: &Path,
     passphrase: Option<&str>,
     progress: Option<DecodeProgressCallback>,
+) -> Result<Vec<String>, String> {
+    streaming_decode_selected_to_dir_encrypted_with_progress(png_path, out_dir, None, passphrase, progress)
+}
+
+pub fn streaming_decode_selected_to_dir_encrypted_with_progress(
+    png_path: &Path,
+    out_dir: &Path,
+    files_opt: Option<&[String]>,
+    passphrase: Option<&str>,
+    progress: Option<DecodeProgressCallback>,
 ) -> Result<Vec<String>, String> {
     let file = std::fs::File::open(png_path).map_err(|e| format!("open: {}", e))?;
     let mmap = unsafe { memmap2::Mmap::map(&file).map_err(|e| format!("mmap: {}", e))? };
@@ -88,7 +99,7 @@ pub fn streaming_decode_to_dir_encrypted_with_progress(
             let mut decoder = zstd::stream::Decoder::new(remaining_reader)
                 .map_err(|e| format!("zstd decoder: {}", e))?;
             decoder.window_log_max(31).map_err(|e| format!("zstd window_log_max: {}", e))?;
-            read_rox1_and_untar_with_progress(decoder, out_dir, progress, total_expected)
+            read_rox1_and_unpack_with_progress(decoder, out_dir, files_opt, progress, total_expected)
         }
         0x03 => {
             let pass = passphrase.ok_or("Passphrase required for AES-CTR decryption")?;
@@ -112,15 +123,16 @@ pub fn streaming_decode_to_dir_encrypted_with_progress(
             let mut decoder = zstd::stream::Decoder::new(ctr_reader)
                 .map_err(|e| format!("zstd decoder: {}", e))?;
             decoder.window_log_max(31).map_err(|e| format!("zstd window_log_max: {}", e))?;
-            read_rox1_and_untar_with_progress(decoder, out_dir, progress, total_expected)
+            read_rox1_and_unpack_with_progress(decoder, out_dir, files_opt, progress, total_expected)
         }
         _ => Err(format!("Unsupported encryption (enc=0x{:02x}) in streaming decode", enc_byte)),
     }
 }
 
-fn read_rox1_and_untar_with_progress<R: Read>(
+fn read_rox1_and_unpack_with_progress<R: Read>(
     mut decoder: R,
     out_dir: &Path,
+    files_opt: Option<&[String]>,
     progress: Option<DecodeProgressCallback>,
     total_expected: u64,
 ) -> Result<Vec<String>, String> {
@@ -130,7 +142,17 @@ fn read_rox1_and_untar_with_progress<R: Read>(
         return Err(format!("Expected ROX1, got {:?}", magic));
     }
     std::fs::create_dir_all(out_dir).map_err(|e| format!("mkdir: {}", e))?;
-    tar_unpack_from_reader_with_progress(decoder, out_dir, progress, total_expected)
+
+    let mut prefix = [0u8; 4];
+    decoder.read_exact(&mut prefix).map_err(|e| format!("read payload magic: {}", e))?;
+    let mut chained = std::io::Cursor::new(prefix).chain(decoder);
+
+    if prefix == PACK_MAGIC {
+        crate::packer::unpack_stream_to_dir(&mut chained, out_dir, files_opt, progress.as_deref(), total_expected)
+            .map_err(|e| format!("pack unpack: {}", e))
+    } else {
+        tar_unpack_from_reader_with_progress(chained, out_dir, files_opt, progress, total_expected)
+    }
 }
 
 fn parse_png_header(data: &[u8]) -> Result<(usize, usize, usize, usize), String> {
@@ -360,6 +382,7 @@ impl<R: Read> Read for CtrDecryptReader<R> {
 fn tar_unpack_from_reader_with_progress<R: Read>(
     reader: R,
     output_dir: &Path,
+    files_opt: Option<&[String]>,
     progress: Option<DecodeProgressCallback>,
     total_expected: u64,
 ) -> Result<Vec<String>, String> {
@@ -369,12 +392,16 @@ fn tar_unpack_from_reader_with_progress<R: Read>(
     let mut created_dirs = std::collections::HashSet::new();
     let mut bytes_extracted: u64 = 0;
     let mut last_pct: u64 = 10;
+    let files_filter: Option<std::collections::HashSet<&str>> = files_opt.map(|files| files.iter().map(|file| file.as_str()).collect());
+    let mut remaining = files_filter.as_ref().map(|files| files.len()).unwrap_or(usize::MAX);
 
     let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
     for entry in entries {
         let mut entry = entry.map_err(|e| format!("tar entry: {}", e))?;
         let entry_size = entry.size();
         let path = entry.path().map_err(|e| format!("tar path: {}", e))?.to_path_buf();
+        let logical_path = path.to_string_lossy().replace('\\', "/");
+        let should_write = files_filter.as_ref().map(|files| files.contains(logical_path.as_str())).unwrap_or(true);
 
         let mut safe = std::path::PathBuf::new();
         for comp in path.components() {
@@ -386,6 +413,23 @@ fn tar_unpack_from_reader_with_progress<R: Read>(
             continue;
         }
 
+        if !should_write {
+            std::io::copy(&mut entry, &mut std::io::sink()).map_err(|e| format!("skip {:?}: {}", safe, e))?;
+            bytes_extracted += entry_size;
+            if let Some(ref cb) = progress {
+                let pct = if total_expected > 0 {
+                    10 + (bytes_extracted * 89 / total_expected).min(89)
+                } else {
+                    (10 + (bytes_extracted / (1024 * 1024))).min(99)
+                };
+                if pct > last_pct {
+                    last_pct = pct;
+                    cb(pct, 100, "extracting");
+                }
+            }
+            continue;
+        }
+
         let dest = output_dir.join(&safe);
         if let Some(parent) = dest.parent() {
             if created_dirs.insert(parent.to_path_buf()) {
@@ -399,6 +443,9 @@ fn tar_unpack_from_reader_with_progress<R: Read>(
         );
         std::io::copy(&mut entry, &mut f).map_err(|e| format!("write {:?}: {}", dest, e))?;
         written.push(safe.to_string_lossy().to_string());
+        if files_filter.is_some() {
+            remaining = remaining.saturating_sub(1);
+        }
 
         bytes_extracted += entry_size;
         if let Some(ref cb) = progress {
@@ -412,6 +459,9 @@ fn tar_unpack_from_reader_with_progress<R: Read>(
                 cb(pct, 100, "extracting");
             }
         }
+        if remaining == 0 {
+            break;
+        }
     }
 
     if let Some(ref cb) = progress {

package/native/streaming_encode.rs

@@ -1,8 +1,9 @@
 use std::io::{Write, BufWriter, Read};
 use std::fs::File;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use rayon::prelude::*;
+use serde::Serialize;
 use walkdir::WalkDir;
-use tar::{Builder, Header};
 
 const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
 const PIXEL_MAGIC: &[u8] = b"PXL1";
@@ -10,9 +11,35 @@ const MARKER_START: [(u8, u8, u8); 3] = [(255, 0, 0), (0, 255, 0), (0, 0, 255)];
 const MARKER_END: [(u8, u8, u8); 3] = [(0, 0, 255), (0, 255, 0), (255, 0, 0)];
 const MARKER_ZSTD: (u8, u8, u8) = (0, 255, 0);
 const MAGIC: &[u8] = b"ROX1";
+const PACK_MAGIC: u32 = 0x524f5850;
+
+const MIN_ZST_CAPACITY: usize = 16 * 1024 * 1024;
+const MB: u64 = 1024 * 1024;
+const MAX_FILE_BUFFER_CAPACITY: usize = 4 * 1024 * 1024;
+const PARALLEL_IO_FILE_THRESHOLD: u64 = MB;
+const PARALLEL_IO_BATCH_BYTES: u64 = 128 * MB;
+const PARALLEL_IO_BATCH_FILES: usize = 512;
+const PARALLEL_IO_MIN_FILES: usize = 8;
 
 pub type ProgressCallback = Box<dyn Fn(u64, u64, &str) + Send>;
 
+struct DirectoryFile {
+    path: PathBuf,
+    rel_path: String,
+    size: u64,
+}
+
+#[derive(Serialize)]
+struct FileListEntry {
+    name: String,
+    size: u64,
+}
+
+struct CollectedDirectory {
+    entries: Vec<DirectoryFile>,
+    total_bytes: u64,
+}
+
 pub fn encode_dir_to_png(
     dir_path: &Path,
     output_path: &Path,
@@ -42,9 +69,7 @@ pub fn encode_dir_to_png_encrypted_with_progress(
     encrypt_type: Option<&str>,
     progress: Option<ProgressCallback>,
 ) -> anyhow::Result<()> {
-    let (zst_buf, file_list) = compress_dir_to_zst_mem(dir_path, compression_level, &progress)?;
-
-    let file_list_json = serde_json::to_string(&file_list)?;
+    let (zst_buf, file_list_json) = compress_dir_to_zst_mem(dir_path, compression_level, &progress)?;
 
     let result = write_png_from_zst_mem(
         zst_buf, output_path, name, Some(&file_list_json),
@@ -62,26 +87,19 @@ fn compress_dir_to_zst_mem(
     dir_path: &Path,
     compression_level: i32,
     progress: &Option<ProgressCallback>,
-) -> anyhow::Result<(Vec<u8>, Vec<serde_json::Value>)> {
-    let base = dir_path;
-
-    let entries: Vec<_> = WalkDir::new(dir_path)
-        .follow_links(false)
-        .into_iter()
-        .filter_map(|e| e.ok())
-        .filter(|e| e.file_type().is_file())
-        .collect();
-
-    let total_bytes: u64 = entries.iter()
-        .filter_map(|e| std::fs::metadata(e.path()).ok())
-        .map(|m| m.len())
-        .sum();
+) -> anyhow::Result<(Vec<u8>, String)> {
+    let collected = collect_directory_files(dir_path);
+    let total_bytes = collected.total_bytes;
+    let entries = collected.entries;
 
     let actual_level = compression_level.min(3);
-    let mut encoder = zstd::stream::Encoder::new(Vec::with_capacity(16 * 1024 * 1024), actual_level)
+    let mut encoder = zstd::stream::Encoder::new(
+        Vec::with_capacity(estimate_zst_capacity(total_bytes)),
+        actual_level,
+    )
         .map_err(|e| anyhow::anyhow!("zstd init: {}", e))?;
 
-    let threads = num_cpus::get() as u32;
+    let threads = select_zstd_threads(total_bytes);
     if threads > 1 {
         let _ = encoder.multithread(threads);
     }
@@ -89,61 +107,221 @@ fn compress_dir_to_zst_mem(
     let _ = encoder.window_log(30);
 
     encoder.write_all(MAGIC)?;
+    encoder.write_all(&PACK_MAGIC.to_be_bytes())?;
+    encoder.write_all(&(entries.len() as u32).to_be_bytes())?;
 
-    let mut file_list = Vec::new();
+    let mut file_list = Vec::with_capacity(entries.len());
     let mut bytes_processed: u64 = 0;
     let mut last_pct: u64 = 0;
-    {
-        let mut tar_builder = Builder::new(&mut encoder);
-        for entry in entries.iter() {
-            let full = entry.path();
-            let rel = full.strip_prefix(base).unwrap_or(full);
-            let rel_str = rel.to_string_lossy().replace('\\', "/");
-
-            let metadata = match std::fs::metadata(full) {
-                Ok(m) => m,
-                Err(_) => continue,
-            };
-            let size = metadata.len();
-
-            let mut header = Header::new_gnu();
-            header.set_size(size);
-            header.set_mode(0o644);
-            header.set_cksum();
-
-            let file = match File::open(full) {
-                Ok(f) => f,
-                Err(_) => continue,
-            };
-            let buf_reader = std::io::BufReader::with_capacity(
-                (size as usize).min(4 * 1024 * 1024).max(8192),
-                file,
-            );
-
-            tar_builder.append_data(&mut header, &rel_str, buf_reader)
-                .map_err(|e| anyhow::anyhow!("tar append {}: {}", rel_str, e))?;
-
-            file_list.push(serde_json::json!({"name": rel_str, "size": size}));
-
-            bytes_processed += size;
-            if let Some(ref cb) = progress {
-                let pct = if total_bytes > 0 {
-                    (bytes_processed * 89 / total_bytes).min(89)
-                } else {
-                    89
+    let mut entry_index = 0usize;
+    while entry_index < entries.len() {
+        let batch_end = select_parallel_batch_end(&entries, entry_index);
+        if batch_end > entry_index + 1 {
+            let loaded = load_small_file_batch(&entries[entry_index..batch_end])?;
+            for (entry, maybe_bytes) in entries[entry_index..batch_end].iter().zip(loaded.into_iter()) {
+                let Some(bytes) = maybe_bytes else {
+                    continue;
                 };
-                if pct > last_pct {
-                    last_pct = pct;
-                    cb(pct, 100, "compressing");
-                }
+
+                write_pack_entry_header(&mut encoder, &entry.rel_path, entry.size)?;
+                encoder.write_all(&bytes)
+                    .map_err(|e| anyhow::anyhow!("pack write {}: {}", entry.rel_path, e))?;
+
+                file_list.push(FileListEntry {
+                    name: entry.rel_path.clone(),
+                    size: entry.size,
+                });
+
+                bytes_processed += entry.size;
+                report_compress_progress(progress, total_bytes, bytes_processed, &mut last_pct);
             }
+            entry_index = batch_end;
+            continue;
+        }
+
+        let entry = &entries[entry_index];
+        if write_directory_entry(&mut encoder, entry)? {
+            file_list.push(FileListEntry {
+                name: entry.rel_path.clone(),
+                size: entry.size,
+            });
+
+            bytes_processed += entry.size;
+            report_compress_progress(progress, total_bytes, bytes_processed, &mut last_pct);
         }
-        tar_builder.finish().map_err(|e| anyhow::anyhow!("tar finish: {}", e))?;
+        entry_index += 1;
     }
 
     let zst_buf = encoder.finish().map_err(|e| anyhow::anyhow!("zstd finish: {}", e))?;
+    let file_list_json = serde_json::to_string(&file_list)?;
 
-    Ok((zst_buf, file_list))
+    Ok((zst_buf, file_list_json))
+}
+
+fn write_pack_entry_header<W: Write>(writer: &mut W, rel_path: &str, size: u64) -> anyhow::Result<()> {
+    let name_bytes = rel_path.as_bytes();
+    let name_len = u16::try_from(name_bytes.len())
+        .map_err(|_| anyhow::anyhow!("path too long for pack entry: {}", rel_path))?;
+    writer.write_all(&name_len.to_be_bytes())?;
+    writer.write_all(name_bytes)?;
+    writer.write_all(&size.to_be_bytes())?;
+    Ok(())
+}
+
+fn write_directory_entry<W: Write>(writer: &mut W, entry: &DirectoryFile) -> anyhow::Result<bool> {
+    let file = match File::open(&entry.path) {
+        Ok(file) => file,
+        Err(_) => return Ok(false),
+    };
+
+    write_pack_entry_header(writer, &entry.rel_path, entry.size)?;
+
+    let mut buf_reader = std::io::BufReader::with_capacity(file_buffer_capacity(entry.size), file);
+    std::io::copy(&mut buf_reader, writer)
+        .map_err(|e| anyhow::anyhow!("pack write {}: {}", entry.rel_path, e))?;
+
+    Ok(true)
+}
+
+fn load_small_file_batch(entries: &[DirectoryFile]) -> anyhow::Result<Vec<Option<Vec<u8>>>> {
+    entries.par_iter().map(load_directory_entry_bytes).collect()
+}
+
+fn load_directory_entry_bytes(entry: &DirectoryFile) -> anyhow::Result<Option<Vec<u8>>> {
+    let mut file = match File::open(&entry.path) {
+        Ok(file) => file,
+        Err(_) => return Ok(None),
+    };
+
+    let reserve = usize::try_from(entry.size.min(PARALLEL_IO_BATCH_BYTES)).unwrap_or(MAX_FILE_BUFFER_CAPACITY);
+    let mut bytes = Vec::with_capacity(reserve.max(8192));
+    file.read_to_end(&mut bytes)
+        .map_err(|e| anyhow::anyhow!("pack read {}: {}", entry.rel_path, e))?;
+
+    Ok(Some(bytes))
+}
+
+fn select_parallel_batch_end(entries: &[DirectoryFile], start: usize) -> usize {
+    let Some(first) = entries.get(start) else {
+        return start;
+    };
+    if !should_parallelize_entry(first) {
+        return start + 1;
+    }
+
+    let mut end = start;
+    let mut batch_bytes = 0u64;
+    while end < entries.len() {
+        let entry = &entries[end];
+        if !should_parallelize_entry(entry) {
+            break;
+        }
+        if end > start {
+            if end - start >= PARALLEL_IO_BATCH_FILES {
+                break;
+            }
+            if batch_bytes.saturating_add(entry.size) > PARALLEL_IO_BATCH_BYTES {
+                break;
+            }
+        }
+        batch_bytes = batch_bytes.saturating_add(entry.size);
+        end += 1;
+    }
+
+    if end - start >= PARALLEL_IO_MIN_FILES {
+        end
+    } else {
+        start + 1
+    }
+}
+
+fn should_parallelize_entry(entry: &DirectoryFile) -> bool {
+    entry.size <= PARALLEL_IO_FILE_THRESHOLD
+}
+
+fn file_buffer_capacity(size: u64) -> usize {
+    usize::try_from(size)
+        .unwrap_or(MAX_FILE_BUFFER_CAPACITY)
+        .min(MAX_FILE_BUFFER_CAPACITY)
+        .max(8192)
+}
+
+fn report_compress_progress(
+    progress: &Option<ProgressCallback>,
+    total_bytes: u64,
+    bytes_processed: u64,
+    last_pct: &mut u64,
+) {
+    if let Some(ref cb) = progress {
+        let pct = if total_bytes > 0 {
+            (bytes_processed * 89 / total_bytes).min(89)
+        } else {
+            89
+        };
+        if pct > *last_pct {
+            *last_pct = pct;
+            cb(pct, 100, "compressing");
+        }
+    }
+}
+
+fn collect_directory_files(dir_path: &Path) -> CollectedDirectory {
+    let mut entries = Vec::new();
+    let mut total_bytes = 0u64;
+
+    for entry in WalkDir::new(dir_path)
+        .follow_links(false)
+        .into_iter()
+        .filter_map(|entry| entry.ok())
+        .filter(|entry| entry.file_type().is_file())
+    {
+        let size = match entry.metadata() {
+            Ok(metadata) => metadata.len(),
+            Err(_) => continue,
+        };
+        let path = entry.into_path();
+        let rel = path.strip_prefix(dir_path).unwrap_or(path.as_path());
+        let rel_path = normalize_rel_path(rel);
+
+        total_bytes += size;
+        entries.push(DirectoryFile {
+            path,
+            rel_path,
+            size,
+        });
+    }
+
+    CollectedDirectory {
+        entries,
+        total_bytes,
+    }
+}
+
+fn normalize_rel_path(path: &Path) -> String {
+    let rel_path = path.to_string_lossy();
+    if rel_path.contains('\\') {
+        rel_path.replace('\\', "/")
+    } else {
+        rel_path.into_owned()
+    }
+}
+
+fn estimate_zst_capacity(total_bytes: u64) -> usize {
+    let capped = total_bytes.min(usize::MAX as u64) as usize;
+    (capped / 3).max(MIN_ZST_CAPACITY)
+}
+
+fn select_zstd_threads(total_bytes: u64) -> u32 {
+    let max_threads = num_cpus::get().max(1) as u32;
+    if total_bytes <= 32 * MB {
+        1
+    } else if total_bytes <= 128 * MB {
+        max_threads.min(2)
+    } else if total_bytes <= 512 * MB {
+        max_threads.min(4)
+    } else {
+        max_threads.min(8)
+    }
 }
 
 fn write_png_from_zst_mem(
@@ -312,13 +490,18 @@ fn write_idat_streaming<W: Write, R: Read>(
     let fl_chunk_data = file_list_chunk.unwrap_or(&[]);
     let payload_total = header_bytes.len() + zst_size + hmac_trailer_len + fl_chunk_data.len();
     let padding_after = total_data_bytes - payload_total.min(total_data_bytes);
-
     let marker_end_bytes = build_marker_end_bytes();
 
     let mut flat_pos: usize = 0;
     let mut scanline_pos: usize = 0;
     let mut deflate_block_remaining: usize = 0;
 
+    let mut adler = simd_adler32::Adler32::new();
+
+    let buf_size = 1024 * 1024;
+    let mut transfer_buf = vec![0u8; buf_size];
+    let zero_buf = vec![0u8; buf_size];
+
     let mut header_pos: usize = 0;
     let mut zst_remaining = zst_size;
     let mut hmac_pos: usize = 0;
@@ -327,12 +510,6 @@ fn write_idat_streaming<W: Write, R: Read>(
     let mut fl_pos: usize = 0;
     let mut zero_remaining = padding_after;
 
-    let mut adler = simd_adler32::Adler32::new();
-
-    let buf_size = 1024 * 1024;
-    let mut transfer_buf = vec![0u8; buf_size];
-    let zero_buf = vec![0u8; buf_size];
-
     let mut last_png_pct: u64 = 89;
 
     for row_idx in 0..height {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "roxify",
-  "version": "1.13.3",
+  "version": "1.13.4",
   "type": "module",
   "description": "Ultra-lightweight PNG steganography with native Rust acceleration. Encode binary data into PNG images with zstd compression.",
   "main": "dist/index.js",