roxify 1.13.3 → 1.13.5

package/native/streaming_encode.rs

@@ -1,8 +1,11 @@
 use std::io::{Write, BufWriter, Read};
 use std::fs::File;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+use rayon::prelude::*;
+use serde::Serialize;
 use walkdir::WalkDir;
-use tar::{Builder, Header};
+
+use crate::png_chunk_writer::{ChunkedIdatWriter, write_png_chunk};
 
 const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
 const PIXEL_MAGIC: &[u8] = b"PXL1";
@@ -10,9 +13,36 @@ const MARKER_START: [(u8, u8, u8); 3] = [(255, 0, 0), (0, 255, 0), (0, 0, 255)];
 const MARKER_END: [(u8, u8, u8); 3] = [(0, 0, 255), (0, 255, 0), (255, 0, 0)];
 const MARKER_ZSTD: (u8, u8, u8) = (0, 255, 0);
 const MAGIC: &[u8] = b"ROX1";
+const PACK_MAGIC: u32 = 0x524f5850;
+
+const MIN_ZST_CAPACITY: usize = 16 * 1024 * 1024;
+const MB: u64 = 1024 * 1024;
+const MAX_FILE_BUFFER_CAPACITY: usize = 4 * 1024 * 1024;
+const PARALLEL_IO_FILE_THRESHOLD: u64 = MB;
+const PARALLEL_IO_BATCH_BYTES: u64 = 128 * MB;
+const PARALLEL_IO_BATCH_FILES: usize = 512;
+const PARALLEL_IO_MIN_FILES: usize = 8;
+const HEADER_VERSION_V2: u8 = 2;
 
 pub type ProgressCallback = Box<dyn Fn(u64, u64, &str) + Send>;
 
+struct DirectoryFile {
+    path: PathBuf,
+    rel_path: String,
+    size: u64,
+}
+
+#[derive(Serialize)]
+struct FileListEntry {
+    name: String,
+    size: u64,
+}
+
+struct CollectedDirectory {
+    entries: Vec<DirectoryFile>,
+    total_bytes: u64,
+}
+
 pub fn encode_dir_to_png(
     dir_path: &Path,
     output_path: &Path,
@@ -42,9 +72,7 @@ pub fn encode_dir_to_png_encrypted_with_progress(
     encrypt_type: Option<&str>,
     progress: Option<ProgressCallback>,
 ) -> anyhow::Result<()> {
-    let (zst_buf, file_list) = compress_dir_to_zst_mem(dir_path, compression_level, &progress)?;
-
-    let file_list_json = serde_json::to_string(&file_list)?;
+    let (zst_buf, file_list_json) = compress_dir_to_zst_mem(dir_path, compression_level, &progress)?;
 
     let result = write_png_from_zst_mem(
         zst_buf, output_path, name, Some(&file_list_json),
@@ -62,26 +90,19 @@ fn compress_dir_to_zst_mem(
     dir_path: &Path,
     compression_level: i32,
     progress: &Option<ProgressCallback>,
-) -> anyhow::Result<(Vec<u8>, Vec<serde_json::Value>)> {
-    let base = dir_path;
-
-    let entries: Vec<_> = WalkDir::new(dir_path)
-        .follow_links(false)
-        .into_iter()
-        .filter_map(|e| e.ok())
-        .filter(|e| e.file_type().is_file())
-        .collect();
-
-    let total_bytes: u64 = entries.iter()
-        .filter_map(|e| std::fs::metadata(e.path()).ok())
-        .map(|m| m.len())
-        .sum();
+) -> anyhow::Result<(Vec<u8>, String)> {
+    let collected = collect_directory_files(dir_path);
+    let total_bytes = collected.total_bytes;
+    let entries = collected.entries;
 
     let actual_level = compression_level.min(3);
-    let mut encoder = zstd::stream::Encoder::new(Vec::with_capacity(16 * 1024 * 1024), actual_level)
+    let mut encoder = zstd::stream::Encoder::new(
+        Vec::with_capacity(estimate_zst_capacity(total_bytes)),
+        actual_level,
+    )
         .map_err(|e| anyhow::anyhow!("zstd init: {}", e))?;
 
-    let threads = num_cpus::get() as u32;
+    let threads = select_zstd_threads(total_bytes);
     if threads > 1 {
         let _ = encoder.multithread(threads);
     }
@@ -89,61 +110,221 @@ fn compress_dir_to_zst_mem(
     let _ = encoder.window_log(30);
 
     encoder.write_all(MAGIC)?;
+    encoder.write_all(&PACK_MAGIC.to_be_bytes())?;
+    encoder.write_all(&(entries.len() as u32).to_be_bytes())?;
 
-    let mut file_list = Vec::new();
+    let mut file_list = Vec::with_capacity(entries.len());
     let mut bytes_processed: u64 = 0;
     let mut last_pct: u64 = 0;
-    {
-        let mut tar_builder = Builder::new(&mut encoder);
-        for entry in entries.iter() {
-            let full = entry.path();
-            let rel = full.strip_prefix(base).unwrap_or(full);
-            let rel_str = rel.to_string_lossy().replace('\\', "/");
-
-            let metadata = match std::fs::metadata(full) {
-                Ok(m) => m,
-                Err(_) => continue,
-            };
-            let size = metadata.len();
-
-            let mut header = Header::new_gnu();
-            header.set_size(size);
-            header.set_mode(0o644);
-            header.set_cksum();
-
-            let file = match File::open(full) {
-                Ok(f) => f,
-                Err(_) => continue,
-            };
-            let buf_reader = std::io::BufReader::with_capacity(
-                (size as usize).min(4 * 1024 * 1024).max(8192),
-                file,
-            );
-
-            tar_builder.append_data(&mut header, &rel_str, buf_reader)
-                .map_err(|e| anyhow::anyhow!("tar append {}: {}", rel_str, e))?;
-
-            file_list.push(serde_json::json!({"name": rel_str, "size": size}));
-
-            bytes_processed += size;
-            if let Some(ref cb) = progress {
-                let pct = if total_bytes > 0 {
-                    (bytes_processed * 89 / total_bytes).min(89)
-                } else {
-                    89
+    let mut entry_index = 0usize;
+    while entry_index < entries.len() {
+        let batch_end = select_parallel_batch_end(&entries, entry_index);
+        if batch_end > entry_index + 1 {
+            let loaded = load_small_file_batch(&entries[entry_index..batch_end])?;
+            for (entry, maybe_bytes) in entries[entry_index..batch_end].iter().zip(loaded.into_iter()) {
+                let Some(bytes) = maybe_bytes else {
+                    continue;
                 };
-                if pct > last_pct {
-                    last_pct = pct;
-                    cb(pct, 100, "compressing");
-                }
+
+                write_pack_entry_header(&mut encoder, &entry.rel_path, entry.size)?;
+                encoder.write_all(&bytes)
+                    .map_err(|e| anyhow::anyhow!("pack write {}: {}", entry.rel_path, e))?;
+
+                file_list.push(FileListEntry {
+                    name: entry.rel_path.clone(),
+                    size: entry.size,
+                });
+
+                bytes_processed += entry.size;
+                report_compress_progress(progress, total_bytes, bytes_processed, &mut last_pct);
             }
+            entry_index = batch_end;
+            continue;
         }
-        tar_builder.finish().map_err(|e| anyhow::anyhow!("tar finish: {}", e))?;
+
+        let entry = &entries[entry_index];
+        if write_directory_entry(&mut encoder, entry)? {
+            file_list.push(FileListEntry {
+                name: entry.rel_path.clone(),
+                size: entry.size,
+            });
+
+            bytes_processed += entry.size;
+            report_compress_progress(progress, total_bytes, bytes_processed, &mut last_pct);
+        }
+        entry_index += 1;
     }
 
     let zst_buf = encoder.finish().map_err(|e| anyhow::anyhow!("zstd finish: {}", e))?;
+    let file_list_json = serde_json::to_string(&file_list)?;
 
-    Ok((zst_buf, file_list))
+    Ok((zst_buf, file_list_json))
+}
+
+fn write_pack_entry_header<W: Write>(writer: &mut W, rel_path: &str, size: u64) -> anyhow::Result<()> {
+    let name_bytes = rel_path.as_bytes();
+    let name_len = u16::try_from(name_bytes.len())
+        .map_err(|_| anyhow::anyhow!("path too long for pack entry: {}", rel_path))?;
+    writer.write_all(&name_len.to_be_bytes())?;
+    writer.write_all(name_bytes)?;
+    writer.write_all(&size.to_be_bytes())?;
+    Ok(())
+}
+
+fn write_directory_entry<W: Write>(writer: &mut W, entry: &DirectoryFile) -> anyhow::Result<bool> {
+    let file = match File::open(&entry.path) {
+        Ok(file) => file,
+        Err(_) => return Ok(false),
+    };
+
+    write_pack_entry_header(writer, &entry.rel_path, entry.size)?;
+
+    let mut buf_reader = std::io::BufReader::with_capacity(file_buffer_capacity(entry.size), file);
+    std::io::copy(&mut buf_reader, writer)
+        .map_err(|e| anyhow::anyhow!("pack write {}: {}", entry.rel_path, e))?;
+
+    Ok(true)
+}
+
+fn load_small_file_batch(entries: &[DirectoryFile]) -> anyhow::Result<Vec<Option<Vec<u8>>>> {
+    entries.par_iter().map(load_directory_entry_bytes).collect()
+}
+
+fn load_directory_entry_bytes(entry: &DirectoryFile) -> anyhow::Result<Option<Vec<u8>>> {
+    let mut file = match File::open(&entry.path) {
+        Ok(file) => file,
+        Err(_) => return Ok(None),
+    };
+
+    let reserve = usize::try_from(entry.size.min(PARALLEL_IO_BATCH_BYTES)).unwrap_or(MAX_FILE_BUFFER_CAPACITY);
+    let mut bytes = Vec::with_capacity(reserve.max(8192));
+    file.read_to_end(&mut bytes)
+        .map_err(|e| anyhow::anyhow!("pack read {}: {}", entry.rel_path, e))?;
+
+    Ok(Some(bytes))
+}
+
+fn select_parallel_batch_end(entries: &[DirectoryFile], start: usize) -> usize {
+    let Some(first) = entries.get(start) else {
+        return start;
+    };
+    if !should_parallelize_entry(first) {
+        return start + 1;
+    }
+
+    let mut end = start;
+    let mut batch_bytes = 0u64;
+    while end < entries.len() {
+        let entry = &entries[end];
+        if !should_parallelize_entry(entry) {
+            break;
+        }
+        if end > start {
+            if end - start >= PARALLEL_IO_BATCH_FILES {
+                break;
+            }
+            if batch_bytes.saturating_add(entry.size) > PARALLEL_IO_BATCH_BYTES {
+                break;
+            }
+        }
+        batch_bytes = batch_bytes.saturating_add(entry.size);
+        end += 1;
+    }
+
+    if end - start >= PARALLEL_IO_MIN_FILES {
+        end
+    } else {
+        start + 1
+    }
+}
+
+fn should_parallelize_entry(entry: &DirectoryFile) -> bool {
+    entry.size <= PARALLEL_IO_FILE_THRESHOLD
+}
+
+fn file_buffer_capacity(size: u64) -> usize {
+    usize::try_from(size)
+        .unwrap_or(MAX_FILE_BUFFER_CAPACITY)
+        .min(MAX_FILE_BUFFER_CAPACITY)
+        .max(8192)
+}
+
+fn report_compress_progress(
+    progress: &Option<ProgressCallback>,
+    total_bytes: u64,
+    bytes_processed: u64,
+    last_pct: &mut u64,
+) {
+    if let Some(ref cb) = progress {
+        let pct = if total_bytes > 0 {
+            (bytes_processed * 89 / total_bytes).min(89)
+        } else {
+            89
+        };
+        if pct > *last_pct {
+            *last_pct = pct;
+            cb(pct, 100, "compressing");
+        }
+    }
+}
+
+fn collect_directory_files(dir_path: &Path) -> CollectedDirectory {
+    let mut entries = Vec::new();
+    let mut total_bytes = 0u64;
+
+    for entry in WalkDir::new(dir_path)
+        .follow_links(false)
+        .into_iter()
+        .filter_map(|entry| entry.ok())
+        .filter(|entry| entry.file_type().is_file())
+    {
+        let size = match entry.metadata() {
+            Ok(metadata) => metadata.len(),
+            Err(_) => continue,
+        };
+        let path = entry.into_path();
+        let rel = path.strip_prefix(dir_path).unwrap_or(path.as_path());
+        let rel_path = normalize_rel_path(rel);
+
+        total_bytes += size;
+        entries.push(DirectoryFile {
+            path,
+            rel_path,
+            size,
+        });
+    }
+
+    CollectedDirectory {
+        entries,
+        total_bytes,
+    }
+}
+
+fn normalize_rel_path(path: &Path) -> String {
+    let rel_path = path.to_string_lossy();
+    if rel_path.contains('\\') {
+        rel_path.replace('\\', "/")
+    } else {
+        rel_path.into_owned()
+    }
+}
+
+fn estimate_zst_capacity(total_bytes: u64) -> usize {
+    let capped = total_bytes.min(usize::MAX as u64) as usize;
+    (capped / 3).max(MIN_ZST_CAPACITY)
+}
+
+fn select_zstd_threads(total_bytes: u64) -> u32 {
+    let max_threads = num_cpus::get().max(1) as u32;
+    if total_bytes <= 32 * MB {
+        1
+    } else if total_bytes <= 128 * MB {
+        max_threads.min(2)
+    } else if total_bytes <= 512 * MB {
+        max_threads.min(4)
+    } else {
+        max_threads.min(8)
+    }
 }
 
 fn write_png_from_zst_mem(
@@ -167,12 +348,12 @@ fn write_png_from_zst_mem(
 
     let encrypted_payload_len = enc_header_len + zst_size + hmac_trailer_len;
 
-    let version = 1u8;
+    let version = HEADER_VERSION_V2;
     let name_bytes = name.map(|n| n.as_bytes()).unwrap_or(&[]);
     let name_len = name_bytes.len().min(255) as u8;
-    let payload_len_bytes = (encrypted_payload_len as u32).to_be_bytes();
+    let payload_len_bytes = (encrypted_payload_len as u64).to_be_bytes();
 
-    let mut meta_header = Vec::with_capacity(1 + 1 + name_len as usize + 4);
+    let mut meta_header = Vec::with_capacity(1 + 1 + name_len as usize + 8);
     meta_header.push(version);
     meta_header.push(name_len);
     if name_len > 0 {
@@ -220,13 +401,6 @@ fn write_png_from_zst_mem(
 
     let header_bytes = build_header_bytes(&meta_header, &enc_header_bytes);
 
-    let stride = row_bytes + 1;
-    let scanlines_total = height * stride;
-
-    const MAX_BLOCK: usize = 65535;
-    let num_blocks = (scanlines_total + MAX_BLOCK - 1) / MAX_BLOCK;
-    let idat_len = 2 + num_blocks * 5 + scanlines_total + 4;
-
     let out_file = File::create(output_path)?;
     let buf_capacity = if total_data_bytes > 256 * 1024 * 1024 { 16 * 1024 * 1024 }
         else if total_data_bytes > 16 * 1024 * 1024 { 8 * 1024 * 1024 }
@@ -240,7 +414,7 @@ fn write_png_from_zst_mem(
     ihdr[4..8].copy_from_slice(&(height as u32).to_be_bytes());
     ihdr[8] = 8;
     ihdr[9] = 2;
-    write_chunk_hdr(&mut w, b"IHDR", &ihdr)?;
+    write_png_chunk(&mut w, b"IHDR", &ihdr)?;
 
     let mut zst_reader = std::io::Cursor::new(zst_buf);
 
@@ -255,15 +429,14 @@ fn write_png_from_zst_mem(
         height,
         row_bytes,
         marker_end_pos,
-        idat_len,
         total_data_bytes,
         progress,
     )?;
 
     if let Some(fl) = file_list {
-        write_chunk_hdr(&mut w, b"rXFL", fl.as_bytes())?;
+        write_png_chunk(&mut w, b"rXFL", fl.as_bytes())?;
     }
-    write_chunk_hdr(&mut w, b"IEND", &[])?;
+    write_png_chunk(&mut w, b"IEND", &[])?;
     w.flush()?;
 
     Ok(())
@@ -292,33 +465,32 @@ fn write_idat_streaming<W: Write, R: Read>(
     height: usize,
     row_bytes: usize,
     marker_end_pos: usize,
-    idat_len: usize,
     total_data_bytes: usize,
     progress: &Option<ProgressCallback>,
 ) -> anyhow::Result<()> {
-    w.write_all(&(idat_len as u32).to_be_bytes())?;
-    w.write_all(b"IDAT")?;
-
-    let mut crc = crc32fast::Hasher::new();
-    crc.update(b"IDAT");
+    let mut idat = ChunkedIdatWriter::new(w);
 
     let stride = row_bytes + 1;
     let scanlines_total = height * stride;
 
     let zlib = [0x78u8, 0x01];
-    w.write_all(&zlib)?;
-    crc.update(&zlib);
+    idat.write_all(&zlib)?;
 
     let fl_chunk_data = file_list_chunk.unwrap_or(&[]);
     let payload_total = header_bytes.len() + zst_size + hmac_trailer_len + fl_chunk_data.len();
     let padding_after = total_data_bytes - payload_total.min(total_data_bytes);
-
     let marker_end_bytes = build_marker_end_bytes();
 
     let mut flat_pos: usize = 0;
     let mut scanline_pos: usize = 0;
     let mut deflate_block_remaining: usize = 0;
 
+    let mut adler = simd_adler32::Adler32::new();
+
+    let buf_size = 1024 * 1024;
+    let mut transfer_buf = vec![0u8; buf_size];
+    let zero_buf = vec![0u8; buf_size];
+
     let mut header_pos: usize = 0;
     let mut zst_remaining = zst_size;
     let mut hmac_pos: usize = 0;
@@ -327,12 +499,6 @@ fn write_idat_streaming<W: Write, R: Read>(
     let mut fl_pos: usize = 0;
     let mut zero_remaining = padding_after;
 
-    let mut adler = simd_adler32::Adler32::new();
-
-    let buf_size = 1024 * 1024;
-    let mut transfer_buf = vec![0u8; buf_size];
-    let zero_buf = vec![0u8; buf_size];
-
     let mut last_png_pct: u64 = 89;
 
     for row_idx in 0..height {
@@ -347,14 +513,12 @@ fn write_idat_streaming<W: Write, R: Read>(
                 !block_size as u8,
                 (!(block_size >> 8)) as u8,
             ];
-            w.write_all(&header)?;
-            crc.update(&header);
+            idat.write_all(&header)?;
             deflate_block_remaining = block_size;
         }
 
         let filter_byte = [0u8];
-        w.write_all(&filter_byte)?;
-        crc.update(&filter_byte);
+        idat.write_all(&filter_byte)?;
         adler.write(&filter_byte);
         scanline_pos += 1;
         deflate_block_remaining -= 1;
@@ -372,8 +536,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                     !block_size as u8,
                     (!(block_size >> 8)) as u8,
                 ];
-                w.write_all(&header)?;
-                crc.update(&header);
+                idat.write_all(&header)?;
                 deflate_block_remaining = block_size;
             }
 
@@ -390,8 +553,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                     let me_remaining = 9 - me_offset;
                     let take = need.min(me_remaining);
                     let slice = &marker_end_bytes[me_offset..me_offset + take];
-                    w.write_all(slice)?;
-                    crc.update(slice);
+                    idat.write_all(slice)?;
                     adler.write(slice);
                     flat_pos += take;
                     chunk_written += take;
@@ -405,8 +567,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                     let avail = header_bytes.len() - header_pos;
                     let take = need.min(avail);
                     let slice = &header_bytes[header_pos..header_pos + take];
-                    w.write_all(slice)?;
-                    crc.update(slice);
+                    idat.write_all(slice)?;
                     adler.write(slice);
                     header_pos += take;
                     flat_pos += take;
@@ -422,8 +583,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                     if let Some(ref mut enc) = encryptor {
                         enc.encrypt_chunk(&mut transfer_buf[..got]);
                     }
-                    w.write_all(&transfer_buf[..got])?;
-                    crc.update(&transfer_buf[..got]);
+                    idat.write_all(&transfer_buf[..got])?;
                     adler.write(&transfer_buf[..got]);
                     zst_remaining -= got;
                     flat_pos += got;
@@ -441,8 +601,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                         let avail = hmac_trailer_len - hmac_pos;
                         let take = need.min(avail);
                         let slice = &hmac_bytes[hmac_pos..hmac_pos + take];
-                        w.write_all(slice)?;
-                        crc.update(slice);
+                        idat.write_all(slice)?;
                         adler.write(slice);
                         hmac_pos += take;
                         flat_pos += take;
@@ -460,8 +619,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                     let avail = fl_chunk_data.len() - fl_pos;
                     let take = need.min(avail);
                     let slice = &fl_chunk_data[fl_pos..fl_pos + take];
-                    w.write_all(slice)?;
-                    crc.update(slice);
+                    idat.write_all(slice)?;
                     adler.write(slice);
                     fl_pos += take;
                     flat_pos += take;
@@ -477,8 +635,7 @@ fn write_idat_streaming<W: Write, R: Read>(
                     };
                     let take = need.min(zero_remaining).min(buf_size).min(max_before_marker);
                     if take == 0 { break; }
-                    w.write_all(&zero_buf[..take])?;
-                    crc.update(&zero_buf[..take]);
+                    idat.write_all(&zero_buf[..take])?;
                     adler.write(&zero_buf[..take]);
                     zero_remaining -= take;
                     flat_pos += take;
@@ -501,11 +658,8 @@ fn write_idat_streaming<W: Write, R: Read>(
 
     let adler_val = adler.finish();
     let adler_bytes = adler_val.to_be_bytes();
-    w.write_all(&adler_bytes)?;
-    crc.update(&adler_bytes);
-
-    w.write_all(&crc.finalize().to_be_bytes())?;
-    Ok(())
+    idat.write_all(&adler_bytes)?;
+    idat.finish()
 }
 
 fn build_marker_end_bytes() -> [u8; 9] {
@@ -518,13 +672,3 @@ fn build_marker_end_bytes() -> [u8; 9] {
     buf
 }
 
-fn write_chunk_hdr<W: Write>(w: &mut W, chunk_type: &[u8; 4], data: &[u8]) -> anyhow::Result<()> {
-    w.write_all(&(data.len() as u32).to_be_bytes())?;
-    w.write_all(chunk_type)?;
-    w.write_all(data)?;
-    let mut h = crc32fast::Hasher::new();
-    h.update(chunk_type);
-    h.update(data);
-    w.write_all(&h.finalize().to_be_bytes())?;
-    Ok(())
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "roxify",
-  "version": "1.13.3",
+  "version": "1.13.5",
   "type": "module",
   "description": "Ultra-lightweight PNG steganography with native Rust acceleration. Encode binary data into PNG images with zstd compression.",
   "main": "dist/index.js",