roxify 1.11.0 → 1.11.1

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.11.0"
+version = "1.11.1"
 edition = "2021"
 publish = false
 

package/native/archive.rs

@@ -4,7 +4,12 @@ use rayon::prelude::*;
 use tar::{Archive, Builder, Header};
 use walkdir::WalkDir;
 
-pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+pub struct TarPackResult {
+    pub data: Vec<u8>,
+    pub file_list: Vec<(String, u64)>,
+}
+
+pub fn tar_pack_directory_with_list(dir_path: &Path) -> Result<TarPackResult, String> {
     let base = dir_path;
 
     let entries: Vec<_> = WalkDir::new(dir_path)
@@ -27,6 +32,10 @@ pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
         })
         .collect();
 
+    let file_list: Vec<(String, u64)> = file_data.iter()
+        .map(|(name, data)| (name.clone(), data.len() as u64))
+        .collect();
+
     let total_estimate: usize = file_data.iter().map(|(n, d)| 512 + d.len() + 512 + n.len()).sum();
     let mut buf = Vec::with_capacity(total_estimate + 1024);
     {
@@ -42,20 +51,42 @@ pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
         }
         builder.finish().map_err(|e| format!("tar finish: {}", e))?;
     }
-    Ok(buf)
+    Ok(TarPackResult { data: buf, file_list })
+}
+
+pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+    tar_pack_directory_with_list(dir_path).map(|r| r.data)
+}
+
+pub fn tar_file_list_fast(tar_data: &[u8]) -> Vec<(String, u64)> {
+    let mut list = Vec::new();
+    let mut pos = 0;
+    while pos + 512 <= tar_data.len() {
+        let header = &tar_data[pos..pos + 512];
+        if header.iter().all(|&b| b == 0) {
+            break;
+        }
+        let name_end = header[..100].iter().position(|&b| b == 0).unwrap_or(100);
+        let name = String::from_utf8_lossy(&header[..name_end]).to_string();
+        let size_str = String::from_utf8_lossy(&header[124..136]);
+        let size = u64::from_str_radix(size_str.trim().trim_matches('\0'), 8).unwrap_or(0);
+        if !name.is_empty() {
+            list.push((name, size));
+        }
+        let data_blocks = (size as usize + 511) / 512;
+        pos += 512 + data_blocks * 512;
+    }
+    list
 }
 
 pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, String> {
     let mut archive = Archive::new(Cursor::new(tar_data));
-    let mut written = Vec::new();
+    let mut entries_data: Vec<(std::path::PathBuf, Vec<u8>)> = Vec::new();
 
     let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
     for entry in entries {
         let mut entry = entry.map_err(|e| format!("tar entry: {}", e))?;
-        let path = entry
-            .path()
-            .map_err(|e| format!("tar entry path: {}", e))?
-            .to_path_buf();
+        let path = entry.path().map_err(|e| format!("tar entry path: {}", e))?.to_path_buf();
 
         let mut safe = std::path::PathBuf::new();
         for comp in path.components() {
@@ -63,23 +94,36 @@ pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, Str
                 safe.push(osstr);
             }
         }
-
         if safe.as_os_str().is_empty() {
             continue;
         }
 
-        let dest = output_dir.join(&safe);
-        if let Some(parent) = dest.parent() {
-            std::fs::create_dir_all(parent)
-                .map_err(|e| format!("mkdir {:?}: {}", parent, e))?;
-        }
+        let mut data = Vec::with_capacity(entry.size() as usize);
+        std::io::Read::read_to_end(&mut entry, &mut data)
+            .map_err(|e| format!("tar read {:?}: {}", safe, e))?;
+        entries_data.push((safe, data));
+    }
 
-        entry
-            .unpack(&dest)
-            .map_err(|e| format!("tar unpack {:?}: {}", dest, e))?;
-        written.push(safe.to_string_lossy().to_string());
+    let dirs: std::collections::HashSet<_> = entries_data.iter()
+        .filter_map(|(p, _)| {
+            let dest = output_dir.join(p);
+            dest.parent().map(|d| d.to_path_buf())
+        })
+        .collect();
+    for dir in &dirs {
+        std::fs::create_dir_all(dir).map_err(|e| format!("mkdir {:?}: {}", dir, e))?;
     }
 
+    let written: Vec<String> = entries_data.par_iter()
+        .filter_map(|(safe, data)| {
+            let dest = output_dir.join(safe);
+            match std::fs::write(&dest, data) {
+                Ok(_) => Some(safe.to_string_lossy().to_string()),
+                Err(_) => None,
+            }
+        })
+        .collect();
+
     Ok(written)
 }
 

package/native/core.rs

@@ -85,15 +85,55 @@ pub fn crc32_bytes(buf: &[u8]) -> u32 {
 
 pub fn adler32_bytes(buf: &[u8]) -> u32 {
     const MOD: u32 = 65521;
+    const NMAX: usize = 5552;
+
+    if buf.len() > 4 * 1024 * 1024 {
+        return adler32_parallel(buf);
+    }
+
     let mut a: u32 = 1;
     let mut b: u32 = 0;
-    for &v in buf {
-        a = (a + v as u32) % MOD;
-        b = (b + a) % MOD;
+
+    for chunk in buf.chunks(NMAX) {
+        for &v in chunk {
+            a += v as u32;
+            b += a;
+        }
+        a %= MOD;
+        b %= MOD;
     }
+
     (b << 16) | a
 }
 
+fn adler32_parallel(buf: &[u8]) -> u32 {
+    use rayon::prelude::*;
+    const MOD: u32 = 65521;
+    const CHUNK: usize = 1024 * 1024;
+
+    let chunks: Vec<&[u8]> = buf.chunks(CHUNK).collect();
+    let partials: Vec<(u32, u32, usize)> = chunks.par_iter().map(|chunk| {
+        let mut a: u32 = 0;
+        let mut b: u32 = 0;
+        for &v in *chunk {
+            a += v as u32;
+            b += a;
+        }
+        a %= MOD;
+        b %= MOD;
+        (a, b, chunk.len())
+    }).collect();
+
+    let mut a: u64 = 1;
+    let mut b: u64 = 0;
+    for (pa, pb, len) in partials {
+        b = (b + pb as u64 + a * len as u64) % MOD as u64;
+        a = (a + pa as u64) % MOD as u64;
+    }
+
+    ((b as u32) << 16) | (a as u32)
+}
+
 pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {
     let len = buf.len();
     if len == 0 {
@@ -173,40 +213,85 @@ pub fn train_zstd_dictionary(sample_paths: &[PathBuf], dict_size: usize) -> Resu
 /// For large buffers (>50 MiB) without a dictionary, multiple chunk sizes
 /// are benchmarked on a sample and the best is selected automatically.
 pub fn zstd_compress_bytes(buf: &[u8], level: i32, dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
+    zstd_compress_with_prefix(buf, level, dict, &[])
+}
+
+pub fn zstd_compress_with_prefix(buf: &[u8], level: i32, dict: Option<&[u8]>, prefix: &[u8]) -> std::result::Result<Vec<u8>, String> {
     use std::io::Write;
 
     let actual_level = level.min(22).max(1);
+    let total_len = prefix.len() + buf.len();
+
+    let adaptive_level = if total_len > 2 * 1024 * 1024 * 1024 {
+        actual_level.min(1)
+    } else if total_len > 1024 * 1024 * 1024 {
+        actual_level.min(3)
+    } else if total_len > 256 * 1024 * 1024 {
+        actual_level.min(6)
+    } else if total_len > 64 * 1024 * 1024 {
+        actual_level.min(12)
+    } else {
+        actual_level
+    };
+
+    if dict.is_none() && total_len < 4 * 1024 * 1024 {
+        if prefix.is_empty() {
+            return zstd::bulk::compress(buf, adaptive_level)
+                .map_err(|e| format!("zstd bulk compress error: {}", e));
+        }
+        let mut combined = Vec::with_capacity(total_len);
+        combined.extend_from_slice(prefix);
+        combined.extend_from_slice(buf);
+        return zstd::bulk::compress(&combined, adaptive_level)
+            .map_err(|e| format!("zstd bulk compress error: {}", e));
+    }
+
     let mut encoder = if let Some(d) = dict {
-        zstd::stream::Encoder::with_dictionary(Vec::new(), actual_level, d)
+        zstd::stream::Encoder::with_dictionary(Vec::with_capacity(total_len / 2), adaptive_level, d)
             .map_err(|e| format!("zstd encoder init error: {}", e))?
     } else {
-        zstd::stream::Encoder::new(Vec::new(), actual_level)
+        zstd::stream::Encoder::new(Vec::with_capacity(total_len / 2), adaptive_level)
             .map_err(|e| format!("zstd encoder init error: {}", e))?
     };
 
     let threads = num_cpus::get() as u32;
     if threads > 1 {
-        let max_threads = if actual_level >= 20 { threads.min(4) } else { threads };
+        let max_threads = if adaptive_level >= 20 { threads.min(4) } else { threads };
         let _ = encoder.multithread(max_threads);
     }
 
-    if buf.len() > 1024 * 1024 {
+    if total_len > 256 * 1024 && adaptive_level >= 3 {
         let _ = encoder.long_distance_matching(true);
-        let wlog = if buf.len() > 512 * 1024 * 1024 { 28 }
-            else if buf.len() > 64 * 1024 * 1024 { 27 }
+    }
+    if total_len > 256 * 1024 {
+        let wlog = if total_len > 1024 * 1024 * 1024 { 30 }
+            else if total_len > 512 * 1024 * 1024 { 29 }
+            else if total_len > 64 * 1024 * 1024 { 28 }
+            else if total_len > 8 * 1024 * 1024 { 27 }
             else { 26 };
         let _ = encoder.window_log(wlog);
     }
 
-    let _ = encoder.set_pledged_src_size(Some(buf.len() as u64));
+    let _ = encoder.set_pledged_src_size(Some(total_len as u64));
+
+    if !prefix.is_empty() {
+        encoder.write_all(prefix).map_err(|e| format!("zstd write prefix error: {}", e))?;
+    }
+
+    let chunk_size = if total_len > 256 * 1024 * 1024 { 16 * 1024 * 1024 }
+        else if total_len > 64 * 1024 * 1024 { 8 * 1024 * 1024 }
+        else { buf.len() };
+
+    for chunk in buf.chunks(chunk_size) {
+        encoder.write_all(chunk).map_err(|e| format!("zstd write error: {}", e))?;
+    }
 
-    encoder.write_all(buf).map_err(|e| format!("zstd write error: {}", e))?;
     encoder.finish().map_err(|e| format!("zstd finish error: {}", e))
 }
 
 pub fn zstd_decompress_bytes(buf: &[u8], dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
     use std::io::Read;
-    let mut out = Vec::new();
+    let mut out = Vec::with_capacity(buf.len() * 2);
     if let Some(d) = dict {
         let mut decoder = zstd::stream::Decoder::with_dictionary(std::io::Cursor::new(buf), d)
             .map_err(|e| format!("zstd decoder init error: {}", e))?;

package/native/crypto.rs

@@ -10,7 +10,7 @@ use sha2::Sha256;
 const ENC_NONE: u8 = 0x00;
 const ENC_AES: u8 = 0x01;
 const ENC_XOR: u8 = 0x02;
-const PBKDF2_ITERS: u32 = 1_000_000;
+const PBKDF2_ITERS: u32 = 600_000;
 
 pub fn encrypt_xor(data: &[u8], passphrase: &str) -> Vec<u8> {
     let key = passphrase.as_bytes();

package/native/encoder.rs

@@ -2,7 +2,6 @@ use anyhow::Result;
 use std::process::{Command, Stdio};
 
 const MAGIC: &[u8] = b"ROX1";
-const ENC_NONE: u8 = 0x00;
 const PIXEL_MAGIC: &[u8] = b"PXL1";
 const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
 
@@ -112,9 +111,7 @@ pub fn encode_to_wav_with_encryption_name_and_filelist(
     file_list: Option<&str>,
 ) -> Result<Vec<u8>> {
     // Same compression + encryption pipeline as PNG
-    let payload_input = [MAGIC, data].concat();
-
-    let compressed = crate::core::zstd_compress_bytes(&payload_input, compression_level, None)
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, None, MAGIC)
         .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
 
     let encrypted = if let Some(pass) = passphrase {
@@ -180,9 +177,7 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
     file_list: Option<&str>,
     dict: Option<&[u8]>,
 ) -> Result<Vec<u8>> {
-    let payload_input = [MAGIC, data].concat();
-
-    let compressed = crate::core::zstd_compress_bytes(&payload_input, compression_level, dict)
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, dict, MAGIC)
         .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
 
     let encrypted = if let Some(pass) = passphrase {
@@ -194,65 +189,73 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
     } else {
         crate::crypto::no_encryption(&compressed)
     };
+    drop(compressed);
 
     let meta_pixel = build_meta_pixel_with_name_and_filelist(&encrypted, name, file_list)?;
-    let data_without_markers = [PIXEL_MAGIC, &meta_pixel].concat();
-
-    let padding_needed = (3 - (data_without_markers.len() % 3)) % 3;
-    let padded_data = if padding_needed > 0 {
-        [&data_without_markers[..], &vec![0u8; padding_needed]].concat()
-    } else {
-        data_without_markers
-    };
+    drop(encrypted);
 
-    let mut marker_bytes = Vec::with_capacity(12);
-    for m in &MARKER_START {
-        marker_bytes.extend_from_slice(&[m.0, m.1, m.2]);
-    }
-    marker_bytes.extend_from_slice(&[MARKER_ZSTD.0, MARKER_ZSTD.1, MARKER_ZSTD.2]);
+    let raw_payload_len = PIXEL_MAGIC.len() + meta_pixel.len();
+    let padding_needed = (3 - (raw_payload_len % 3)) % 3;
+    let padded_len = raw_payload_len + padding_needed;
 
-    let data_with_markers = [&marker_bytes[..], &padded_data[..]].concat();
+    let marker_start_len = 12;
+    let marker_end_len = 9;
 
-    let mut marker_end_bytes = Vec::with_capacity(9);
-    for m in &MARKER_END {
-        marker_end_bytes.extend_from_slice(&[m.0, m.1, m.2]);
-    }
-
-    let data_pixels = (data_with_markers.len() + 2) / 3;
+    let data_with_markers_len = marker_start_len + padded_len;
+    let data_pixels = (data_with_markers_len + 2) / 3;
     let end_marker_pixels = 3;
     let total_pixels = data_pixels + end_marker_pixels;
 
     let side = (total_pixels as f64).sqrt().ceil() as usize;
     let side = side.max(end_marker_pixels);
-
     let width = side;
     let height = side;
 
     let total_data_bytes = width * height * 3;
-    let mut full_data = vec![0u8; total_data_bytes];
+    let marker_end_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
 
-    let marker_start_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
+    let flat = build_flat_pixel_buffer(&meta_pixel, padding_needed, marker_end_pos, marker_end_len, total_data_bytes);
+    drop(meta_pixel);
 
-    let copy_len = data_with_markers.len().min(marker_start_pos);
-    full_data[..copy_len].copy_from_slice(&data_with_markers[..copy_len]);
+    let row_bytes = width * 3;
+    let idat_data = create_raw_deflate_from_rows(&flat, row_bytes, height);
+    drop(flat);
 
-    let end_len = marker_end_bytes.len().min(total_data_bytes - marker_start_pos);
-    full_data[marker_start_pos..marker_start_pos + end_len]
-        .copy_from_slice(&marker_end_bytes[..end_len]);
-
-    let stride = width * 3 + 1;
-    let mut scanlines = Vec::with_capacity(height * stride);
+    build_png(width, height, &idat_data, file_list)
+}
 
-    for row in 0..height {
-        scanlines.push(0u8);
-        let src_start = row * width * 3;
-        let src_end = (row + 1) * width * 3;
-        scanlines.extend_from_slice(&full_data[src_start..src_end]);
+fn build_flat_pixel_buffer(
+    meta_pixel: &[u8],
+    padding_needed: usize,
+    marker_end_pos: usize,
+    marker_end_len: usize,
+    total_data_bytes: usize,
+) -> Vec<u8> {
+    let header_len = 12 + PIXEL_MAGIC.len() + meta_pixel.len() + padding_needed;
+    let mut flat = vec![0u8; total_data_bytes];
+
+    let mut pos = 0;
+    for m in &MARKER_START {
+        flat[pos] = m.0; flat[pos + 1] = m.1; flat[pos + 2] = m.2;
+        pos += 3;
     }
+    flat[pos] = MARKER_ZSTD.0; flat[pos + 1] = MARKER_ZSTD.1; flat[pos + 2] = MARKER_ZSTD.2;
+    pos += 3;
 
-    let idat_data = create_raw_deflate(&scanlines);
+    flat[pos..pos + PIXEL_MAGIC.len()].copy_from_slice(PIXEL_MAGIC);
+    pos += PIXEL_MAGIC.len();
 
-    build_png(width, height, &idat_data, file_list)
+    flat[pos..pos + meta_pixel.len()].copy_from_slice(meta_pixel);
+    pos += meta_pixel.len();
+
+    if marker_end_pos + marker_end_len <= total_data_bytes {
+        for (i, m) in MARKER_END.iter().enumerate() {
+            let off = marker_end_pos + i * 3;
+            flat[off] = m.0; flat[off + 1] = m.1; flat[off + 2] = m.2;
+        }
+    }
+
+    flat
 }
 
 fn build_meta_pixel(payload: &[u8]) -> Result<Vec<u8>> {
@@ -323,33 +326,37 @@ fn write_chunk(out: &mut Vec<u8>, chunk_type: &[u8; 4], data: &[u8]) -> Result<(
     out.extend_from_slice(chunk_type);
     out.extend_from_slice(data);
 
-    let mut crc_data = Vec::with_capacity(chunk_type.len() + data.len());
-    crc_data.extend_from_slice(chunk_type);
-    crc_data.extend_from_slice(data);
-    let crc = crate::core::crc32_bytes(&crc_data);
+    let mut hasher = crc32fast::Hasher::new();
+    hasher.update(chunk_type);
+    hasher.update(data);
+    let crc = hasher.finalize();
 
     out.extend_from_slice(&crc.to_be_bytes());
     Ok(())
 }
 
 fn create_raw_deflate(data: &[u8]) -> Vec<u8> {
-    let mut result = Vec::with_capacity(data.len() + 6 + (data.len() / 65535 + 1) * 5);
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (data.len() + MAX_BLOCK - 1) / MAX_BLOCK;
+    let total_size = 2 + num_blocks * 5 + data.len() + 4;
+    let mut result = Vec::with_capacity(total_size);
 
     result.push(0x78);
     result.push(0x01);
 
     let mut offset = 0;
     while offset < data.len() {
-        let chunk_size = (data.len() - offset).min(65535);
+        let chunk_size = (data.len() - offset).min(MAX_BLOCK);
         let is_last = offset + chunk_size >= data.len();
 
-        result.push(if is_last { 0x01 } else { 0x00 });
-
-        result.push(chunk_size as u8);
-        result.push((chunk_size >> 8) as u8);
-        result.push(!chunk_size as u8);
-        result.push((!(chunk_size >> 8)) as u8);
-
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        result.extend_from_slice(&header);
         result.extend_from_slice(&data[offset..offset + chunk_size]);
         offset += chunk_size;
     }
@@ -360,6 +367,51 @@ fn create_raw_deflate(data: &[u8]) -> Vec<u8> {
     result
 }
 
+fn create_raw_deflate_from_rows(flat: &[u8], row_bytes: usize, height: usize) -> Vec<u8> {
+    let stride = row_bytes + 1;
+    let scanlines_total = height * stride;
+
+    let mut scanlines = vec![0u8; scanlines_total];
+    for row in 0..height {
+        let flat_start = row * row_bytes;
+        let flat_end = (flat_start + row_bytes).min(flat.len());
+        let copy_len = flat_end.saturating_sub(flat_start);
+        if copy_len > 0 {
+            let dst_start = row * stride + 1;
+            scanlines[dst_start..dst_start + copy_len].copy_from_slice(&flat[flat_start..flat_end]);
+        }
+    }
+
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (scanlines_total + MAX_BLOCK - 1) / MAX_BLOCK;
+    let total_size = 2 + num_blocks * 5 + scanlines_total + 4;
+    let mut result = Vec::with_capacity(total_size);
+
+    result.push(0x78);
+    result.push(0x01);
+
+    let mut offset = 0;
+    while offset < scanlines.len() {
+        let chunk_size = (scanlines.len() - offset).min(MAX_BLOCK);
+        let is_last = offset + chunk_size >= scanlines.len();
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        result.extend_from_slice(&header);
+        result.extend_from_slice(&scanlines[offset..offset + chunk_size]);
+        offset += chunk_size;
+    }
+
+    let adler = crate::core::adler32_bytes(&scanlines);
+    result.extend_from_slice(&adler.to_be_bytes());
+
+    result
+}
+
 fn predict_best_format(data: &[u8]) -> ImageFormat {
     if data.len() < 2048 {
         return ImageFormat::Png;

package/native/hybrid.rs

@@ -5,7 +5,7 @@ use crate::mtf::{mtf_encode, mtf_decode, rle0_encode, rle0_decode};
 use crate::rans_byte::{SymbolStats, rans_encode_block, rans_decode_block};
 use crate::context_mixing::analyze_entropy;
 
-const BLOCK_SIZE: usize = 256 * 1024;
+const BLOCK_SIZE: usize = 1024 * 1024;
 
 const BLOCK_FLAG_BWT: u8 = 0;
 const BLOCK_FLAG_ZSTD: u8 = 1;

package/native/lib.rs

@@ -24,6 +24,7 @@ mod audio;
 mod progress;
 mod reconstitution;
 mod archive;
+mod streaming;
 
 pub use core::*;
 #[cfg(feature = "gpu")]

package/native/main.rs

@@ -14,6 +14,7 @@ mod png_utils;
 mod audio;
 mod reconstitution;
 mod archive;
+mod streaming;
 
 use crate::encoder::ImageFormat;
 use std::path::PathBuf;
@@ -104,15 +105,28 @@ enum Commands {
 }
 
 fn read_all(path: &PathBuf) -> anyhow::Result<Vec<u8>> {
-    let mut f = File::open(path)?;
-    let mut buf = Vec::new();
-    f.read_to_end(&mut buf)?;
-    Ok(buf)
+    let metadata = std::fs::metadata(path)?;
+    let size = metadata.len() as usize;
+
+    if size > 256 * 1024 * 1024 {
+        let file = File::open(path)?;
+        let mmap = unsafe { memmap2::Mmap::map(&file)? };
+        Ok(mmap.to_vec())
+    } else {
+        let mut f = File::open(path)?;
+        let mut buf = Vec::with_capacity(size);
+        f.read_to_end(&mut buf)?;
+        Ok(buf)
+    }
 }
 
 fn write_all(path: &PathBuf, data: &[u8]) -> anyhow::Result<()> {
-    let mut f = File::create(path)?;
-    f.write_all(data)?;
+    let f = File::create(path)?;
+    let buf_size = if data.len() > 64 * 1024 * 1024 { 16 * 1024 * 1024 }
+        else { (8 * 1024 * 1024).min(data.len().max(8192)) };
+    let mut writer = std::io::BufWriter::with_capacity(buf_size, f);
+    writer.write_all(data)?;
+    writer.flush()?;
     Ok(())
 }
 
@@ -144,14 +158,12 @@ fn main() -> anyhow::Result<()> {
         Commands::Encode { input, output, level, passphrase, encrypt, name, dict } => {
             let is_dir = input.is_dir();
             let (payload, file_list_json) = if is_dir {
-                let tar_data = archive::tar_pack_directory(&input)
+                let result = archive::tar_pack_directory_with_list(&input)
                     .map_err(|e| anyhow::anyhow!(e))?;
-                let list = archive::tar_file_list(&tar_data)
-                    .map_err(|e| anyhow::anyhow!(e))?;
-                let json_list: Vec<serde_json::Value> = list.iter()
+                let json_list: Vec<serde_json::Value> = result.file_list.iter()
                     .map(|(name, size)| serde_json::json!({"name": name, "size": size}))
                     .collect();
-                (tar_data, Some(serde_json::to_string(&json_list)?))
+                (result.data, Some(serde_json::to_string(&json_list)?))
             } else {
                 let pack_result = packer::pack_path_with_metadata(&input)?;
                 (pack_result.data, pack_result.file_list_json)
@@ -165,31 +177,45 @@ fn main() -> anyhow::Result<()> {
                 None => None,
             };
 
-            let png = if let Some(ref pass) = passphrase {
-                encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+            let use_streaming = payload.len() > 64 * 1024 * 1024;
+
+            if use_streaming {
+                streaming::encode_to_png_file(
                     &payload,
+                    &output,
                     level,
-                    Some(pass),
+                    passphrase.as_deref(),
                     Some(&encrypt),
-                    ImageFormat::Png,
                     file_name,
                     file_list_json.as_deref(),
                     dict_bytes.as_deref(),
-                )?
+                )?;
             } else {
-                encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
-                    &payload,
-                    level,
-                    None,
-                    None,
-                    ImageFormat::Png,
-                    file_name,
-                    file_list_json.as_deref(),
-                    dict_bytes.as_deref(),
-                )?
-            };
-
-            write_all(&output, &png)?;
+                let png = if let Some(ref pass) = passphrase {
+                    encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+                        &payload,
+                        level,
+                        Some(pass),
+                        Some(&encrypt),
+                        ImageFormat::Png,
+                        file_name,
+                        file_list_json.as_deref(),
+                        dict_bytes.as_deref(),
+                    )?
+                } else {
+                    encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+                        &payload,
+                        level,
+                        None,
+                        None,
+                        ImageFormat::Png,
+                        file_name,
+                        file_list_json.as_deref(),
+                        dict_bytes.as_deref(),
+                    )?
+                };
+                write_all(&output, &png)?;
+            }
 
             if file_list_json.is_some() {
                 if is_dir {

package/native/streaming.rs

@@ -0,0 +1,214 @@
+use std::io::{Write, BufWriter};
+use std::fs::File;
+use std::path::Path;
+
+const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
+const PIXEL_MAGIC: &[u8] = b"PXL1";
+const MARKER_START: [(u8, u8, u8); 3] = [(255, 0, 0), (0, 255, 0), (0, 0, 255)];
+const MARKER_END: [(u8, u8, u8); 3] = [(0, 0, 255), (0, 255, 0), (255, 0, 0)];
+const MARKER_ZSTD: (u8, u8, u8) = (0, 255, 0);
+const MAGIC: &[u8] = b"ROX1";
+
+pub fn encode_to_png_file(
+    data: &[u8],
+    output_path: &Path,
+    compression_level: i32,
+    passphrase: Option<&str>,
+    encrypt_type: Option<&str>,
+    name: Option<&str>,
+    file_list: Option<&str>,
+    dict: Option<&[u8]>,
+) -> anyhow::Result<()> {
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, dict, MAGIC)
+        .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
+
+    let encrypted = if let Some(pass) = passphrase {
+        match encrypt_type.unwrap_or("aes") {
+            "xor" => crate::crypto::encrypt_xor(&compressed, pass),
+            "aes" => crate::crypto::encrypt_aes(&compressed, pass)?,
+            _ => crate::crypto::encrypt_aes(&compressed, pass)?,
+        }
+    } else {
+        crate::crypto::no_encryption(&compressed)
+    };
+    drop(compressed);
+
+    let meta_pixel = build_meta_pixel(&encrypted, name, file_list)?;
+    drop(encrypted);
+
+    let raw_payload_len = PIXEL_MAGIC.len() + meta_pixel.len();
+    let padding_needed = (3 - (raw_payload_len % 3)) % 3;
+    let padded_len = raw_payload_len + padding_needed;
+
+    let marker_start_len = 12;
+    let data_with_markers_len = marker_start_len + padded_len;
+    let data_pixels = (data_with_markers_len + 2) / 3;
+    let end_marker_pixels = 3;
+    let total_pixels = data_pixels + end_marker_pixels;
+
+    let side = (total_pixels as f64).sqrt().ceil() as usize;
+    let side = side.max(end_marker_pixels);
+    let width = side;
+    let height = side;
+    let row_bytes = width * 3;
+    let total_data_bytes = width * height * 3;
+    let marker_end_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
+
+    let flat = build_flat_buffer(&meta_pixel, padding_needed, marker_end_pos, total_data_bytes);
+    drop(meta_pixel);
+
+    let stride = row_bytes + 1;
+    let scanlines_total = height * stride;
+
+    let mut scanlines = vec![0u8; scanlines_total];
+    for row in 0..height {
+        let flat_start = row * row_bytes;
+        let flat_end = (flat_start + row_bytes).min(flat.len());
+        let copy_len = flat_end.saturating_sub(flat_start);
+        if copy_len > 0 {
+            let dst = row * stride + 1;
+            scanlines[dst..dst + copy_len].copy_from_slice(&flat[flat_start..flat_end]);
+        }
+    }
+    drop(flat);
+
+    let adler = crate::core::adler32_bytes(&scanlines);
+
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (scanlines_total + MAX_BLOCK - 1) / MAX_BLOCK;
+    let idat_len = 2 + num_blocks * 5 + scanlines_total + 4;
+
+    let f = File::create(output_path)?;
+    let mut w = BufWriter::with_capacity(16 * 1024 * 1024, f);
+
+    w.write_all(PNG_HEADER)?;
+
+    let mut ihdr = [0u8; 13];
+    ihdr[0..4].copy_from_slice(&(width as u32).to_be_bytes());
+    ihdr[4..8].copy_from_slice(&(height as u32).to_be_bytes());
+    ihdr[8] = 8;
+    ihdr[9] = 2;
+    write_chunk_small(&mut w, b"IHDR", &ihdr)?;
+
+    write_idat_direct(&mut w, &scanlines, idat_len, adler)?;
+    drop(scanlines);
+
+    if let Some(fl) = file_list {
+        write_chunk_small(&mut w, b"rXFL", fl.as_bytes())?;
+    }
+    write_chunk_small(&mut w, b"IEND", &[])?;
+    w.flush()?;
+
+    Ok(())
+}
+
+fn write_idat_direct<W: Write>(
+    w: &mut W,
+    scanlines: &[u8],
+    idat_len: usize,
+    adler: u32,
+) -> anyhow::Result<()> {
+    const MAX_BLOCK: usize = 65535;
+
+    w.write_all(&(idat_len as u32).to_be_bytes())?;
+    w.write_all(b"IDAT")?;
+
+    let mut crc = crc32fast::Hasher::new();
+    crc.update(b"IDAT");
+
+    let zlib = [0x78u8, 0x01];
+    w.write_all(&zlib)?;
+    crc.update(&zlib);
+
+    let mut offset = 0;
+    while offset < scanlines.len() {
+        let chunk_size = (scanlines.len() - offset).min(MAX_BLOCK);
+        let is_last = offset + chunk_size >= scanlines.len();
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        w.write_all(&header)?;
+        crc.update(&header);
+        let slice = &scanlines[offset..offset + chunk_size];
+        w.write_all(slice)?;
+        crc.update(slice);
+        offset += chunk_size;
+    }
+
+    let adler_bytes = adler.to_be_bytes();
+    w.write_all(&adler_bytes)?;
+    crc.update(&adler_bytes);
+
+    w.write_all(&crc.finalize().to_be_bytes())?;
+    Ok(())
+}
+
+fn build_flat_buffer(
+    meta_pixel: &[u8],
+    _padding_needed: usize,
+    marker_end_pos: usize,
+    total_data_bytes: usize,
+) -> Vec<u8> {
+    let mut flat = vec![0u8; total_data_bytes];
+
+    let mut pos = 0;
+    for m in &MARKER_START {
+        flat[pos] = m.0; flat[pos + 1] = m.1; flat[pos + 2] = m.2;
+        pos += 3;
+    }
+    flat[pos] = MARKER_ZSTD.0; flat[pos + 1] = MARKER_ZSTD.1; flat[pos + 2] = MARKER_ZSTD.2;
+    pos += 3;
+    flat[pos..pos + PIXEL_MAGIC.len()].copy_from_slice(PIXEL_MAGIC);
+    pos += PIXEL_MAGIC.len();
+    flat[pos..pos + meta_pixel.len()].copy_from_slice(meta_pixel);
+
+    if marker_end_pos + 9 <= total_data_bytes {
+        for (i, m) in MARKER_END.iter().enumerate() {
+            let off = marker_end_pos + i * 3;
+            flat[off] = m.0; flat[off + 1] = m.1; flat[off + 2] = m.2;
+        }
+    }
+
+    flat
+}
+
+fn build_meta_pixel(payload: &[u8], name: Option<&str>, file_list: Option<&str>) -> anyhow::Result<Vec<u8>> {
+    let version = 1u8;
+    let name_bytes = name.map(|n| n.as_bytes()).unwrap_or(&[]);
+    let name_len = name_bytes.len().min(255) as u8;
+    let payload_len_bytes = (payload.len() as u32).to_be_bytes();
+
+    let mut result = Vec::with_capacity(1 + 1 + name_len as usize + 4 + payload.len() + 256);
+    result.push(version);
+    result.push(name_len);
+    if name_len > 0 {
+        result.extend_from_slice(&name_bytes[..name_len as usize]);
+    }
+    result.extend_from_slice(&payload_len_bytes);
+    result.extend_from_slice(payload);
+
+    if let Some(fl) = file_list {
+        result.extend_from_slice(b"rXFL");
+        let json_bytes = fl.as_bytes();
+        result.extend_from_slice(&(json_bytes.len() as u32).to_be_bytes());
+        result.extend_from_slice(json_bytes);
+    }
+
+    Ok(result)
+}
+
+fn write_chunk_small<W: Write>(w: &mut W, chunk_type: &[u8; 4], data: &[u8]) -> anyhow::Result<()> {
+    w.write_all(&(data.len() as u32).to_be_bytes())?;
+    w.write_all(chunk_type)?;
+    w.write_all(data)?;
+
+    let mut h = crc32fast::Hasher::new();
+    h.update(chunk_type);
+    h.update(data);
+    w.write_all(&h.finalize().to_be_bytes())?;
+    Ok(())
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "roxify",
-  "version": "1.11.0",
+  "version": "1.11.1",
   "type": "module",
   "description": "Ultra-lightweight PNG steganography with native Rust acceleration. Encode binary data into PNG images with zstd compression.",
   "main": "dist/index.js",