roxify 1.11.0 → 1.12.0

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.11.0"
+version = "1.12.0"
 edition = "2021"
 publish = false
 
@@ -43,6 +43,10 @@ image = { version = "0.25", default-features = false, features = ["png"] }
 walkdir = "2.5.0"
 tar = "0.4"
 aes-gcm = "0.10"
+aes = "0.8"
+ctr = "0.9"
+cipher = { version = "0.4", features = ["std"] }
+hmac = "0.12"
 pbkdf2 = "0.12"
 rand = "0.8"
 sha2 = "0.10"

package/dist/cli.js

@@ -6,7 +6,7 @@ import { DataFormatError, decodePngToBinary, encodeBinaryToPng, hasPassphraseInP
 import { packPathsGenerator, unpackBuffer } from './pack.js';
 import * as cliProgress from './stub-progress.js';
 import { encodeWithRustCLI, isRustBinaryAvailable, } from './utils/rust-cli-wrapper.js';
-const VERSION = '1.9.2';
+const VERSION = '1.12.0';
 function getDirectorySize(dirPath) {
     let totalSize = 0;
     try {

package/native/archive.rs

@@ -4,7 +4,12 @@ use rayon::prelude::*;
 use tar::{Archive, Builder, Header};
 use walkdir::WalkDir;
 
-pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+pub struct TarPackResult {
+    pub data: Vec<u8>,
+    pub file_list: Vec<(String, u64)>,
+}
+
+pub fn tar_pack_directory_with_list(dir_path: &Path) -> Result<TarPackResult, String> {
     let base = dir_path;
 
     let entries: Vec<_> = WalkDir::new(dir_path)
@@ -27,6 +32,10 @@ pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
         })
         .collect();
 
+    let file_list: Vec<(String, u64)> = file_data.iter()
+        .map(|(name, data)| (name.clone(), data.len() as u64))
+        .collect();
+
     let total_estimate: usize = file_data.iter().map(|(n, d)| 512 + d.len() + 512 + n.len()).sum();
     let mut buf = Vec::with_capacity(total_estimate + 1024);
     {
@@ -42,20 +51,42 @@ pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
         }
         builder.finish().map_err(|e| format!("tar finish: {}", e))?;
     }
-    Ok(buf)
+    Ok(TarPackResult { data: buf, file_list })
+}
+
+pub fn tar_pack_directory(dir_path: &Path) -> Result<Vec<u8>, String> {
+    tar_pack_directory_with_list(dir_path).map(|r| r.data)
+}
+
+pub fn tar_file_list_fast(tar_data: &[u8]) -> Vec<(String, u64)> {
+    let mut list = Vec::new();
+    let mut pos = 0;
+    while pos + 512 <= tar_data.len() {
+        let header = &tar_data[pos..pos + 512];
+        if header.iter().all(|&b| b == 0) {
+            break;
+        }
+        let name_end = header[..100].iter().position(|&b| b == 0).unwrap_or(100);
+        let name = String::from_utf8_lossy(&header[..name_end]).to_string();
+        let size_str = String::from_utf8_lossy(&header[124..136]);
+        let size = u64::from_str_radix(size_str.trim().trim_matches('\0'), 8).unwrap_or(0);
+        if !name.is_empty() {
+            list.push((name, size));
+        }
+        let data_blocks = (size as usize + 511) / 512;
+        pos += 512 + data_blocks * 512;
+    }
+    list
 }
 
 pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, String> {
     let mut archive = Archive::new(Cursor::new(tar_data));
-    let mut written = Vec::new();
+    let mut entries_data: Vec<(std::path::PathBuf, Vec<u8>)> = Vec::new();
 
     let entries = archive.entries().map_err(|e| format!("tar entries: {}", e))?;
     for entry in entries {
         let mut entry = entry.map_err(|e| format!("tar entry: {}", e))?;
-        let path = entry
-            .path()
-            .map_err(|e| format!("tar entry path: {}", e))?
-            .to_path_buf();
+        let path = entry.path().map_err(|e| format!("tar entry path: {}", e))?.to_path_buf();
 
         let mut safe = std::path::PathBuf::new();
         for comp in path.components() {
@@ -63,23 +94,36 @@ pub fn tar_unpack(tar_data: &[u8], output_dir: &Path) -> Result<Vec<String>, Str
                 safe.push(osstr);
             }
         }
-
         if safe.as_os_str().is_empty() {
             continue;
         }
 
-        let dest = output_dir.join(&safe);
-        if let Some(parent) = dest.parent() {
-            std::fs::create_dir_all(parent)
-                .map_err(|e| format!("mkdir {:?}: {}", parent, e))?;
-        }
+        let mut data = Vec::with_capacity(entry.size() as usize);
+        std::io::Read::read_to_end(&mut entry, &mut data)
+            .map_err(|e| format!("tar read {:?}: {}", safe, e))?;
+        entries_data.push((safe, data));
+    }
 
-        entry
-            .unpack(&dest)
-            .map_err(|e| format!("tar unpack {:?}: {}", dest, e))?;
-        written.push(safe.to_string_lossy().to_string());
+    let dirs: std::collections::HashSet<_> = entries_data.iter()
+        .filter_map(|(p, _)| {
+            let dest = output_dir.join(p);
+            dest.parent().map(|d| d.to_path_buf())
+        })
+        .collect();
+    for dir in &dirs {
+        std::fs::create_dir_all(dir).map_err(|e| format!("mkdir {:?}: {}", dir, e))?;
     }
 
+    let written: Vec<String> = entries_data.par_iter()
+        .filter_map(|(safe, data)| {
+            let dest = output_dir.join(safe);
+            match std::fs::write(&dest, data) {
+                Ok(_) => Some(safe.to_string_lossy().to_string()),
+                Err(_) => None,
+            }
+        })
+        .collect();
+
     Ok(written)
 }
 

package/native/core.rs

@@ -85,15 +85,55 @@ pub fn crc32_bytes(buf: &[u8]) -> u32 {
 
 pub fn adler32_bytes(buf: &[u8]) -> u32 {
     const MOD: u32 = 65521;
+    const NMAX: usize = 5552;
+
+    if buf.len() > 4 * 1024 * 1024 {
+        return adler32_parallel(buf);
+    }
+
     let mut a: u32 = 1;
     let mut b: u32 = 0;
-    for &v in buf {
-        a = (a + v as u32) % MOD;
-        b = (b + a) % MOD;
+
+    for chunk in buf.chunks(NMAX) {
+        for &v in chunk {
+            a += v as u32;
+            b += a;
+        }
+        a %= MOD;
+        b %= MOD;
     }
+
     (b << 16) | a
 }
 
+fn adler32_parallel(buf: &[u8]) -> u32 {
+    use rayon::prelude::*;
+    const MOD: u32 = 65521;
+    const CHUNK: usize = 1024 * 1024;
+
+    let chunks: Vec<&[u8]> = buf.chunks(CHUNK).collect();
+    let partials: Vec<(u32, u32, usize)> = chunks.par_iter().map(|chunk| {
+        let mut a: u32 = 0;
+        let mut b: u32 = 0;
+        for &v in *chunk {
+            a += v as u32;
+            b += a;
+        }
+        a %= MOD;
+        b %= MOD;
+        (a, b, chunk.len())
+    }).collect();
+
+    let mut a: u64 = 1;
+    let mut b: u64 = 0;
+    for (pa, pb, len) in partials {
+        b = (b + pb as u64 + a * len as u64) % MOD as u64;
+        a = (a + pa as u64) % MOD as u64;
+    }
+
+    ((b as u32) << 16) | (a as u32)
+}
+
 pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {
     let len = buf.len();
     if len == 0 {
@@ -173,40 +213,85 @@ pub fn train_zstd_dictionary(sample_paths: &[PathBuf], dict_size: usize) -> Resu
 /// For large buffers (>50 MiB) without a dictionary, multiple chunk sizes
 /// are benchmarked on a sample and the best is selected automatically.
 pub fn zstd_compress_bytes(buf: &[u8], level: i32, dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
+    zstd_compress_with_prefix(buf, level, dict, &[])
+}
+
+pub fn zstd_compress_with_prefix(buf: &[u8], level: i32, dict: Option<&[u8]>, prefix: &[u8]) -> std::result::Result<Vec<u8>, String> {
     use std::io::Write;
 
     let actual_level = level.min(22).max(1);
+    let total_len = prefix.len() + buf.len();
+
+    let adaptive_level = if total_len > 2 * 1024 * 1024 * 1024 {
+        actual_level.min(1)
+    } else if total_len > 1024 * 1024 * 1024 {
+        actual_level.min(3)
+    } else if total_len > 256 * 1024 * 1024 {
+        actual_level.min(6)
+    } else if total_len > 64 * 1024 * 1024 {
+        actual_level.min(12)
+    } else {
+        actual_level
+    };
+
+    if dict.is_none() && total_len < 4 * 1024 * 1024 {
+        if prefix.is_empty() {
+            return zstd::bulk::compress(buf, adaptive_level)
+                .map_err(|e| format!("zstd bulk compress error: {}", e));
+        }
+        let mut combined = Vec::with_capacity(total_len);
+        combined.extend_from_slice(prefix);
+        combined.extend_from_slice(buf);
+        return zstd::bulk::compress(&combined, adaptive_level)
+            .map_err(|e| format!("zstd bulk compress error: {}", e));
+    }
+
     let mut encoder = if let Some(d) = dict {
-        zstd::stream::Encoder::with_dictionary(Vec::new(), actual_level, d)
+        zstd::stream::Encoder::with_dictionary(Vec::with_capacity(total_len / 2), adaptive_level, d)
             .map_err(|e| format!("zstd encoder init error: {}", e))?
     } else {
-        zstd::stream::Encoder::new(Vec::new(), actual_level)
+        zstd::stream::Encoder::new(Vec::with_capacity(total_len / 2), adaptive_level)
             .map_err(|e| format!("zstd encoder init error: {}", e))?
     };
 
     let threads = num_cpus::get() as u32;
     if threads > 1 {
-        let max_threads = if actual_level >= 20 { threads.min(4) } else { threads };
+        let max_threads = if adaptive_level >= 20 { threads.min(4) } else { threads };
         let _ = encoder.multithread(max_threads);
     }
 
-    if buf.len() > 1024 * 1024 {
+    if total_len > 256 * 1024 && adaptive_level >= 3 {
         let _ = encoder.long_distance_matching(true);
-        let wlog = if buf.len() > 512 * 1024 * 1024 { 28 }
-            else if buf.len() > 64 * 1024 * 1024 { 27 }
+    }
+    if total_len > 256 * 1024 {
+        let wlog = if total_len > 1024 * 1024 * 1024 { 30 }
+            else if total_len > 512 * 1024 * 1024 { 29 }
+            else if total_len > 64 * 1024 * 1024 { 28 }
+            else if total_len > 8 * 1024 * 1024 { 27 }
             else { 26 };
         let _ = encoder.window_log(wlog);
     }
 
-    let _ = encoder.set_pledged_src_size(Some(buf.len() as u64));
+    let _ = encoder.set_pledged_src_size(Some(total_len as u64));
+
+    if !prefix.is_empty() {
+        encoder.write_all(prefix).map_err(|e| format!("zstd write prefix error: {}", e))?;
+    }
+
+    let chunk_size = if total_len > 256 * 1024 * 1024 { 16 * 1024 * 1024 }
+        else if total_len > 64 * 1024 * 1024 { 8 * 1024 * 1024 }
+        else { buf.len() };
+
+    for chunk in buf.chunks(chunk_size) {
+        encoder.write_all(chunk).map_err(|e| format!("zstd write error: {}", e))?;
+    }
 
-    encoder.write_all(buf).map_err(|e| format!("zstd write error: {}", e))?;
     encoder.finish().map_err(|e| format!("zstd finish error: {}", e))
 }
 
 pub fn zstd_decompress_bytes(buf: &[u8], dict: Option<&[u8]>) -> std::result::Result<Vec<u8>, String> {
     use std::io::Read;
-    let mut out = Vec::new();
+    let mut out = Vec::with_capacity(buf.len() * 2);
     if let Some(d) = dict {
         let mut decoder = zstd::stream::Decoder::with_dictionary(std::io::Cursor::new(buf), d)
             .map_err(|e| format!("zstd decoder init error: {}", e))?;

package/native/crypto.rs

@@ -7,10 +7,18 @@ use pbkdf2::pbkdf2_hmac;
 use rand::RngCore;
 use sha2::Sha256;
 
+use aes::Aes256;
+use cipher::{KeyIvInit, StreamCipher};
+use hmac::{Hmac, Mac};
+
+type Aes256Ctr = ctr::Ctr64BE<Aes256>;
+type HmacSha256 = Hmac<Sha256>;
+
 const ENC_NONE: u8 = 0x00;
 const ENC_AES: u8 = 0x01;
 const ENC_XOR: u8 = 0x02;
-const PBKDF2_ITERS: u32 = 1_000_000;
+const ENC_AES_CTR: u8 = 0x03;
+const PBKDF2_ITERS: u32 = 600_000;
 
 pub fn encrypt_xor(data: &[u8], passphrase: &str) -> Vec<u8> {
     let key = passphrase.as_bytes();
@@ -114,6 +122,83 @@ pub fn try_decrypt(buf: &[u8], passphrase: Option<&str>) -> Result<Vec<u8>> {
             let pass = passphrase.ok_or_else(|| anyhow!("Passphrase required for AES decryption"))?;
             decrypt_aes(buf, pass)
         }
+        ENC_AES_CTR => {
+            let pass = passphrase.ok_or_else(|| anyhow!("Passphrase required for AES-CTR decryption"))?;
+            decrypt_aes_ctr(buf, pass)
+        }
         _ => Err(anyhow!("Unknown encryption flag: {}", flag)),
     }
 }
+
+pub fn derive_aes_ctr_key(passphrase: &str, salt: &[u8]) -> [u8; 32] {
+    let mut key = [0u8; 32];
+    pbkdf2_hmac::<Sha256>(passphrase.as_bytes(), salt, PBKDF2_ITERS, &mut key);
+    key
+}
+
+pub struct StreamingEncryptor {
+    cipher: Aes256Ctr,
+    hmac: HmacSha256,
+    pub header: Vec<u8>,
+}
+
+impl StreamingEncryptor {
+    pub fn new(passphrase: &str) -> Result<Self> {
+        let mut salt = [0u8; 16];
+        rand::thread_rng().fill_bytes(&mut salt);
+        let mut iv = [0u8; 16];
+        rand::thread_rng().fill_bytes(&mut iv);
+
+        let key = derive_aes_ctr_key(passphrase, &salt);
+        let cipher = Aes256Ctr::new_from_slices(&key, &iv)
+            .map_err(|e| anyhow!("AES-CTR init: {}", e))?;
+        let hmac = <HmacSha256 as Mac>::new_from_slice(&key)
+            .map_err(|e| anyhow!("HMAC init: {}", e))?;
+
+        let mut header = Vec::with_capacity(1 + 16 + 16);
+        header.push(ENC_AES_CTR);
+        header.extend_from_slice(&salt);
+        header.extend_from_slice(&iv);
+
+        Ok(Self { cipher, hmac, header })
+    }
+
+    pub fn header_len(&self) -> usize {
+        self.header.len()
+    }
+
+    pub fn encrypt_chunk(&mut self, buf: &mut [u8]) {
+        self.cipher.apply_keystream(buf);
+        self.hmac.update(buf);
+    }
+
+    pub fn finalize_hmac(self) -> [u8; 32] {
+        let result = self.hmac.finalize();
+        result.into_bytes().into()
+    }
+}
+
+pub fn decrypt_aes_ctr(data: &[u8], passphrase: &str) -> Result<Vec<u8>> {
+    if data.len() < 1 + 16 + 16 + 32 {
+        return Err(anyhow!("Invalid AES-CTR payload length"));
+    }
+    let salt = &data[1..17];
+    let iv = &data[17..33];
+    let hmac_tag = &data[data.len() - 32..];
+    let ciphertext = &data[33..data.len() - 32];
+
+    let key = derive_aes_ctr_key(passphrase, salt);
+
+    let mut mac = <HmacSha256 as Mac>::new_from_slice(&key)
+        .map_err(|e| anyhow!("HMAC init: {}", e))?;
+    mac.update(ciphertext);
+    mac.verify_slice(hmac_tag)
+        .map_err(|_| anyhow!("HMAC verification failed - wrong passphrase or corrupted data"))?;
+
+    let mut decrypted = ciphertext.to_vec();
+    let mut cipher = Aes256Ctr::new_from_slices(&key, iv)
+        .map_err(|e| anyhow!("AES-CTR init: {}", e))?;
+    cipher.apply_keystream(&mut decrypted);
+
+    Ok(decrypted)
+}

package/native/encoder.rs

@@ -2,7 +2,6 @@ use anyhow::Result;
 use std::process::{Command, Stdio};
 
 const MAGIC: &[u8] = b"ROX1";
-const ENC_NONE: u8 = 0x00;
 const PIXEL_MAGIC: &[u8] = b"PXL1";
 const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
 
@@ -112,9 +111,7 @@ pub fn encode_to_wav_with_encryption_name_and_filelist(
     file_list: Option<&str>,
 ) -> Result<Vec<u8>> {
     // Same compression + encryption pipeline as PNG
-    let payload_input = [MAGIC, data].concat();
-
-    let compressed = crate::core::zstd_compress_bytes(&payload_input, compression_level, None)
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, None, MAGIC)
         .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
 
     let encrypted = if let Some(pass) = passphrase {
@@ -180,9 +177,7 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
     file_list: Option<&str>,
     dict: Option<&[u8]>,
 ) -> Result<Vec<u8>> {
-    let payload_input = [MAGIC, data].concat();
-
-    let compressed = crate::core::zstd_compress_bytes(&payload_input, compression_level, dict)
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, dict, MAGIC)
         .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
 
     let encrypted = if let Some(pass) = passphrase {
@@ -194,65 +189,71 @@ fn encode_to_png_with_encryption_name_and_filelist_internal(
     } else {
         crate::crypto::no_encryption(&compressed)
     };
+    drop(compressed);
 
     let meta_pixel = build_meta_pixel_with_name_and_filelist(&encrypted, name, file_list)?;
-    let data_without_markers = [PIXEL_MAGIC, &meta_pixel].concat();
-
-    let padding_needed = (3 - (data_without_markers.len() % 3)) % 3;
-    let padded_data = if padding_needed > 0 {
-        [&data_without_markers[..], &vec![0u8; padding_needed]].concat()
-    } else {
-        data_without_markers
-    };
+    drop(encrypted);
 
-    let mut marker_bytes = Vec::with_capacity(12);
-    for m in &MARKER_START {
-        marker_bytes.extend_from_slice(&[m.0, m.1, m.2]);
-    }
-    marker_bytes.extend_from_slice(&[MARKER_ZSTD.0, MARKER_ZSTD.1, MARKER_ZSTD.2]);
+    let raw_payload_len = PIXEL_MAGIC.len() + meta_pixel.len();
+    let padding_needed = (3 - (raw_payload_len % 3)) % 3;
+    let padded_len = raw_payload_len + padding_needed;
 
-    let data_with_markers = [&marker_bytes[..], &padded_data[..]].concat();
+    let marker_start_len = 12;
+    let marker_end_len = 9;
 
-    let mut marker_end_bytes = Vec::with_capacity(9);
-    for m in &MARKER_END {
-        marker_end_bytes.extend_from_slice(&[m.0, m.1, m.2]);
-    }
-
-    let data_pixels = (data_with_markers.len() + 2) / 3;
+    let data_with_markers_len = marker_start_len + padded_len;
+    let data_pixels = (data_with_markers_len + 2) / 3;
     let end_marker_pixels = 3;
     let total_pixels = data_pixels + end_marker_pixels;
 
     let side = (total_pixels as f64).sqrt().ceil() as usize;
     let side = side.max(end_marker_pixels);
-
     let width = side;
     let height = side;
 
     let total_data_bytes = width * height * 3;
-    let mut full_data = vec![0u8; total_data_bytes];
+    let marker_end_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
 
-    let marker_start_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
+    let flat = build_flat_pixel_buffer(&meta_pixel, padding_needed, marker_end_pos, marker_end_len, total_data_bytes);
+    drop(meta_pixel);
 
-    let copy_len = data_with_markers.len().min(marker_start_pos);
-    full_data[..copy_len].copy_from_slice(&data_with_markers[..copy_len]);
+    let row_bytes = width * 3;
+    let idat_data = create_raw_deflate_from_rows(&flat, row_bytes, height);
+    drop(flat);
 
-    let end_len = marker_end_bytes.len().min(total_data_bytes - marker_start_pos);
-    full_data[marker_start_pos..marker_start_pos + end_len]
-        .copy_from_slice(&marker_end_bytes[..end_len]);
+    build_png(width, height, &idat_data, file_list)
+}
 
-    let stride = width * 3 + 1;
-    let mut scanlines = Vec::with_capacity(height * stride);
+fn build_flat_pixel_buffer(
+    meta_pixel: &[u8],
+    _padding_needed: usize,
+    marker_end_pos: usize,
+    marker_end_len: usize,
+    total_data_bytes: usize,
+) -> Vec<u8> {
+    let mut flat = vec![0u8; total_data_bytes];
 
-    for row in 0..height {
-        scanlines.push(0u8);
-        let src_start = row * width * 3;
-        let src_end = (row + 1) * width * 3;
-        scanlines.extend_from_slice(&full_data[src_start..src_end]);
+    let mut pos = 0;
+    for m in &MARKER_START {
+        flat[pos] = m.0; flat[pos + 1] = m.1; flat[pos + 2] = m.2;
+        pos += 3;
     }
+    flat[pos] = MARKER_ZSTD.0; flat[pos + 1] = MARKER_ZSTD.1; flat[pos + 2] = MARKER_ZSTD.2;
+    pos += 3;
 
-    let idat_data = create_raw_deflate(&scanlines);
+    flat[pos..pos + PIXEL_MAGIC.len()].copy_from_slice(PIXEL_MAGIC);
+    pos += PIXEL_MAGIC.len();
 
-    build_png(width, height, &idat_data, file_list)
+    flat[pos..pos + meta_pixel.len()].copy_from_slice(meta_pixel);
+
+    if marker_end_pos + marker_end_len <= total_data_bytes {
+        for (i, m) in MARKER_END.iter().enumerate() {
+            let off = marker_end_pos + i * 3;
+            flat[off] = m.0; flat[off + 1] = m.1; flat[off + 2] = m.2;
+        }
+    }
+
+    flat
 }
 
 fn build_meta_pixel(payload: &[u8]) -> Result<Vec<u8>> {
@@ -323,33 +324,37 @@ fn write_chunk(out: &mut Vec<u8>, chunk_type: &[u8; 4], data: &[u8]) -> Result<(
     out.extend_from_slice(chunk_type);
     out.extend_from_slice(data);
 
-    let mut crc_data = Vec::with_capacity(chunk_type.len() + data.len());
-    crc_data.extend_from_slice(chunk_type);
-    crc_data.extend_from_slice(data);
-    let crc = crate::core::crc32_bytes(&crc_data);
+    let mut hasher = crc32fast::Hasher::new();
+    hasher.update(chunk_type);
+    hasher.update(data);
+    let crc = hasher.finalize();
 
     out.extend_from_slice(&crc.to_be_bytes());
     Ok(())
 }
 
 fn create_raw_deflate(data: &[u8]) -> Vec<u8> {
-    let mut result = Vec::with_capacity(data.len() + 6 + (data.len() / 65535 + 1) * 5);
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (data.len() + MAX_BLOCK - 1) / MAX_BLOCK;
+    let total_size = 2 + num_blocks * 5 + data.len() + 4;
+    let mut result = Vec::with_capacity(total_size);
 
     result.push(0x78);
     result.push(0x01);
 
     let mut offset = 0;
     while offset < data.len() {
-        let chunk_size = (data.len() - offset).min(65535);
+        let chunk_size = (data.len() - offset).min(MAX_BLOCK);
         let is_last = offset + chunk_size >= data.len();
 
-        result.push(if is_last { 0x01 } else { 0x00 });
-
-        result.push(chunk_size as u8);
-        result.push((chunk_size >> 8) as u8);
-        result.push(!chunk_size as u8);
-        result.push((!(chunk_size >> 8)) as u8);
-
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        result.extend_from_slice(&header);
         result.extend_from_slice(&data[offset..offset + chunk_size]);
         offset += chunk_size;
     }
@@ -360,6 +365,51 @@ fn create_raw_deflate(data: &[u8]) -> Vec<u8> {
     result
 }
 
+fn create_raw_deflate_from_rows(flat: &[u8], row_bytes: usize, height: usize) -> Vec<u8> {
+    let stride = row_bytes + 1;
+    let scanlines_total = height * stride;
+
+    let mut scanlines = vec![0u8; scanlines_total];
+    for row in 0..height {
+        let flat_start = row * row_bytes;
+        let flat_end = (flat_start + row_bytes).min(flat.len());
+        let copy_len = flat_end.saturating_sub(flat_start);
+        if copy_len > 0 {
+            let dst_start = row * stride + 1;
+            scanlines[dst_start..dst_start + copy_len].copy_from_slice(&flat[flat_start..flat_end]);
+        }
+    }
+
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (scanlines_total + MAX_BLOCK - 1) / MAX_BLOCK;
+    let total_size = 2 + num_blocks * 5 + scanlines_total + 4;
+    let mut result = Vec::with_capacity(total_size);
+
+    result.push(0x78);
+    result.push(0x01);
+
+    let mut offset = 0;
+    while offset < scanlines.len() {
+        let chunk_size = (scanlines.len() - offset).min(MAX_BLOCK);
+        let is_last = offset + chunk_size >= scanlines.len();
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        result.extend_from_slice(&header);
+        result.extend_from_slice(&scanlines[offset..offset + chunk_size]);
+        offset += chunk_size;
+    }
+
+    let adler = crate::core::adler32_bytes(&scanlines);
+    result.extend_from_slice(&adler.to_be_bytes());
+
+    result
+}
+
 fn predict_best_format(data: &[u8]) -> ImageFormat {
     if data.len() < 2048 {
         return ImageFormat::Png;

package/native/hybrid.rs

@@ -5,7 +5,7 @@ use crate::mtf::{mtf_encode, mtf_decode, rle0_encode, rle0_decode};
 use crate::rans_byte::{SymbolStats, rans_encode_block, rans_decode_block};
 use crate::context_mixing::analyze_entropy;
 
-const BLOCK_SIZE: usize = 256 * 1024;
+const BLOCK_SIZE: usize = 1024 * 1024;
 
 const BLOCK_FLAG_BWT: u8 = 0;
 const BLOCK_FLAG_ZSTD: u8 = 1;

package/native/lib.rs

@@ -24,6 +24,7 @@ mod audio;
 mod progress;
 mod reconstitution;
 mod archive;
+mod streaming;
 
 pub use core::*;
 #[cfg(feature = "gpu")]