roxify 1.11.0 → 1.12.0

package/native/main.rs

@@ -14,6 +14,9 @@ mod png_utils;
 mod audio;
 mod reconstitution;
 mod archive;
+mod streaming;
+mod streaming_decode;
+mod streaming_encode;
 
 use crate::encoder::ImageFormat;
 use std::path::PathBuf;
@@ -104,15 +107,28 @@ enum Commands {
 }
 
 fn read_all(path: &PathBuf) -> anyhow::Result<Vec<u8>> {
-    let mut f = File::open(path)?;
-    let mut buf = Vec::new();
-    f.read_to_end(&mut buf)?;
-    Ok(buf)
+    let metadata = std::fs::metadata(path)?;
+    let size = metadata.len() as usize;
+
+    if size > 256 * 1024 * 1024 {
+        let file = File::open(path)?;
+        let mmap = unsafe { memmap2::Mmap::map(&file)? };
+        Ok(mmap.to_vec())
+    } else {
+        let mut f = File::open(path)?;
+        let mut buf = Vec::with_capacity(size);
+        f.read_to_end(&mut buf)?;
+        Ok(buf)
+    }
 }
 
 fn write_all(path: &PathBuf, data: &[u8]) -> anyhow::Result<()> {
-    let mut f = File::create(path)?;
-    f.write_all(data)?;
+    let f = File::create(path)?;
+    let buf_size = if data.len() > 64 * 1024 * 1024 { 16 * 1024 * 1024 }
+        else { (8 * 1024 * 1024).min(data.len().max(8192)) };
+    let mut writer = std::io::BufWriter::with_capacity(buf_size, f);
+    writer.write_all(data)?;
+    writer.flush()?;
     Ok(())
 }
 
@@ -143,53 +159,79 @@ fn main() -> anyhow::Result<()> {
         }
         Commands::Encode { input, output, level, passphrase, encrypt, name, dict } => {
             let is_dir = input.is_dir();
+
+            let file_name = name.as_deref()
+                .or_else(|| input.file_name().and_then(|n| n.to_str()));
+
+            if is_dir && dict.is_none() {
+                streaming_encode::encode_dir_to_png_encrypted(
+                    &input,
+                    &output,
+                    level,
+                    file_name,
+                    passphrase.as_deref(),
+                    Some(&encrypt),
+                )?;
+                println!("(TAR archive, rXFL chunk embedded)");
+                return Ok(());
+            }
+
             let (payload, file_list_json) = if is_dir {
-                let tar_data = archive::tar_pack_directory(&input)
-                    .map_err(|e| anyhow::anyhow!(e))?;
-                let list = archive::tar_file_list(&tar_data)
+                let result = archive::tar_pack_directory_with_list(&input)
                     .map_err(|e| anyhow::anyhow!(e))?;
-                let json_list: Vec<serde_json::Value> = list.iter()
+                let json_list: Vec<serde_json::Value> = result.file_list.iter()
                     .map(|(name, size)| serde_json::json!({"name": name, "size": size}))
                     .collect();
-                (tar_data, Some(serde_json::to_string(&json_list)?))
+                (result.data, Some(serde_json::to_string(&json_list)?))
             } else {
                 let pack_result = packer::pack_path_with_metadata(&input)?;
                 (pack_result.data, pack_result.file_list_json)
             };
 
-            let file_name = name.as_deref()
-                .or_else(|| input.file_name().and_then(|n| n.to_str()));
-
             let dict_bytes: Option<Vec<u8>> = match dict {
                 Some(path) => Some(read_all(&path)?),
                 None => None,
             };
 
-            let png = if let Some(ref pass) = passphrase {
-                encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+            let use_streaming = payload.len() > 64 * 1024 * 1024;
+
+            if use_streaming {
+                streaming::encode_to_png_file(
                     &payload,
+                    &output,
                     level,
-                    Some(pass),
+                    passphrase.as_deref(),
                     Some(&encrypt),
-                    ImageFormat::Png,
                     file_name,
                     file_list_json.as_deref(),
                     dict_bytes.as_deref(),
-                )?
+                )?;
             } else {
-                encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
-                    &payload,
-                    level,
-                    None,
-                    None,
-                    ImageFormat::Png,
-                    file_name,
-                    file_list_json.as_deref(),
-                    dict_bytes.as_deref(),
-                )?
-            };
-
-            write_all(&output, &png)?;
+                let png = if let Some(ref pass) = passphrase {
+                    encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+                        &payload,
+                        level,
+                        Some(pass),
+                        Some(&encrypt),
+                        ImageFormat::Png,
+                        file_name,
+                        file_list_json.as_deref(),
+                        dict_bytes.as_deref(),
+                    )?
+                } else {
+                    encoder::encode_to_png_with_encryption_name_and_format_and_filelist(
+                        &payload,
+                        level,
+                        None,
+                        None,
+                        ImageFormat::Png,
+                        file_name,
+                        file_list_json.as_deref(),
+                        dict_bytes.as_deref(),
+                    )?
+                };
+                write_all(&output, &png)?;
+            }
 
             if file_list_json.is_some() {
                 if is_dir {
@@ -246,13 +288,13 @@ fn main() -> anyhow::Result<()> {
             let is_png = buf.len() >= 8 && &buf[0..8] == &[137, 80, 78, 71, 13, 10, 26, 10];
             if is_png {
                 let payload = png_utils::extract_payload_from_png(&buf).map_err(|e| anyhow::anyhow!(e))?;
-                if !payload.is_empty() && (payload[0] == 0x01 || payload[0] == 0x02) {
+                if !payload.is_empty() && (payload[0] == 0x01 || payload[0] == 0x02 || payload[0] == 0x03) {
                     println!("Passphrase detected.");
                 } else {
                     println!("No passphrase detected.");
                 }
             } else {
-                if !buf.is_empty() && (buf[0] == 0x01 || buf[0] == 0x02) {
+                if !buf.is_empty() && (buf[0] == 0x01 || buf[0] == 0x02 || buf[0] == 0x03) {
                     println!("Passphrase detected.");
                 } else {
                     println!("No passphrase detected.");
@@ -294,6 +336,27 @@ fn main() -> anyhow::Result<()> {
             write_all(&dest, &out)?;
         }
         Commands::Decompress { input, output, files, passphrase, dict } => {
+            let file_size = std::fs::metadata(&input).map(|m| m.len()).unwrap_or(0);
+            let is_png_file = input.extension().map(|e| e == "png").unwrap_or(false)
+                || (file_size >= 8 && {
+                    let mut sig = [0u8; 8];
+                    std::fs::File::open(&input).and_then(|mut f| { use std::io::Read; f.read_exact(&mut sig) }).is_ok()
+                        && sig == [137, 80, 78, 71, 13, 10, 26, 10]
+                });
+
+            if is_png_file && files.is_none() && dict.is_none() && file_size > 100_000_000 {
+                let out_dir = output.clone().unwrap_or_else(|| PathBuf::from("out.raw"));
+                match streaming_decode::streaming_decode_to_dir_encrypted(&input, &out_dir, passphrase.as_deref()) {
+                    Ok(written) => {
+                        println!("Unpacked {} files (TAR)", written.len());
+                        return Ok(());
+                    }
+                    Err(e) => {
+                        eprintln!("Streaming decode failed ({}), falling back to in-memory", e);
+                    }
+                }
+            }
+
             let buf = read_all(&input)?;
             let dict_bytes: Option<Vec<u8>> = match dict {
                 Some(path) => Some(read_all(&path)?),
@@ -333,7 +396,7 @@ fn main() -> anyhow::Result<()> {
                         buf[1..].to_vec()
                     } else if buf.starts_with(b"ROX1") {
                         buf[4..].to_vec()
-                    } else if buf[0] == 0x01u8 || buf[0] == 0x02u8 {
+                    } else if buf[0] == 0x01u8 || buf[0] == 0x02u8 || buf[0] == 0x03u8 {
                         let pass = passphrase.as_ref().map(|s: &String| s.as_str());
                         match crate::crypto::try_decrypt(&buf, pass) {
                             Ok(v) => v,

package/native/png_utils.rs

@@ -174,7 +174,7 @@ pub fn extract_payload_from_png(png_data: &[u8]) -> Result<Vec<u8>, String> {
 
 fn validate_payload_deep(payload: &[u8]) -> bool {
     if payload.len() < 5 { return false; }
-    if payload[0] == 0x01 || payload[0] == 0x02 { return true; }
+    if payload[0] == 0x01 || payload[0] == 0x02 || payload[0] == 0x03 { return true; }
     let compressed = if payload[0] == 0x00 { &payload[1..] } else { payload };
     if compressed.starts_with(b"ROX1") { return true; }
     crate::core::zstd_decompress_bytes(compressed, None).is_ok()

package/native/streaming.rs

@@ -0,0 +1,214 @@
+use std::io::{Write, BufWriter};
+use std::fs::File;
+use std::path::Path;
+
+const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
+const PIXEL_MAGIC: &[u8] = b"PXL1";
+const MARKER_START: [(u8, u8, u8); 3] = [(255, 0, 0), (0, 255, 0), (0, 0, 255)];
+const MARKER_END: [(u8, u8, u8); 3] = [(0, 0, 255), (0, 255, 0), (255, 0, 0)];
+const MARKER_ZSTD: (u8, u8, u8) = (0, 255, 0);
+const MAGIC: &[u8] = b"ROX1";
+
+pub fn encode_to_png_file(
+    data: &[u8],
+    output_path: &Path,
+    compression_level: i32,
+    passphrase: Option<&str>,
+    encrypt_type: Option<&str>,
+    name: Option<&str>,
+    file_list: Option<&str>,
+    dict: Option<&[u8]>,
+) -> anyhow::Result<()> {
+    let compressed = crate::core::zstd_compress_with_prefix(data, compression_level, dict, MAGIC)
+        .map_err(|e| anyhow::anyhow!("Compression failed: {}", e))?;
+
+    let encrypted = if let Some(pass) = passphrase {
+        match encrypt_type.unwrap_or("aes") {
+            "xor" => crate::crypto::encrypt_xor(&compressed, pass),
+            "aes" => crate::crypto::encrypt_aes(&compressed, pass)?,
+            _ => crate::crypto::encrypt_aes(&compressed, pass)?,
+        }
+    } else {
+        crate::crypto::no_encryption(&compressed)
+    };
+    drop(compressed);
+
+    let meta_pixel = build_meta_pixel(&encrypted, name, file_list)?;
+    drop(encrypted);
+
+    let raw_payload_len = PIXEL_MAGIC.len() + meta_pixel.len();
+    let padding_needed = (3 - (raw_payload_len % 3)) % 3;
+    let padded_len = raw_payload_len + padding_needed;
+
+    let marker_start_len = 12;
+    let data_with_markers_len = marker_start_len + padded_len;
+    let data_pixels = (data_with_markers_len + 2) / 3;
+    let end_marker_pixels = 3;
+    let total_pixels = data_pixels + end_marker_pixels;
+
+    let side = (total_pixels as f64).sqrt().ceil() as usize;
+    let side = side.max(end_marker_pixels);
+    let width = side;
+    let height = side;
+    let row_bytes = width * 3;
+    let total_data_bytes = width * height * 3;
+    let marker_end_pos = (height - 1) * width * 3 + (width - end_marker_pixels) * 3;
+
+    let flat = build_flat_buffer(&meta_pixel, padding_needed, marker_end_pos, total_data_bytes);
+    drop(meta_pixel);
+
+    let stride = row_bytes + 1;
+    let scanlines_total = height * stride;
+
+    let mut scanlines = vec![0u8; scanlines_total];
+    for row in 0..height {
+        let flat_start = row * row_bytes;
+        let flat_end = (flat_start + row_bytes).min(flat.len());
+        let copy_len = flat_end.saturating_sub(flat_start);
+        if copy_len > 0 {
+            let dst = row * stride + 1;
+            scanlines[dst..dst + copy_len].copy_from_slice(&flat[flat_start..flat_end]);
+        }
+    }
+    drop(flat);
+
+    let adler = crate::core::adler32_bytes(&scanlines);
+
+    const MAX_BLOCK: usize = 65535;
+    let num_blocks = (scanlines_total + MAX_BLOCK - 1) / MAX_BLOCK;
+    let idat_len = 2 + num_blocks * 5 + scanlines_total + 4;
+
+    let f = File::create(output_path)?;
+    let mut w = BufWriter::with_capacity(16 * 1024 * 1024, f);
+
+    w.write_all(PNG_HEADER)?;
+
+    let mut ihdr = [0u8; 13];
+    ihdr[0..4].copy_from_slice(&(width as u32).to_be_bytes());
+    ihdr[4..8].copy_from_slice(&(height as u32).to_be_bytes());
+    ihdr[8] = 8;
+    ihdr[9] = 2;
+    write_chunk_small(&mut w, b"IHDR", &ihdr)?;
+
+    write_idat_direct(&mut w, &scanlines, idat_len, adler)?;
+    drop(scanlines);
+
+    if let Some(fl) = file_list {
+        write_chunk_small(&mut w, b"rXFL", fl.as_bytes())?;
+    }
+    write_chunk_small(&mut w, b"IEND", &[])?;
+    w.flush()?;
+
+    Ok(())
+}
+
+fn write_idat_direct<W: Write>(
+    w: &mut W,
+    scanlines: &[u8],
+    idat_len: usize,
+    adler: u32,
+) -> anyhow::Result<()> {
+    const MAX_BLOCK: usize = 65535;
+
+    w.write_all(&(idat_len as u32).to_be_bytes())?;
+    w.write_all(b"IDAT")?;
+
+    let mut crc = crc32fast::Hasher::new();
+    crc.update(b"IDAT");
+
+    let zlib = [0x78u8, 0x01];
+    w.write_all(&zlib)?;
+    crc.update(&zlib);
+
+    let mut offset = 0;
+    while offset < scanlines.len() {
+        let chunk_size = (scanlines.len() - offset).min(MAX_BLOCK);
+        let is_last = offset + chunk_size >= scanlines.len();
+        let header = [
+            if is_last { 0x01 } else { 0x00 },
+            chunk_size as u8,
+            (chunk_size >> 8) as u8,
+            !chunk_size as u8,
+            (!(chunk_size >> 8)) as u8,
+        ];
+        w.write_all(&header)?;
+        crc.update(&header);
+        let slice = &scanlines[offset..offset + chunk_size];
+        w.write_all(slice)?;
+        crc.update(slice);
+        offset += chunk_size;
+    }
+
+    let adler_bytes = adler.to_be_bytes();
+    w.write_all(&adler_bytes)?;
+    crc.update(&adler_bytes);
+
+    w.write_all(&crc.finalize().to_be_bytes())?;
+    Ok(())
+}
+
+fn build_flat_buffer(
+    meta_pixel: &[u8],
+    _padding_needed: usize,
+    marker_end_pos: usize,
+    total_data_bytes: usize,
+) -> Vec<u8> {
+    let mut flat = vec![0u8; total_data_bytes];
+
+    let mut pos = 0;
+    for m in &MARKER_START {
+        flat[pos] = m.0; flat[pos + 1] = m.1; flat[pos + 2] = m.2;
+        pos += 3;
+    }
+    flat[pos] = MARKER_ZSTD.0; flat[pos + 1] = MARKER_ZSTD.1; flat[pos + 2] = MARKER_ZSTD.2;
+    pos += 3;
+    flat[pos..pos + PIXEL_MAGIC.len()].copy_from_slice(PIXEL_MAGIC);
+    pos += PIXEL_MAGIC.len();
+    flat[pos..pos + meta_pixel.len()].copy_from_slice(meta_pixel);
+
+    if marker_end_pos + 9 <= total_data_bytes {
+        for (i, m) in MARKER_END.iter().enumerate() {
+            let off = marker_end_pos + i * 3;
+            flat[off] = m.0; flat[off + 1] = m.1; flat[off + 2] = m.2;
+        }
+    }
+
+    flat
+}
+
+fn build_meta_pixel(payload: &[u8], name: Option<&str>, file_list: Option<&str>) -> anyhow::Result<Vec<u8>> {
+    let version = 1u8;
+    let name_bytes = name.map(|n| n.as_bytes()).unwrap_or(&[]);
+    let name_len = name_bytes.len().min(255) as u8;
+    let payload_len_bytes = (payload.len() as u32).to_be_bytes();
+
+    let mut result = Vec::with_capacity(1 + 1 + name_len as usize + 4 + payload.len() + 256);
+    result.push(version);
+    result.push(name_len);
+    if name_len > 0 {
+        result.extend_from_slice(&name_bytes[..name_len as usize]);
+    }
+    result.extend_from_slice(&payload_len_bytes);
+    result.extend_from_slice(payload);
+
+    if let Some(fl) = file_list {
+        result.extend_from_slice(b"rXFL");
+        let json_bytes = fl.as_bytes();
+        result.extend_from_slice(&(json_bytes.len() as u32).to_be_bytes());
+        result.extend_from_slice(json_bytes);
+    }
+
+    Ok(result)
+}
+
+fn write_chunk_small<W: Write>(w: &mut W, chunk_type: &[u8; 4], data: &[u8]) -> anyhow::Result<()> {
+    w.write_all(&(data.len() as u32).to_be_bytes())?;
+    w.write_all(chunk_type)?;
+    w.write_all(data)?;
+
+    let mut h = crc32fast::Hasher::new();
+    h.update(chunk_type);
+    h.update(data);
+    w.write_all(&h.finalize().to_be_bytes())?;
+    Ok(())
+}