roxify 1.13.4 → 1.13.5

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.13.4"
+version = "1.13.5"
 edition = "2021"
 publish = false
 
@@ -59,6 +59,7 @@ bytemuck = { version = "1.14", features = ["derive"] }
 tokio = { version = "1", features = ["sync", "rt"], optional = true }
 parking_lot = "0.12"
 libsais = { version = "0.2.0", default-features = false }
+libc = "0.2"
 
 [features]
 default = []

package/README.md

@@ -77,6 +77,17 @@ Roxify 1.13.4 adds adaptive parallel preload for small files before feeding Zstd
 | Glados-Disc (19,645 files) | NTFS under Linux | 81.608 s | 2.189 s | 37.3x |
 | Gmod (3,936 files) | NTFS under Linux | 22.578 s | 4.517 s | 5.0x |
 
+### Portal 2 comparative reference: ZIP vs PNG
+
+Measured on the full `Portal 2` game directory (`3,731 files`, `193 folders`, `12.83 GiB` logical source) to compare classic ZIP packaging against Roxify PNG packing on the same dataset.
+
+| Format | Time (s) | Time (min:sec) | Throughput | Compression ratio |
+| --- | ---: | --- | ---: | ---: |
+| ZIP Encode | 633,87 | 10 min 33 s | 21,73 Mo/s | 36,08% |
+| ZIP Decode | 232,88 | 3 min 52 s | 59,15 Mo/s | - |
+| PNG Encode | 157,80 | 2 min 37 s | 87,30 Mo/s | 41,09% |
+| PNG Decode | 156,00 | 2 min 36 s | 88,30 Mo/s | - |
+
 ### Data integrity
 
 All benchmark runs completed with byte-exact roundtrip validation. Decode output matched original logical source bytes on every dataset.

package/dist/utils/decoder.js

@@ -12,9 +12,51 @@ import { cropAndReconstitute } from './reconstitution.js';
 import { decodeRobustAudio, isRobustAudioWav } from './robust-audio.js';
 import { decodeRobustImage, isRobustImage } from './robust-image.js';
 import { parallelZstdDecompress } from './zstd.js';
+const HEADER_VERSION_V1 = 1;
+const HEADER_VERSION_V2 = 2;
 function isColorMatch(r1, g1, b1, r2, g2, b2) {
     return Math.abs(r1 - r2) + Math.abs(g1 - g2) + Math.abs(b1 - b2) < 50;
 }
+function readPayloadLength(buf, idx, version) {
+    if (version === HEADER_VERSION_V1) {
+        if (idx + 4 > buf.length) {
+            throw new DataFormatError('Truncated payload length');
+        }
+        return { payloadLen: buf.readUInt32BE(idx), nextIdx: idx + 4 };
+    }
+    if (version === HEADER_VERSION_V2) {
+        if (idx + 8 > buf.length) {
+            throw new DataFormatError('Truncated payload length');
+        }
+        const payloadLen = buf.readBigUInt64BE(idx);
+        if (payloadLen > BigInt(Number.MAX_SAFE_INTEGER)) {
+            throw new DataFormatError('Payload too large for JS decoder');
+        }
+        return { payloadLen: Number(payloadLen), nextIdx: idx + 8 };
+    }
+    throw new DataFormatError(`Unsupported pixel header version ${version}`);
+}
+function readPixelPayloadHeader(buf, idx) {
+    if (idx + 2 > buf.length) {
+        throw new DataFormatError('Pixel payload header truncated');
+    }
+    const version = buf[idx++];
+    const nameLen = buf[idx++];
+    let name;
+    if (nameLen > 0) {
+        if (idx + nameLen > buf.length) {
+            throw new DataFormatError('Pixel payload name truncated');
+        }
+        name = buf.subarray(idx, idx + nameLen).toString('utf8');
+        idx += nameLen;
+    }
+    const { payloadLen, nextIdx } = readPayloadLength(buf, idx, version);
+    const available = buf.length - nextIdx;
+    if (available < payloadLen) {
+        throw new DataFormatError(`Pixel payload truncated: expected ${payloadLen} bytes but only ${available} available`);
+    }
+    return { version, name, payloadOffset: nextIdx, payloadLen };
+}
 /**
  * Un-stretch an image that was nearest-neighbor scaled.
  * 1. Crops to non-background bounding box
@@ -318,18 +360,12 @@ export async function decodePngToBinary(input, opts = {}) {
         const pcmData = wavToBytes(processedBuf);
         // The WAV payload starts with PIXEL_MAGIC ("PXL1")
         if (pcmData.length >= 4 && pcmData.subarray(0, 4).equals(PIXEL_MAGIC)) {
-            let idx = 4; // skip PIXEL_MAGIC
-            const version = pcmData[idx++];
-            const nameLen = pcmData[idx++];
-            let name;
-            if (nameLen > 0) {
-                name = pcmData.subarray(idx, idx + nameLen).toString('utf8');
-                idx += nameLen;
-            }
-            const payloadLen = pcmData.readUInt32BE(idx);
-            idx += 4;
-            const rawPayload = pcmData.subarray(idx, idx + payloadLen);
-            idx += payloadLen;
+            const header = readPixelPayloadHeader(pcmData, 4);
+            let idx = header.payloadOffset;
+            const version = header.version;
+            const name = header.name;
+            const rawPayload = pcmData.subarray(idx, idx + header.payloadLen);
+            idx += header.payloadLen;
             // Check for rXFL file list after payload
             let fileListJson;
             if (idx + 8 < pcmData.length &&
@@ -652,21 +688,10 @@ export async function decodePngToBinary(input, opts = {}) {
             if (logicalData.length < 8 + PIXEL_MAGIC.length) {
                 throw new DataFormatError('Pixel mode data too short');
             }
-            let idx = 8 + PIXEL_MAGIC.length;
-            const version = logicalData[idx++];
-            const nameLen = logicalData[idx++];
-            let name;
-            if (nameLen > 0 && nameLen < 256) {
-                name = logicalData.slice(idx, idx + nameLen).toString('utf8');
-                idx += nameLen;
-            }
-            const payloadLen = logicalData.readUInt32BE(idx);
-            idx += 4;
-            const available = logicalData.length - idx;
-            if (available < payloadLen) {
-                throw new DataFormatError(`Pixel payload truncated: expected ${payloadLen} bytes but only ${available} available`);
-            }
-            const rawPayload = logicalData.slice(idx, idx + payloadLen);
+            const header = readPixelPayloadHeader(logicalData, 8 + PIXEL_MAGIC.length);
+            const version = header.version;
+            const name = header.name;
+            const rawPayload = logicalData.slice(header.payloadOffset, header.payloadOffset + header.payloadLen);
             let payload = tryDecryptIfNeeded(rawPayload, opts.passphrase);
             try {
                 payload = await tryDecompress(payload, (info) => {
@@ -945,31 +970,10 @@ export async function decodePngToBinary(input, opts = {}) {
                 }
             }
             if (idx > 0) {
-                const version = pixelBytes[idx++];
-                const nameLen = pixelBytes[idx++];
-                let name;
-                if (nameLen > 0 && nameLen < 256) {
-                    name = pixelBytes.slice(idx, idx + nameLen).toString('utf8');
-                    idx += nameLen;
-                }
-                const payloadLen = pixelBytes.readUInt32BE(idx);
-                idx += 4;
-                if (idx + 4 <= pixelBytes.length) {
-                    const marker = pixelBytes.slice(idx, idx + 4).toString('utf8');
-                    if (marker === 'rXFL') {
-                        idx += 4;
-                        if (idx + 4 <= pixelBytes.length) {
-                            const jsonLen = pixelBytes.readUInt32BE(idx);
-                            idx += 4;
-                            idx += jsonLen;
-                        }
-                    }
-                }
-                const available = pixelBytes.length - idx;
-                if (available < payloadLen) {
-                    throw new DataFormatError(`Pixel payload truncated: expected ${payloadLen} bytes but only ${available} available`);
-                }
-                const rawPayload = pixelBytes.slice(idx, idx + payloadLen);
+                const header = readPixelPayloadHeader(pixelBytes, idx);
+                const version = header.version;
+                const name = header.name;
+                const rawPayload = pixelBytes.slice(header.payloadOffset, header.payloadOffset + header.payloadLen);
                 let payload = tryDecryptIfNeeded(rawPayload, opts.passphrase);
                 try {
                     payload = await tryDecompress(payload, (info) => {
@@ -1162,10 +1166,10 @@ export async function decodePngToBinary(input, opts = {}) {
                                     ii = found + PIXEL_MAGIC.length;
                             }
                             if (ii > 0) {
-                                const version2 = pixelBytes2[ii++];
-                                const nameLen2 = pixelBytes2[ii++];
-                                const payloadLen2 = pixelBytes2.readUInt32BE(ii + nameLen2);
-                                const rawPayload2 = pixelBytes2.slice(ii + nameLen2 + 4, ii + nameLen2 + 4 + payloadLen2);
+                                const header2 = readPixelPayloadHeader(pixelBytes2, ii);
+                                const version2 = header2.version;
+                                const name2 = header2.name;
+                                const rawPayload2 = pixelBytes2.slice(header2.payloadOffset, header2.payloadOffset + header2.payloadLen);
                                 let payload2 = tryDecryptIfNeeded(rawPayload2, opts.passphrase);
                                 payload2 = await tryDecompress(payload2, (info) => {
                                     if (opts.onProgress)
@@ -1181,13 +1185,13 @@ export async function decodePngToBinary(input, opts = {}) {
                                         if (opts.onProgress)
                                             opts.onProgress({ phase: 'done' });
                                         progressBar?.stop();
-                                        return { files: unpacked2.files, meta: { name } };
+                                        return { files: unpacked2.files, meta: { name: name2 } };
                                     }
                                 }
                                 if (opts.onProgress)
                                     opts.onProgress({ phase: 'done' });
                                 progressBar?.stop();
-                                return { buf: payload2, meta: { name } };
+                                return { buf: payload2, meta: { name: name2 } };
                             }
                         }
                         throw new DataFormatError('Screenshot mode zstd decompression failed: ' + errMsg);

package/dist/utils/encoder.js

@@ -9,6 +9,15 @@ import { native } from './native.js';
 import { encodeRobustAudio } from './robust-audio.js';
 import { encodeRobustImage } from './robust-image.js';
 import { parallelZstdCompress } from './zstd.js';
+const HEADER_VERSION_V2 = 2;
+function writePayloadLength(payloadLen) {
+    if (!Number.isSafeInteger(payloadLen) || payloadLen < 0) {
+        throw new RangeError(`Invalid payload length: ${payloadLen}`);
+    }
+    const buf = Buffer.alloc(8);
+    buf.writeBigUInt64BE(BigInt(payloadLen), 0);
+    return buf;
+}
 /**
  * Encode a buffer or array of buffers into a PNG image (ROX format).
  *
@@ -309,9 +318,8 @@ export async function encodeBinaryToPng(input, opts = {}) {
     if (opts.container === 'sound') {
         const nameBuf = opts.name ? Buffer.from(opts.name, 'utf8') : Buffer.alloc(0);
         const nameLen = nameBuf.length;
-        const payloadLenBuf = Buffer.alloc(4);
-        payloadLenBuf.writeUInt32BE(payloadTotalLen, 0);
-        const version = 1;
+        const payloadLenBuf = writePayloadLength(payloadTotalLen);
+        const version = HEADER_VERSION_V2;
         let wavPayload = [
             PIXEL_MAGIC,
             Buffer.from([version]),
@@ -357,9 +365,8 @@ export async function encodeBinaryToPng(input, opts = {}) {
     {
         const nameBuf = opts.name ? Buffer.from(opts.name, 'utf8') : Buffer.alloc(0);
         const nameLen = nameBuf.length;
-        const payloadLenBuf = Buffer.alloc(4);
-        payloadLenBuf.writeUInt32BE(payloadTotalLen, 0);
-        const version = 1;
+        const payloadLenBuf = writePayloadLength(payloadTotalLen);
+        const version = HEADER_VERSION_V2;
         let metaPixel = [
             Buffer.from([version]),
             Buffer.from([nameLen]),

package/native/encoder.rs

@@ -1,8 +1,11 @@
 use anyhow::Result;
 
+use crate::png_chunk_writer::{write_chunked_idat_bytes, write_png_chunk};
+
 const MAGIC: &[u8] = b"ROX1";
 const PIXEL_MAGIC: &[u8] = b"PXL1";
 const PNG_HEADER: &[u8] = &[137, 80, 78, 71, 13, 10, 26, 10];
+const HEADER_VERSION_V2: u8 = 2;
 
 const MARKER_START: [(u8, u8, u8); 3] = [(255, 0, 0), (0, 255, 0), (0, 0, 255)];
 const MARKER_END: [(u8, u8, u8); 3] = [(0, 0, 255), (0, 255, 0), (255, 0, 0)];
@@ -204,12 +207,12 @@ fn build_flat_pixel_buffer(
 }
 
 fn build_meta_pixel_with_name_and_filelist(payload: &[u8], name: Option<&str>, file_list: Option<&str>) -> Result<Vec<u8>> {
-    let version = 1u8;
+    let version = HEADER_VERSION_V2;
     let name_bytes = name.map(|n| n.as_bytes()).unwrap_or(&[]);
     let name_len = name_bytes.len().min(255) as u8;
-    let payload_len_bytes = (payload.len() as u32).to_be_bytes();
+    let payload_len_bytes = (payload.len() as u64).to_be_bytes();
 
-    let mut result = Vec::with_capacity(1 + 1 + name_len as usize + 4 + payload.len() + 256);
+    let mut result = Vec::with_capacity(1 + 1 + name_len as usize + 8 + payload.len() + 256);
     result.push(version);
     result.push(name_len);
 
@@ -245,33 +248,18 @@ fn build_png(width: usize, height: usize, idat_data: &[u8], file_list: Option<&s
     ihdr_data[11] = 0;
     ihdr_data[12] = 0;
 
-    write_chunk(&mut png, b"IHDR", &ihdr_data)?;
-    write_chunk(&mut png, b"IDAT", idat_data)?;
+    write_png_chunk(&mut png, b"IHDR", &ihdr_data)?;
+    write_chunked_idat_bytes(&mut png, idat_data)?;
 
     if let Some(file_list_json) = file_list {
-        write_chunk(&mut png, b"rXFL", file_list_json.as_bytes())?;
+        write_png_chunk(&mut png, b"rXFL", file_list_json.as_bytes())?;
     }
 
-    write_chunk(&mut png, b"IEND", &[])?;
+    write_png_chunk(&mut png, b"IEND", &[])?;
 
     Ok(png)
 }
 
-fn write_chunk(out: &mut Vec<u8>, chunk_type: &[u8; 4], data: &[u8]) -> Result<()> {
-    let len = data.len() as u32;
-    out.extend_from_slice(&len.to_be_bytes());
-    out.extend_from_slice(chunk_type);
-    out.extend_from_slice(data);
-
-    let mut hasher = crc32fast::Hasher::new();
-    hasher.update(chunk_type);
-    hasher.update(data);
-    let crc = hasher.finalize();
-
-    out.extend_from_slice(&crc.to_be_bytes());
-    Ok(())
-}
-
 fn create_raw_deflate_from_rows(flat: &[u8], row_bytes: usize, height: usize) -> Vec<u8> {
     let stride = row_bytes + 1;
     let scanlines_total = height * stride;

package/native/io_advice.rs

@@ -0,0 +1,43 @@
+use std::fs::File;
+#[cfg(target_os = "linux")]
+use std::os::fd::AsRawFd;
+
+pub const INPUT_DROP_GRANULARITY: u64 = 8 * 1024 * 1024;
+
+pub fn advise_file_sequential(file: &File) {
+    #[cfg(target_os = "linux")]
+    unsafe {
+        let _ = libc::posix_fadvise(file.as_raw_fd(), 0, 0, libc::POSIX_FADV_SEQUENTIAL);
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    let _ = file;
+}
+
+pub fn advise_drop(file: &File, offset: u64, len: u64) {
+    if len == 0 {
+        return;
+    }
+
+    #[cfg(target_os = "linux")]
+    unsafe {
+        let _ = libc::posix_fadvise(
+            file.as_raw_fd(),
+            offset as libc::off_t,
+            len as libc::off_t,
+            libc::POSIX_FADV_DONTNEED,
+        );
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    let _ = (file, offset, len);
+}
+
+pub fn sync_and_drop(file: &File, len: u64) {
+    if len == 0 {
+        return;
+    }
+
+    let _ = file.sync_data();
+    advise_drop(file, 0, len);
+}

package/native/lib.rs

@@ -17,6 +17,8 @@ mod encoder;
 mod packer;
 mod crypto;
 mod png_utils;
+mod png_chunk_writer;
+mod io_advice;
 mod image_utils;
 mod audio;
 mod progress;

package/native/main.rs

@@ -11,6 +11,8 @@ mod encoder;
 mod packer;
 mod crypto;
 mod png_utils;
+mod png_chunk_writer;
+mod io_advice;
 mod audio;
 mod reconstitution;
 mod archive;
@@ -255,32 +257,39 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
         }
         Commands::List { input } => {
             let mut file = File::open(&input)?;
-            let chunks = png_utils::extract_png_chunks_streaming(&mut file).map_err(|e| anyhow::anyhow!(e))?;
+            let mut chunk_scan_error: Option<anyhow::Error> = None;
 
-            if let Some(rxfl_chunk) = chunks.iter().find(|c| c.name == "rXFL") {
-                println!("{}", String::from_utf8_lossy(&rxfl_chunk.data));
-                return Ok(());
-            }
+            match png_utils::extract_png_chunks_streaming(&mut file) {
+                Ok(chunks) => {
+                    if let Some(rxfl_chunk) = chunks.iter().find(|c| c.name == "rXFL") {
+                        println!("{}", String::from_utf8_lossy(&rxfl_chunk.data));
+                        return Ok(());
+                    }
 
-            if let Some(meta_chunk) = chunks.iter().find(|c| c.name == "rOXm") {
-                if let Some(pos) = meta_chunk.data.windows(4).position(|w| w == b"rXFL") {
-                    if pos + 8 <= meta_chunk.data.len() {
-                        let json_len = u32::from_be_bytes([
-                            meta_chunk.data[pos + 4],
-                            meta_chunk.data[pos + 5],
-                            meta_chunk.data[pos + 6],
-                            meta_chunk.data[pos + 7],
-                        ]) as usize;
-
-                        let json_start = pos + 8;
-                        let json_end = json_start + json_len;
-
-                        if json_end <= meta_chunk.data.len() {
-                            println!("{}", String::from_utf8_lossy(&meta_chunk.data[json_start..json_end]));
-                            return Ok(());
+                    if let Some(meta_chunk) = chunks.iter().find(|c| c.name == "rOXm") {
+                        if let Some(pos) = meta_chunk.data.windows(4).position(|w| w == b"rXFL") {
+                            if pos + 8 <= meta_chunk.data.len() {
+                                let json_len = u32::from_be_bytes([
+                                    meta_chunk.data[pos + 4],
+                                    meta_chunk.data[pos + 5],
+                                    meta_chunk.data[pos + 6],
+                                    meta_chunk.data[pos + 7],
+                                ]) as usize;
+
+                                let json_start = pos + 8;
+                                let json_end = json_start + json_len;
+
+                                if json_end <= meta_chunk.data.len() {
+                                    println!("{}", String::from_utf8_lossy(&meta_chunk.data[json_start..json_end]));
+                                    return Ok(());
+                                }
+                            }
                         }
                     }
                 }
+                Err(err) => {
+                    chunk_scan_error = Some(anyhow::anyhow!(err));
+                }
             }
 
             let png_data = std::fs::read(&input)?;
@@ -289,11 +298,14 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
                     println!("{}", json);
                     return Ok(());
                 }
-                Err(_) => {}
+                Err(pixel_err) => {
+                    if let Some(chunk_err) = chunk_scan_error {
+                        return Err(anyhow::anyhow!("chunk scan: {}; pixel scan: {}", chunk_err, pixel_err));
+                    }
+                }
             }
 
-            eprintln!("No file list found in PNG");
-            std::process::exit(1);
+            return Err(anyhow::anyhow!("No file list found in PNG"));
         }
         Commands::Havepassphrase { input } => {
             let buf = read_all(&input)?;
@@ -377,7 +389,7 @@ fn parse_requested_files(files: &str) -> anyhow::Result<Vec<String>> {
                         return Ok(());
                     }
                     Err(e) => {
-                        eprintln!("Streaming decode failed ({}), falling back to in-memory", e);
+                        return Err(anyhow::anyhow!("Streaming decode failed: {}", e));
                     }
                 }
             }