roxify 1.12.10 → 1.13.0

package/Cargo.toml

@@ -1,98 +1,98 @@
-[package]
-name = "roxify_native"
-version = "1.12.10"
-edition = "2021"
-publish = false
-
-[lib]
-name = "roxify_native"
-crate-type = ["cdylib"]
-path = "native/lib.rs"
-
-[[bin]]
-name = "roxify_native"
-path = "native/main.rs"
-
-[[bin]]
-name = "bench_hybrid"
-path = "native/bench_hybrid.rs"
-
-[[bin]]
-name = "test_stages"
-path = "native/test_stages.rs"
-
-[[bin]]
-name = "test_small_bwt"
-path = "native/test_small_bwt.rs"
-
-[dev-dependencies]
-
-[dependencies]
-napi = "2"
-napi-derive = "2"
-rayon = "1.7"
-zstd = { version = "0.11", features = ["zstdmt"] }
-crc32fast = "1.3"
-num_cpus = "1.16"
-clap = { version = "4", features = ["derive"] }
-serde_json = "1.0"
-anyhow = "1.0"
-png = "0.18.0"
-image = { version = "0.25", default-features = false, features = ["png"] }
-# indicatif removed from default deps to reduce heavy build graph
-walkdir = "2.5.0"
-tar = "0.4"
-aes-gcm = "0.10"
-aes = "0.8"
-ctr = "0.9"
-cipher = { version = "0.4", features = ["std"] }
-hmac = "0.12"
-pbkdf2 = "0.12"
-rand = "0.8"
-sha2 = "0.10"
-mimalloc = "0.1"
-
-wgpu = { version = "0.19", optional = true }
-memmap2 = "0.9"
-bytemuck = { version = "1.14", features = ["derive"] }
-# tokio is optional now; enable via the 'async' feature
-tokio = { version = "1", features = ["sync", "rt"], optional = true }
-parking_lot = "0.12"
-pollster = { version = "0.3", optional = true }
-libsais = { version = "0.2.0", default-features = false }
-
-[features]
-# default is intentionally empty so the crate compiles fast for local checks.
-# Enable 'gpu' to pull in the WGPU and pollster dependencies (heavy).
-# Enable 'async' to include tokio runtime (optional).
-# Example: `cargo build -p roxify_native --features gpu`
-default = []
-
-gpu = ["wgpu", "pollster"]
-async = ["tokio"]
-full = ["gpu", "async"]
-
-[profile.release]
-opt-level = 3
-lto = "fat"
-codegen-units = 1
-strip = true
-panic = "abort"
-
-[profile.release-size]
-inherits = "release"
-opt-level = "z"
-lto = true
-strip = true
-
-[profile.fastdev]
-# Fast development profile for minimal user CPU and fast compilation.
-# Lower optimization and high codegen units to parallelize compilation work,
-# enable incremental to speed up subsequent incremental builds.
-inherits = "release"
-opt-level = 1
-lto = false
-codegen-units = 16
-debug = false
-incremental = true
-panic = "abort"
+[package]
+name = "roxify_native"
+version = "1.13.0"
+edition = "2021"
+publish = false
+
+[lib]
+name = "roxify_native"
+crate-type = ["cdylib"]
+path = "native/lib.rs"
+
+[[bin]]
+name = "roxify_native"
+path = "native/main.rs"
+
+[[bin]]
+name = "bench_hybrid"
+path = "native/bench_hybrid.rs"
+
+[[bin]]
+name = "test_stages"
+path = "native/test_stages.rs"
+
+[[bin]]
+name = "test_small_bwt"
+path = "native/test_small_bwt.rs"
+
+[dev-dependencies]
+
+[dependencies]
+napi = "2"
+napi-derive = "2"
+rayon = "1.7"
+zstd = { version = "0.11", features = ["zstdmt"] }
+crc32fast = "1.3"
+num_cpus = "1.16"
+clap = { version = "4", features = ["derive"] }
+serde_json = "1.0"
+anyhow = "1.0"
+png = "0.18.0"
+image = { version = "0.25", default-features = false, features = ["png"] }
+# indicatif removed from default deps to reduce heavy build graph
+walkdir = "2.5.0"
+tar = "0.4"
+aes-gcm = "0.10"
+aes = "0.8"
+ctr = "0.9"
+cipher = { version = "0.4", features = ["std"] }
+hmac = "0.12"
+pbkdf2 = "0.12"
+rand = "0.8"
+sha2 = "0.10"
+mimalloc = "0.1"
+
+wgpu = { version = "0.19", optional = true }
+memmap2 = "0.9"
+bytemuck = { version = "1.14", features = ["derive"] }
+# tokio is optional now; enable via the 'async' feature
+tokio = { version = "1", features = ["sync", "rt"], optional = true }
+parking_lot = "0.12"
+pollster = { version = "0.3", optional = true }
+libsais = { version = "0.2.0", default-features = false }
+
+[features]
+# default is intentionally empty so the crate compiles fast for local checks.
+# Enable 'gpu' to pull in the WGPU and pollster dependencies (heavy).
+# Enable 'async' to include tokio runtime (optional).
+# Example: `cargo build -p roxify_native --features gpu`
+default = []
+
+gpu = ["wgpu", "pollster"]
+async = ["tokio"]
+full = ["gpu", "async"]
+
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+strip = true
+panic = "abort"
+
+[profile.release-size]
+inherits = "release"
+opt-level = "z"
+lto = true
+strip = true
+
+[profile.fastdev]
+# Fast development profile for minimal user CPU and fast compilation.
+# Lower optimization and high codegen units to parallelize compilation work,
+# enable incremental to speed up subsequent incremental builds.
+inherits = "release"
+opt-level = 1
+lto = false
+codegen-units = 16
+debug = false
+incremental = true
+panic = "abort"

package/dist/cli.js

@@ -51,54 +51,54 @@ async function readLargeFile(filePath) {
     return Buffer.concat(chunks);
 }
 function showHelp() {
-    console.log(`
-ROX CLI — Encode/decode binary in PNG or WAV
-
-Usage:
-  npx rox <command> [options]
-
-Commands:
-  encode <input>... [output]   Encode one or more files/directories
-  decode <input> [output]      Decode PNG/WAV to original file
-  list <input>                 List files in a Rox archive
-  havepassphrase <input>       Check whether the archive requires a passphrase
-
-Options:
-  --image                   Use PNG container (default)
-  --sound                   Use WAV audio container (smaller overhead, faster)
-  --bwt-ans                 Use BWT-ANS compression instead of Zstd
-  -p, --passphrase <pass>   Use passphrase (AES-256-GCM)
-  -m, --mode <mode>         Mode: screenshot (default)
-  -e, --encrypt <type>      auto|aes|xor|none
-  --no-compress             Disable compression
-  --dict <file>             Use zstd dictionary when compressing
-  --force-ts                Force TypeScript encoder (slower but supports encryption)
-  -o, --output <path>       Output file path
-  -s, --sizes               Show file sizes in 'list' output (default)
-  --no-sizes                Disable file size reporting in 'list'
-  --files <list>            Extract only specified files (comma-separated)
-  --view-reconst            Export the reconstituted PNG for debugging
-  --debug                   Export debug images (doubled.png, reconstructed.png)
-  -v, --verbose             Show detailed errors
-
-Lossy-Resilient Encoding:
-  --lossy-resilient         Enable lossy-resilient mode (survives JPEG/MP3)
-  --ecc-level <level>       ECC redundancy: low|medium|quartile|high (default: medium)
-  --block-size <n>          Robust image block size: 2-8 pixels (default: 4)
-
-  When --lossy-resilient is active, data is encoded with Reed-Solomon ECC
-  and rendered as a QR-code-style grid (image) or MFSK tones (audio).
-  Use --sound or --image to choose the container format.
-
-Examples:
-  npx rox encode secret.pdf                      Encode to PNG
-  npx rox encode secret.pdf --sound               Encode to WAV
-  npx rox encode secret.pdf --lossy-resilient     Lossy-resilient PNG
-  npx rox encode secret.pdf --lossy-resilient --sound --ecc-level high
-  npx rox decode secret.pdf.png                   Decode back
-  npx rox decode secret.pdf.wav                   Decode WAV back
-
-Run "npx rox help" for this message.
+    console.log(`
+ROX CLI — Encode/decode binary in PNG or WAV
+
+Usage:
+  npx rox <command> [options]
+
+Commands:
+  encode <input>... [output]   Encode one or more files/directories
+  decode <input> [output]      Decode PNG/WAV to original file
+  list <input>                 List files in a Rox archive
+  havepassphrase <input>       Check whether the archive requires a passphrase
+
+Options:
+  --image                   Use PNG container (default)
+  --sound                   Use WAV audio container (smaller overhead, faster)
+  --bwt-ans                 Use BWT-ANS compression instead of Zstd
+  -p, --passphrase <pass>   Use passphrase (AES-256-GCM)
+  -m, --mode <mode>         Mode: screenshot (default)
+  -e, --encrypt <type>      auto|aes|xor|none
+  --no-compress             Disable compression
+  --dict <file>             Use zstd dictionary when compressing
+  --force-ts                Force TypeScript encoder (slower but supports encryption)
+  -o, --output <path>       Output file path
+  -s, --sizes               Show file sizes in 'list' output (default)
+  --no-sizes                Disable file size reporting in 'list'
+  --files <list>            Extract only specified files (comma-separated)
+  --view-reconst            Export the reconstituted PNG for debugging
+  --debug                   Export debug images (doubled.png, reconstructed.png)
+  -v, --verbose             Show detailed errors
+
+Lossy-Resilient Encoding:
+  --lossy-resilient         Enable lossy-resilient mode (survives JPEG/MP3)
+  --ecc-level <level>       ECC redundancy: low|medium|quartile|high (default: medium)
+  --block-size <n>          Robust image block size: 2-8 pixels (default: 4)
+
+  When --lossy-resilient is active, data is encoded with Reed-Solomon ECC
+  and rendered as a QR-code-style grid (image) or MFSK tones (audio).
+  Use --sound or --image to choose the container format.
+
+Examples:
+  npx rox encode secret.pdf                      Encode to PNG
+  npx rox encode secret.pdf --sound               Encode to WAV
+  npx rox encode secret.pdf --lossy-resilient     Lossy-resilient PNG
+  npx rox encode secret.pdf --lossy-resilient --sound --ecc-level high
+  npx rox decode secret.pdf.png                   Decode back
+  npx rox decode secret.pdf.wav                   Decode WAV back
+
+Run "npx rox help" for this message.
 `);
 }
 function parseArgs(args) {

package/dist/pack.js

@@ -161,7 +161,7 @@ export async function packPathsGenerator(paths, baseDir, onProgress) {
         indexHeader.writeUInt32BE(indexBuf.length, 4);
         yield Buffer.concat([indexHeader, indexBuf]);
         let readSoFar = 0;
-        const BATCH_SIZE = 1000;
+        const BATCH_SIZE = 100;
         const chunks = [];
         let chunkSize = 0;
         for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {

package/dist/utils/decoder.js

@@ -849,6 +849,32 @@ export async function decodePngToBinary(input, opts = {}) {
                 }
             }
         }
+        if (endStartPixel === -1) {
+            const scanLines = Math.min(logicalHeight, 5);
+            for (let row = logicalHeight - 1; row >= logicalHeight - scanLines && endStartPixel === -1; row--) {
+                const rowStart = row * logicalWidth;
+                for (let col = logicalWidth - MARKER_END.length; col >= 0 && endStartPixel === -1; col--) {
+                    let match = true;
+                    for (let mi = 0; mi < MARKER_END.length && match; mi++) {
+                        const pixelIdx = rowStart + col + mi;
+                        if (pixelIdx >= curTotalPixels) {
+                            match = false;
+                            break;
+                        }
+                        const offset = pixelIdx * 3;
+                        if (!isColorMatch(logicalData[offset], logicalData[offset + 1], logicalData[offset + 2], MARKER_END[mi].r, MARKER_END[mi].g, MARKER_END[mi].b)) {
+                            match = false;
+                        }
+                    }
+                    if (match) {
+                        endStartPixel = rowStart + col - startIdx;
+                        if (process.env.ROX_DEBUG) {
+                            console.log(`DEBUG: Found END marker via scan at row=${row}, col=${col}`);
+                        }
+                    }
+                }
+            }
+        }
         if (endStartPixel === -1) {
             if (process.env.ROX_DEBUG) {
                 console.log('DEBUG: END marker not found at expected position');
@@ -862,13 +888,16 @@ export async function decodePngToBinary(input, opts = {}) {
             endStartPixel = curTotalPixels - startIdx;
         }
         const dataPixelCount = endStartPixel - (MARKER_START.length + 1);
-        const pixelBytes = Buffer.allocUnsafe(dataPixelCount * 3);
-        for (let i = 0; i < dataPixelCount; i++) {
-            const srcOffset = (dataStartPixel + i) * 3;
-            const dstOffset = i * 3;
-            pixelBytes[dstOffset] = logicalData[srcOffset];
-            pixelBytes[dstOffset + 1] = logicalData[srcOffset + 1];
-            pixelBytes[dstOffset + 2] = logicalData[srcOffset + 2];
+        const srcByteOffset = dataStartPixel * 3;
+        const byteCount = dataPixelCount * 3;
+        const pixelBytes = Buffer.allocUnsafe(byteCount);
+        if (Buffer.isBuffer(logicalData)) {
+            logicalData.copy(pixelBytes, 0, srcByteOffset, srcByteOffset + byteCount);
+        }
+        else {
+            for (let i = 0; i < byteCount; i++) {
+                pixelBytes[i] = logicalData[srcByteOffset + i];
+            }
         }
         if (process.env.ROX_DEBUG) {
             console.log('DEBUG: extracted len', pixelBytes.length);
@@ -1079,6 +1108,29 @@ export async function decodePngToBinary(input, opts = {}) {
                                 }
                             }
                         }
+                        if (endStartPixel2 === -1) {
+                            const scanLines2 = Math.min(logicalHeight2, 5);
+                            for (let row2 = logicalHeight2 - 1; row2 >= logicalHeight2 - scanLines2 && endStartPixel2 === -1; row2--) {
+                                const rowStart2 = row2 * logicalWidth2;
+                                for (let col2 = logicalWidth2 - MARKER_END.length; col2 >= 0 && endStartPixel2 === -1; col2--) {
+                                    let m2 = true;
+                                    for (let mi = 0; mi < MARKER_END.length && m2; mi++) {
+                                        const pixelIdx2 = rowStart2 + col2 + mi;
+                                        if (pixelIdx2 >= curTotalPixels2) {
+                                            m2 = false;
+                                            break;
+                                        }
+                                        const off2 = pixelIdx2 * 3;
+                                        if (!isColorMatch(logicalData2[off2], logicalData2[off2 + 1], logicalData2[off2 + 2], MARKER_END[mi].r, MARKER_END[mi].g, MARKER_END[mi].b)) {
+                                            m2 = false;
+                                        }
+                                    }
+                                    if (m2) {
+                                        endStartPixel2 = rowStart2 + col2 - startIdx2;
+                                    }
+                                }
+                            }
+                        }
                         if (endStartPixel2 === -1) {
                             if (process.env.ROX_DEBUG) {
                                 console.log('DEBUG: END marker not found in fallback; using end of grid');

package/dist/utils/encoder.js

@@ -467,40 +467,29 @@ export async function encodeBinaryToPng(input, opts = {}) {
                 console.log(`[DEBUG] Width=${width}, Height=${height}, Pixels=${width * height}`);
                 console.log(`[DEBUG] outputFormat=${opts.outputFormat}, useManualPng=${useManualPng}`);
             }
+            const totalDataBytes = logicalWidth * logicalHeight * 3;
+            const markerEndPos = totalDataBytes - MARKER_END.length * 3;
+            const fullData = Buffer.alloc(totalDataBytes);
+            const flatData = Buffer.concat(dataWithMarkers);
+            flatData.copy(fullData, 0, 0, Math.min(flatData.length, markerEndPos));
+            let mOff = markerEndPos;
+            for (let i = 0; i < MARKER_END.length; i++) {
+                fullData[mOff++] = MARKER_END[i].r;
+                fullData[mOff++] = MARKER_END[i].g;
+                fullData[mOff++] = MARKER_END[i].b;
+            }
             let raw;
             let stride = 0;
             if (useManualPng) {
                 stride = width * 3 + 1;
                 raw = Buffer.alloc(height * stride);
-                const flatData = Buffer.concat(dataWithMarkers);
-                const markerEndBytes = Buffer.alloc(MARKER_END.length * 3);
-                for (let i = 0; i < MARKER_END.length; i++) {
-                    markerEndBytes[i * 3] = MARKER_END[i].r;
-                    markerEndBytes[i * 3 + 1] = MARKER_END[i].g;
-                    markerEndBytes[i * 3 + 2] = MARKER_END[i].b;
-                }
-                const totalDataBytes = logicalWidth * logicalHeight * 3;
-                const fullData = Buffer.alloc(totalDataBytes);
-                const markerEndPos = totalDataBytes - MARKER_END.length * 3;
-                flatData.copy(fullData, 0, 0, Math.min(flatData.length, markerEndPos));
-                markerEndBytes.copy(fullData, markerEndPos);
                 for (let row = 0; row < height; row++) {
                     raw[row * stride] = 0;
                     fullData.copy(raw, row * stride + 1, row * width * 3, (row + 1) * width * 3);
                 }
             }
             else {
-                raw = Buffer.alloc(width * height * 3);
-                const flatData = Buffer.concat(dataWithMarkers);
-                flatData.copy(raw, 0, 0, Math.min(flatData.length, raw.length));
-                const markerEndBytes = Buffer.alloc(MARKER_END.length * 3);
-                for (let i = 0; i < MARKER_END.length; i++) {
-                    markerEndBytes[i * 3] = MARKER_END[i].r;
-                    markerEndBytes[i * 3 + 1] = MARKER_END[i].g;
-                    markerEndBytes[i * 3 + 2] = MARKER_END[i].b;
-                }
-                const markerEndPos = raw.length - MARKER_END.length * 3;
-                markerEndBytes.copy(raw, markerEndPos);
+                raw = fullData;
             }
             if (opts.onProgress)
                 opts.onProgress({ phase: 'png_gen', loaded: 0, total: height });

package/dist/utils/optimization.js

@@ -112,6 +112,7 @@ export async function optimizePngBuffer(pngBuf, fast = false) {
             const row = raw.slice(rowStart, rowStart + rowBytes);
             let bestSum = Infinity;
             let bestFiltered = null;
+            let chosenFilter = 0;
             for (let f = 0; f <= 4; f++) {
                 const filtered = Buffer.alloc(rowBytes);
                 let sum = 0;
@@ -145,33 +146,10 @@ export async function optimizePngBuffer(pngBuf, fast = false) {
                 if (sum < bestSum) {
                     bestSum = sum;
                     bestFiltered = filtered;
-                }
-            }
-            const rowBuf = Buffer.alloc(1 + rowBytes);
-            let chosenFilter = 0;
-            for (let f = 0; f <= 4; f++) {
-                const filtered = Buffer.alloc(rowBytes);
-                for (let i = 0; i < rowBytes; i++) {
-                    const val = row[i];
-                    const left = i - bytesPerPixel >= 0 ? row[i - bytesPerPixel] : 0;
-                    const up = prevRow ? prevRow[i] : 0;
-                    const upLeft = prevRow && i - bytesPerPixel >= 0 ? prevRow[i - bytesPerPixel] : 0;
-                    if (f === 0)
-                        filtered[i] = val;
-                    else if (f === 1)
-                        filtered[i] = (val - left + 256) & 0xff;
-                    else if (f === 2)
-                        filtered[i] = (val - up + 256) & 0xff;
-                    else if (f === 3)
-                        filtered[i] = (val - Math.floor((left + up) / 2) + 256) & 0xff;
-                    else
-                        filtered[i] = (val - paethPredict(left, up, upLeft) + 256) & 0xff;
-                }
-                if (filtered.equals(bestFiltered)) {
                     chosenFilter = f;
-                    break;
                 }
             }
+            const rowBuf = Buffer.alloc(1 + rowBytes);
             rowBuf[0] = chosenFilter;
             bestFiltered.copy(rowBuf, 1);
             outRows.push(rowBuf);

package/dist/utils/zstd.js

@@ -45,7 +45,7 @@ export async function compressStream(stream, level = 19, onProgress, dict) {
     };
 }
 export async function parallelZstdCompress(payload, level = 19, onProgress, dict) {
-    const chunkSize = 8 * 1024 * 1024;
+    const chunkSize = 32 * 1024 * 1024;
     // For small payloads (< chunkSize), concatenate and compress as single frame
     // to avoid multi-chunk overhead (16+ bytes header per chunk boundary).
     let flat = null;

package/native/bwt.rs

@@ -1,56 +1,56 @@
-use anyhow::Result;
-use libsais::bwt::Bwt;
-use libsais::typestate::OwnedBuffer;
-use libsais::BwtConstruction;
-use rayon::prelude::*;
-
-pub struct BwtResult {
-    pub transformed: Vec<u8>,
-    pub primary_index: u32,
-}
-
-pub fn bwt_encode(data: &[u8]) -> Result<BwtResult> {
-    let n = data.len();
-    if n == 0 {
-        return Ok(BwtResult { transformed: Vec::new(), primary_index: 0 });
-    }
-
-    let bwt_result = BwtConstruction::for_text(data)
-        .with_owned_temporary_array_buffer32()
-        .single_threaded()
-        .run()
-        .map_err(|e| anyhow::anyhow!("libsais BWT: {:?}", e))?;
-
-    let primary_index = bwt_result.primary_index() as u32;
-    let transformed = bwt_result.bwt().to_vec();
-
-    Ok(BwtResult { transformed, primary_index })
-}
-
-pub fn bwt_decode(bwt_data: &[u8], primary_index: u32) -> Result<Vec<u8>> {
-    if bwt_data.is_empty() {
-        return Ok(Vec::new());
-    }
-
-    let bwt_obj: Bwt<'static, u8, OwnedBuffer> =
-        unsafe { Bwt::from_parts(bwt_data.to_vec(), primary_index as usize) };
-
-    let text = bwt_obj
-        .unbwt()
-        .with_owned_temporary_array_buffer32()
-        .single_threaded()
-        .run()
-        .map_err(|e| anyhow::anyhow!("libsais UnBWT: {:?}", e))?;
-
-    Ok(text.as_slice().to_vec())
-}
-
-pub fn bwt_encode_streaming(block_size: usize, data: &[u8]) -> Result<Vec<(BwtResult, usize)>> {
-    data.par_chunks(block_size)
-        .enumerate()
-        .map(|(i, chunk)| {
-            let result = bwt_encode(chunk)?;
-            Ok((result, i * block_size))
-        })
-        .collect()
-}
+use anyhow::Result;
+use libsais::bwt::Bwt;
+use libsais::typestate::OwnedBuffer;
+use libsais::BwtConstruction;
+use rayon::prelude::*;
+
+pub struct BwtResult {
+    pub transformed: Vec<u8>,
+    pub primary_index: u32,
+}
+
+pub fn bwt_encode(data: &[u8]) -> Result<BwtResult> {
+    let n = data.len();
+    if n == 0 {
+        return Ok(BwtResult { transformed: Vec::new(), primary_index: 0 });
+    }
+
+    let bwt_result = BwtConstruction::for_text(data)
+        .with_owned_temporary_array_buffer32()
+        .single_threaded()
+        .run()
+        .map_err(|e| anyhow::anyhow!("libsais BWT: {:?}", e))?;
+
+    let primary_index = bwt_result.primary_index() as u32;
+    let transformed = bwt_result.bwt().to_vec();
+
+    Ok(BwtResult { transformed, primary_index })
+}
+
+pub fn bwt_decode(bwt_data: &[u8], primary_index: u32) -> Result<Vec<u8>> {
+    if bwt_data.is_empty() {
+        return Ok(Vec::new());
+    }
+
+    let bwt_obj: Bwt<'static, u8, OwnedBuffer> =
+        unsafe { Bwt::from_parts(bwt_data.to_vec(), primary_index as usize) };
+
+    let text = bwt_obj
+        .unbwt()
+        .with_owned_temporary_array_buffer32()
+        .single_threaded()
+        .run()
+        .map_err(|e| anyhow::anyhow!("libsais UnBWT: {:?}", e))?;
+
+    Ok(text.as_slice().to_vec())
+}
+
+pub fn bwt_encode_streaming(block_size: usize, data: &[u8]) -> Result<Vec<(BwtResult, usize)>> {
+    data.par_chunks(block_size)
+        .enumerate()
+        .map(|(i, chunk)| {
+            let result = bwt_encode(chunk)?;
+            Ok((result, i * block_size))
+        })
+        .collect()
+}