roxify 1.13.5 → 1.13.7

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.13.5"
+version = "1.13.6"
 edition = "2021"
 publish = false
 

package/README.md

@@ -57,40 +57,39 @@ The core compression and image-processing logic is written in Rust and exposed t
 
 ## Benchmarks
 
-All measurements below use Roxify native Rust CLI (`roxify_native`) with streaming directory packing, Zstd L3, multi-threading, long-distance matching, and `window_log(30)`.
-
-### Cold-cache throughput on ext4
-
-Measured with targeted page-cache eviction (`POSIX_FADV_DONTNEED`) before both encode and decode. Raw manifest lives in `docs/COLD_BENCHMARK_2026-04-15.json`.
-
-| Dataset | Files | Source | Output PNG | Encode | Encode throughput | Decode | Decode throughput |
-| --- | --- | --- | --- | --- | --- | --- | --- |
-| Glados-Disc | 19,645 | 208.18 MiB | 54.83 MiB | 2.883 s | 72.22 MiB/s | 0.954 s | 218.16 MiB/s |
-| Gmod | 3,936 | 1.36 GiB | 411.09 MiB | 6.127 s | 227.69 MiB/s | 5.850 s | 238.48 MiB/s |
-
-### High-latency source filesystem encode
-
-Roxify 1.13.4 adds adaptive parallel preload for small files before feeding Zstd. This specifically targets metadata-heavy trees on slower filesystems such as NTFS, APFS, exFAT, and network-backed mounts.
-
-| Dataset | Source FS | Before 1.13.4 | Roxify 1.13.4 | Speedup |
-| --- | --- | --- | --- | --- |
-| Glados-Disc (19,645 files) | NTFS under Linux | 81.608 s | 2.189 s | 37.3x |
-| Gmod (3,936 files) | NTFS under Linux | 22.578 s | 4.517 s | 5.0x |
-
-### Portal 2 comparative reference: ZIP vs PNG
-
-Measured on the full `Portal 2` game directory (`3,731 files`, `193 folders`, `12.83 GiB` logical source) to compare classic ZIP packaging against Roxify PNG packing on the same dataset.
-
-| Format | Time (s) | Time (min:sec) | Throughput | Compression ratio |
-| --- | ---: | --- | ---: | ---: |
-| ZIP Encode | 633,87 | 10 min 33 s | 21,73 Mo/s | 36,08% |
-| ZIP Decode | 232,88 | 3 min 52 s | 59,15 Mo/s | - |
-| PNG Encode | 157,80 | 2 min 37 s | 87,30 Mo/s | 41,09% |
-| PNG Decode | 156,00 | 2 min 36 s | 88,30 Mo/s | - |
+All measurements below use Roxify native Rust CLI (`roxify_native`) against `zip -qry` / `unzip -qq`, with targeted page-cache eviction (`POSIX_FADV_DONTNEED`) before both encode and decode. `Saved` = `100 - final_size / source_size`. ZIP runs preserve symlinks so extracted trees stay logically identical to source.
+
+### Comparative archive benchmark on ext4
+
+| Dataset / Format | Files | Source | Final size | Saved | Encode | Encode throughput | Decode | Decode throughput |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| **Glados-Disc** | **19,645** | **208.18 MiB** | - | - | - | - | - | - |
+| PNG (Roxify) | - | - | 54.83 MiB | 73.66% | 1.63 s | 127.36 MiB/s | 1.98 s | 104.94 MiB/s |
+| ZIP | - | - | 82.44 MiB | 60.40% | 13.27 s | 15.69 MiB/s | 2.68 s | 77.69 MiB/s |
+| **Gmod** | **3,936** | **1.36 GiB** | - | - | - | - | - | - |
+| PNG (Roxify) | - | - | 411.09 MiB | 70.53% | 7.06 s | 197.59 MiB/s | 8.14 s | 171.29 MiB/s |
+| ZIP | - | - | 516.44 MiB | 62.98% | 44.07 s | 31.66 MiB/s | 12.51 s | 111.54 MiB/s |
+| **Portal 2** | **3,731** | **12.83 GiB** | - | - | - | - | - | - |
+| PNG (Roxify) | - | - | 7.62 GiB | 40.60% | 1 min 33.07 s | 141.16 MiB/s | 2 min 07.51 s | 103.03 MiB/s |
+| ZIP | - | - | 8.20 GiB | 36.08% | 9 min 00.66 s | 24.30 MiB/s | 3 min 18.63 s | 66.14 MiB/s |
+
+### Comparative archive benchmark on NTFS
+
+| Dataset / Format | Files | Source | Final size | Saved | Encode | Encode throughput | Decode | Decode throughput |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| **Glados-Disc** | **19,645** | **208.18 MiB** | - | - | - | - | - | - |
+| PNG (Roxify) | - | - | 54.64 MiB | 73.75% | 1 min 11.55 s | 2.91 MiB/s | 3.90 s | 53.31 MiB/s |
+| ZIP | - | - | 82.44 MiB | 60.40% | 1 min 55.28 s | 1.81 MiB/s | 11.99 s | 17.36 MiB/s |
+| **Gmod** | **3,936** | **1.36 GiB** | - | - | - | - | - | - |
+| PNG (Roxify) | - | - | 409.10 MiB | 70.67% | 19.68 s | 70.87 MiB/s | 22.47 s | 62.08 MiB/s |
+| ZIP | - | - | 516.45 MiB | 62.98% | 57.07 s | 24.44 MiB/s | 33.86 s | 41.19 MiB/s |
+| **Portal 2** | **3,731** | **12.83 GiB** | - | - | - | - | - | - |
+| PNG (Roxify) | - | - | 7.56 GiB | 41.09% | 2 min 40.95 s | 81.62 MiB/s | 3 min 13.14 s | 68.02 MiB/s |
+| ZIP | - | - | 8.20 GiB | 36.08% | 10 min 58.95 s | 19.94 MiB/s | 4 min 01.80 s | 54.33 MiB/s |
 
 ### Data integrity
 
-All benchmark runs completed with byte-exact roundtrip validation. Decode output matched original logical source bytes on every dataset.
+All benchmark runs completed with successful roundtrip extraction on the measured datasets. ZIP runs used `-y` to preserve symlinks instead of dereferencing them during archive creation.
 
 ---
 

package/dist/cli.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { mkdirSync, readdirSync, readFileSync, statSync, writeFileSync, } from 'fs';
+import { readdirSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { open } from 'fs/promises';
 import { basename, dirname, join, resolve } from 'path';
 import * as cliProgress from './stub-progress.js';
@@ -20,7 +20,7 @@ async function loadJsEngine() {
         VFSIndexEntry: undefined,
     };
 }
-const VERSION = '1.13.2';
+const VERSION = '1.13.6';
 function getDirectorySize(dirPath) {
     let totalSize = 0;
     try {
@@ -86,6 +86,7 @@ Options:
   -e, --encrypt <type>      auto|aes|xor|none
   --no-compress             Disable compression
   --dict <file>             Use zstd dictionary when compressing
+  --ram-budget-mb <mb>      Max RAM budget used by native encode/decode paths
   --force-ts                Force TypeScript encoder (slower but supports encryption)
   -o, --output <path>       Output file path
   -s, --sizes               Show file sizes in 'list' output (default)
@@ -196,6 +197,15 @@ function parseArgs(args) {
                 parsed.dict = args[i + 1];
                 i += 2;
             }
+            else if (key === 'ram-budget-mb') {
+                const v = Number(args[i + 1]);
+                if (!Number.isFinite(v) || v <= 0) {
+                    console.error(`Invalid --ram-budget-mb: ${args[i + 1]}`);
+                    process.exit(1);
+                }
+                parsed.ramBudgetMb = Math.floor(v);
+                i += 2;
+            }
             else {
                 const value = args[i + 1];
                 parsed[key] = value;
@@ -320,7 +330,7 @@ async function encodeCommand(args) {
             encodeBar.start(100, 0, { step: 'Encoding', elapsed: '0' });
             const encryptType = parsed.encrypt === 'xor' ? 'xor' : 'aes';
             const fileName = basename(inputPaths[0]);
-            await encodeWithRustCLI(inputPaths.length === 1 ? resolvedInputs[0] : resolvedInputs[0], resolvedOutput, 19, parsed.passphrase, encryptType, fileName, (current, total, step) => {
+            await encodeWithRustCLI(inputPaths.length === 1 ? resolvedInputs[0] : resolvedInputs[0], resolvedOutput, 19, parsed.passphrase, encryptType, fileName, parsed.ramBudgetMb, (current, total, step) => {
                 const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
                 const elapsed = Math.floor((Date.now() - startTime) / 1000);
                 encodeBar.update(Math.min(pct, 99), {
@@ -588,204 +598,36 @@ async function decodeCommand(args) {
     }
     const resolvedInput = resolve(inputPath);
     const resolvedOutput = parsed.output || outputPath || '.';
-    if (isRustBinaryAvailable() && !parsed.forceTs && !parsed.lossyResilient) {
-        try {
-            console.log(' ');
-            console.log('Decoding... (Using native Rust decoder)\n');
-            const startTime = Date.now();
-            const decodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
-            decodeBar.start(100, 0, { step: 'Decoding', elapsed: '0' });
-            await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict, (current, total, step) => {
-                const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
-                const elapsed = Math.floor((Date.now() - startTime) / 1000);
-                decodeBar.update(Math.min(pct, 99), {
-                    step: step || 'Decoding',
-                    elapsed: String(elapsed),
-                });
-            });
-            const decodeTime = Date.now() - startTime;
-            decodeBar.update(100, { step: 'done', elapsed: String(Math.floor(decodeTime / 1000)) });
-            decodeBar.stop();
-            console.log(`\nSuccess!`);
-            console.log(`  Time: ${decodeTime}ms`);
-            console.log(`  Output: ${resolve(resolvedOutput)}`);
-            console.log(' ');
-            return;
-        }
-        catch (err) {
-            console.warn('\nRust decoder failed, falling back to TypeScript decoder...');
-            console.warn(`Reason: ${err.message}\n`);
-        }
+    if (!isRustBinaryAvailable()) {
+        console.error('Error: Rust decoder binary not found');
+        process.exit(1);
     }
     try {
-        const options = {};
-        if (parsed.passphrase) {
-            options.passphrase = parsed.passphrase;
-        }
-        if (parsed.debug) {
-            options.debugDir = dirname(resolvedInput);
-        }
-        if (parsed.files) {
-            options.files = parsed.files;
-        }
-        if (parsed.dict) {
-            try {
-                options.dict = readFileSync(parsed.dict);
-            }
-            catch (e) {
-                console.error(`Failed to read dictionary file: ${parsed.dict}`);
-                process.exit(1);
-            }
-        }
-        console.log(' ');
-        console.log(`Decoding...`);
         console.log(' ');
-        const decodeBar = new cliProgress.SingleBar({
-            format: ' {bar} {percentage}% | {step} | {elapsed}s',
-        }, cliProgress.Presets.shades_classic);
-        let barStarted = false;
-        const startDecode = Date.now();
-        let currentPct = 0;
-        let targetPct = 0;
-        let currentStep = 'Decoding';
-        const heartbeat = setInterval(() => {
-            if (currentPct < targetPct) {
-                currentPct = Math.min(currentPct + 2, targetPct);
-            }
-            if (!barStarted && targetPct > 0) {
-                decodeBar.start(100, Math.floor(currentPct), {
-                    step: currentStep,
-                    elapsed: String(Math.floor((Date.now() - startDecode) / 1000)),
-                });
-                barStarted = true;
-            }
-            else if (barStarted) {
-                decodeBar.update(Math.floor(currentPct), {
-                    step: currentStep,
-                    elapsed: String(Math.floor((Date.now() - startDecode) / 1000)),
-                });
-            }
-        }, 100);
-        options.onProgress = (info) => {
-            if (info.phase === 'decompress_start') {
-                targetPct = 50;
-                currentStep = 'Decompressing';
-            }
-            else if (info.phase === 'decompress_progress' &&
-                info.loaded &&
-                info.total) {
-                targetPct = 50 + Math.floor((info.loaded / info.total) * 40);
-                currentStep = `Decompressing (${info.loaded}/${info.total})`;
-            }
-            else if (info.phase === 'decompress_done') {
-                targetPct = 90;
-                currentStep = 'Decompressed';
-            }
-            else if (info.phase === 'done') {
-                targetPct = 100;
-                currentStep = 'Done';
-            }
-        };
-        const inputBuffer = await readLargeFile(resolvedInput);
-        const js = await loadJsEngine();
-        const result = await js.decodePngToBinary(inputBuffer, options);
-        const decodeTime = Date.now() - startDecode;
-        clearInterval(heartbeat);
-        if (barStarted) {
-            currentPct = 100;
-            decodeBar.update(100, {
-                step: 'done',
-                elapsed: String(Math.floor(decodeTime / 1000)),
-            });
-            decodeBar.stop();
-        }
-        if (result.files) {
-            const baseDir = parsed.output || outputPath || '.';
-            const totalBytes = result.files.reduce((s, f) => s + f.buf.length, 0);
-            const extractBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
-            const extractStart = Date.now();
-            extractBar.start(totalBytes, 0, { step: 'Writing files', elapsed: '0' });
-            let written = 0;
-            for (const file of result.files) {
-                const fullPath = join(baseDir, file.path);
-                const dir = dirname(fullPath);
-                mkdirSync(dir, { recursive: true });
-                writeFileSync(fullPath, file.buf);
-                written += file.buf.length;
-                extractBar.update(written, {
-                    step: `Writing ${file.path}`,
-                    elapsed: String(Math.floor((Date.now() - extractStart) / 1000)),
-                });
-            }
-            extractBar.update(totalBytes, {
-                step: 'Done',
-                elapsed: String(Math.floor((Date.now() - extractStart) / 1000)),
+        console.log('Decoding... (Using native Rust decoder)\n');
+        const startTime = Date.now();
+        const decodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
+        decodeBar.start(100, 0, { step: 'Decoding', elapsed: '0' });
+        await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict, parsed.ramBudgetMb, (current, total, step) => {
+            const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
+            const elapsed = Math.floor((Date.now() - startTime) / 1000);
+            decodeBar.update(Math.min(pct, 99), {
+                step: step || 'Decoding',
+                elapsed: String(elapsed),
             });
-            extractBar.stop();
-            console.log(`\nSuccess!`);
-            console.log(`Unpacked ${result.files.length} files to directory : ${resolve(baseDir)}`);
-            console.log(`Time: ${decodeTime}ms`);
-        }
-        else if (result.buf) {
-            const unpacked = js.unpackBuffer(result.buf);
-            if (unpacked) {
-                const baseDir = parsed.output || outputPath || '.';
-                for (const file of unpacked.files) {
-                    const fullPath = join(baseDir, file.path);
-                    const dir = dirname(fullPath);
-                    mkdirSync(dir, { recursive: true });
-                    writeFileSync(fullPath, file.buf);
-                }
-                console.log(`\nSuccess!`);
-                console.log(`Time: ${decodeTime}ms`);
-                console.log(`Unpacked ${unpacked.files.length} files to current directory`);
-            }
-            else {
-                let finalOutput = resolvedOutput;
-                if (!parsed.output && !outputPath && result.meta?.name) {
-                    finalOutput = result.meta.name;
-                }
-                writeFileSync(finalOutput, result.buf);
-                console.log(`\nSuccess!`);
-                if (result.meta?.name) {
-                    console.log(`  Original name: ${result.meta.name}`);
-                }
-                const outputSize = (result.buf.length / 1024 / 1024).toFixed(2);
-                console.log(`  Output size:   ${outputSize} MB`);
-                console.log(`  Time:          ${decodeTime}ms`);
-                console.log(`  Saved:         ${finalOutput}`);
-            }
-        }
-        else {
-            console.log(`\nSuccess!`);
-            console.log(`Time: ${decodeTime}ms`);
-        }
+        });
+        const decodeTime = Date.now() - startTime;
+        decodeBar.update(100, { step: 'done', elapsed: String(Math.floor(decodeTime / 1000)) });
+        decodeBar.stop();
+        console.log(`\nSuccess!`);
+        console.log(`  Time: ${decodeTime}ms`);
+        console.log(`  Output: ${resolve(resolvedOutput)}`);
         console.log(' ');
     }
     catch (err) {
-        if ((err.message && err.message.includes('passphrase required')) ||
-            (err.message && err.message.includes('passphrase') && !parsed.passphrase)) {
-            console.log(' ');
-            console.error('File appears to be encrypted. Provide a passphrase with -p');
-        }
-        else if ((err.message && err.message.includes('Incorrect passphrase')) ||
-            (err.message && err.message.includes('Incorrect passphrase'))) {
-            console.log(' ');
-            console.error('Incorrect passphrase');
-        }
-        else if ((err.message && err.message.includes('data format error')) ||
-            (err.message &&
-                (err.message.includes('decompression failed') ||
-                    err.message.includes('missing ROX1') ||
-                    err.message.includes('Pixel payload truncated') ||
-                    err.message.includes('Marker START not found')))) {
-            console.log(' ');
-            console.error('Data corrupted or unsupported format. Use --verbose for details.');
-        }
-        else {
-            console.log(' ');
-            console.error('Failed to decode file. Use --verbose for details.');
-        }
+        console.log(' ');
+        console.error('Error: Rust decoder failed.');
+        console.error(`Reason: ${err.message}`);
         if (parsed.verbose) {
             console.error('Details:', err.stack || err.message);
         }

package/dist/utils/decoder.d.ts

@@ -1,30 +1,10 @@
 import { DecodeOptions, DecodeResult } from './types.js';
-/**
- * Un-stretch an image that was nearest-neighbor scaled.
- * 1. Crops to non-background bounding box
- * 2. Collapses horizontal runs of identical pixels into single pixels
- * 3. Removes duplicate consecutive rows
- *
- * Returns null if the image doesn't appear to be stretched.
- */
-export declare function unstretchImage(rawRGB: Buffer, width: number, height: number, tolerance?: number): {
-    data: Buffer;
-    width: number;
-    height: number;
-} | null;
 /**
  * Decode a ROX PNG or buffer into the original binary payload or files list.
+ * This function uses the Rust native implementation exclusively.
  *
  * @param input - Buffer or path to a PNG file.
  * @param opts - Optional decode options.
  * @returns A Promise resolving to DecodeResult ({ buf, meta } or { files }).
- *
- * @example
- * ```js
- * import { decodePngToBinary } from 'roxify';
- * const png = fs.readFileSync('out.png');
- * const res = await decodePngToBinary(png);
- * console.log(res.meta?.name, res.buf.toString('utf8'));
- * ```
  */
 export declare function decodePngToBinary(input: Buffer | string, opts?: DecodeOptions): Promise<DecodeResult>;