roxify 1.13.1 → 1.13.3

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.13.1"
+version = "1.13.3"
 edition = "2021"
 publish = false
 
@@ -51,26 +51,17 @@ pbkdf2 = "0.12"
 rand = "0.8"
 sha2 = "0.10"
 mimalloc = "0.1"
+simd-adler32 = "0.3"
 
-wgpu = { version = "0.19", optional = true }
 memmap2 = "0.9"
 bytemuck = { version = "1.14", features = ["derive"] }
-# tokio is optional now; enable via the 'async' feature
 tokio = { version = "1", features = ["sync", "rt"], optional = true }
 parking_lot = "0.12"
-pollster = { version = "0.3", optional = true }
 libsais = { version = "0.2.0", default-features = false }
 
 [features]
-# default is intentionally empty so the crate compiles fast for local checks.
-# Enable 'gpu' to pull in the WGPU and pollster dependencies (heavy).
-# Enable 'async' to include tokio runtime (optional).
-# Example: `cargo build -p roxify_native --features gpu`
 default = []
-
-gpu = ["wgpu", "pollster"]
 async = ["tokio"]
-full = ["gpu", "async"]
 
 [profile.release]
 opt-level = 3

package/README.md

@@ -57,151 +57,27 @@ The core compression and image-processing logic is written in Rust and exposed t
 
 ## Benchmarks
 
-All measurements were taken on Linux x64 (Intel i7-6700K @ 4.0 GHz, 32 GB RAM) with Node.js v20. Every tool uses its **maximum compression** setting: zip -9, gzip -9, 7z LZMA2 -mx=9, and Roxify Zstd level 19. Roxify produces a valid PNG or WAV file rather than a raw archive.
-
-### BWT-ANS Native Compression (Rust, via N-API)
-
-Direct API calls through the native module (BWT → MTF → RLE0 → rANS, 1 MB blocks):
-
-| Dataset | Original | Compressed | Ratio | Encode | Decode | Enc Throughput | Dec Throughput |
-| ------- | -------- | ---------- | ----- | ------ | ------ | -------------- | -------------- |
-| Repetitive text 45 KB | 45 KB | 207 B | 0.5% | < 1 ms | < 1 ms | — | — |
-| Rust source 6 KB | 6 KB | 2.0 KB | 33.7% | < 1 ms | < 1 ms | — | — |
-| node_modules tar 175 MB | 175.5 MB | 38.4 MB | 21.9% | 9.68 s | 5.62 s | 18.1 MB/s | 31.2 MB/s |
-| Random 100 KB | 100 KB | 101 KB | 101.5% | < 1 ms | < 1 ms | — | — |
-| Zeros 100 KB | 100 KB | 51 B | 0.1% | < 1 ms | < 1 ms | — | — |
-
-> BWT-ANS achieves **21.9% on real-world node_modules** (175 MB), with **18 MB/s encode** and **31 MB/s decode** throughput. 100% lossless roundtrip verified on all datasets.
-
-### Compression Ratio (Maximum Compression for All Tools)
-
-| Dataset     | Original | zip -9          | gzip -9         | 7z LZMA2 -9     | Roxify PNG          | Roxify WAV          |
-| ----------- | -------- | --------------- | --------------- | --------------- | ------------------- | ------------------- |
-| Text 1 MB   | 1.00 MB  | 219 KB (21.4%)  | 219 KB (21.4%)  | 187 KB (18.3%)  | **188 KB (18.3%)**  | **187 KB (18.3%)**  |
-| JSON 1 MB   | 1.00 MB  | 263 KB (25.7%)  | 263 KB (25.7%)  | 225 KB (22.0%)  | **220 KB (21.5%)**  | **219 KB (21.4%)**  |
-| Binary 1 MB | 1.00 MB  | 1.00 MB (100%)  | 1.00 MB (100%)  | 1.00 MB (100%)  | 1.00 MB (100%)      | 1.00 MB (100%)      |
-| Mixed 5 MB  | 5.00 MB  | 2.45 MB (49.0%) | 2.45 MB (49.1%) | 2.33 MB (46.6%) | 2.38 MB (47.6%)     | 2.38 MB (47.6%)     |
-| Text 10 MB  | 10.00 MB | 2.13 MB (21.3%) | 2.13 MB (21.3%) | 1.71 MB (17.1%) | **1.71 MB (17.1%)** | **1.70 MB (17.0%)** |
-| Mixed 10 MB | 10.00 MB | 4.90 MB (49.0%) | 4.90 MB (49.0%) | 4.65 MB (46.5%) | 4.73 MB (47.3%)     | 4.73 MB (47.3%)     |
-
-> **Roxify matches 7z LZMA2 ultra-compression on text** (18.3% for both at 1 MB) and **beats LZMA2 on JSON** (21.4% vs 22.0%). On mixed data, Roxify is within 1 percentage point of LZMA2 while producing a shareable PNG/WAV instead of an archive.
-
-### Encode and Decode Speed (CLI)
-
-| Dataset    | Tool           | Encode     | Decode     | Enc Throughput | Dec Throughput |
-| ---------- | -------------- | ---------- | ---------- | -------------- | -------------- |
-| Text 1 MB  | zip -9         | 112 ms     | 36 ms      | 8.9 MB/s       | 27.6 MB/s      |
-|            | gzip -9        | 146 ms     | 38 ms      | 6.9 MB/s       | 26.0 MB/s      |
-|            | 7z LZMA -9     | 303 ms     | 21 ms      | 3.3 MB/s       | 46.6 MB/s      |
-|            | **Roxify PNG** | **859 ms** | **577 ms** | **1.2 MB/s**   | **1.7 MB/s**   |
-|            | **Roxify WAV** | **794 ms** | **480 ms** | **1.3 MB/s**   | **2.1 MB/s**   |
-| JSON 1 MB  | zip -9         | 79 ms      | 20 ms      | 12.7 MB/s      | 50.5 MB/s      |
-|            | 7z LZMA -9     | 197 ms     | 26 ms      | 5.1 MB/s       | 37.9 MB/s      |
-|            | **Roxify PNG** | **1.14 s** | **755 ms** | **0.9 MB/s**   | **1.3 MB/s**   |
-|            | **Roxify WAV** | **1.49 s** | **518 ms** | **0.7 MB/s**   | **1.9 MB/s**   |
-| Text 10 MB | zip -9         | 1.21 s     | 70 ms      | 8.2 MB/s       | 143.8 MB/s     |
-|            | 7z LZMA -9     | 5.05 s     | 99 ms      | 2.0 MB/s       | 100.8 MB/s     |
-|            | **Roxify PNG** | **9.05 s** | **4.53 s** | **1.1 MB/s**   | **2.2 MB/s**   |
-|            | **Roxify WAV** | **9.22 s** | **2.59 s** | **1.1 MB/s**   | **3.9 MB/s**   |
-
-> Roxify CLI includes Node.js startup overhead (~400 ms). In the JS API (below), the same operations are significantly faster. WAV decode is consistently faster than PNG decode due to simpler container parsing.
-
-### JavaScript API Throughput
-
-Direct API calls (no CLI startup overhead):
-
-| Size   | Container | Encode | Decode  | Enc Throughput | Dec Throughput | Output    | Ratio  | Integrity |
-| ------ | --------- | ------ | ------- | -------------- | -------------- | --------- | ------ | --------- |
-| 1 KB   | PNG       | 9 ms   | 12 ms   | 0.1 MB/s       | 0.1 MB/s       | 1.14 KB   | 114.3% | ✓         |
-| 10 KB  | PNG       | 18 ms  | 34 ms   | 0.5 MB/s       | 0.3 MB/s       | 10.32 KB  | 103.2% | ✓         |
-| 100 KB | PNG       | 52 ms  | 109 ms  | 1.9 MB/s       | 0.9 MB/s       | 100.52 KB | 100.5% | ✓         |
-| 500 KB | PNG       | 339 ms | 541 ms  | 1.4 MB/s       | 0.9 MB/s       | 502.64 KB | 100.5% | ✓         |
-| 1 MB   | PNG       | 875 ms | 1.24 s  | 1.1 MB/s       | 0.8 MB/s       | 1.00 MB   | 100.3% | ✓         |
-| 5 MB   | PNG       | 3.39 s | 4.12 s  | 1.5 MB/s       | 1.2 MB/s       | 5.01 MB   | 100.2% | ✓         |
-| 10 MB  | PNG       | 6.84 s | 12.28 s | 1.5 MB/s       | 0.8 MB/s       | 10.01 MB  | 100.1% | ✓         |
-| 1 KB   | WAV       | 2 ms   | 2 ms    | 0.6 MB/s       | 0.6 MB/s       | 1.08 KB   | 107.5% | ✓         |
-| 10 KB  | WAV       | 4 ms   | 5 ms    | 2.3 MB/s       | 1.8 MB/s       | 10.08 KB  | 100.8% | ✓         |
-| 100 KB | WAV       | 39 ms  | 28 ms   | 2.5 MB/s       | 3.5 MB/s       | 100.08 KB | 100.1% | ✓         |
-| 500 KB | WAV       | 172 ms | 190 ms  | 2.8 MB/s       | 2.6 MB/s       | 500.09 KB | 100.0% | ✓         |
-| 1 MB   | WAV       | 452 ms | 276 ms  | 2.2 MB/s       | 3.6 MB/s       | 1.00 MB   | 100.0% | ✓         |
-| 5 MB   | WAV       | 2.70 s | 1.65 s  | 1.8 MB/s       | 3.0 MB/s       | 5.00 MB   | 100.0% | ✓         |
-| 10 MB  | WAV       | 4.81 s | 2.56 s  | 2.1 MB/s       | 3.9 MB/s       | 10.00 MB  | 100.0% | ✓         |
-
-> WAV container is **2–4× faster** than PNG for decoding at large sizes, and produces slightly smaller output thanks to simpler framing.
-
-### Reed-Solomon ECC Throughput
-
-| Size   | Encode | Decode | Enc Throughput | Dec Throughput | Overhead |
-| ------ | ------ | ------ | -------------- | -------------- | -------- |
-| 1 KB   | 6 ms   | 4 ms   | 0.2 MB/s       | 0.2 MB/s       | 125.7%   |
-| 10 KB  | 7 ms   | 6 ms   | 1.3 MB/s       | 1.5 MB/s       | 119.6%   |
-| 100 KB | 49 ms  | 45 ms  | 2.0 MB/s       | 2.1 MB/s       | 118.8%   |
-| 1 MB   | 483 ms | 377 ms | 2.1 MB/s       | 2.7 MB/s       | 118.6%   |
-
-### Lossy-Resilient Encoding
-
-#### Robust Image (QR-code-style, block size 4×4)
-
-| Data Size | Encode Time | Output (PNG) |
-| --------- | ----------- | ------------ |
-| 32 B      | 32 ms       | 122 KB       |
-| 128 B     | 39 ms       | 122 KB       |
-| 512 B     | 76 ms       | 316 KB       |
-| 1 KB      | 139 ms      | 508 KB       |
-| 2 KB      | 251 ms      | 986 KB       |
-
-#### Robust Audio (MFSK 8-channel, medium ECC)
-
-| Data Size | Encode | Decode | Output (WAV) | Integrity |
-| --------- | ------ | ------ | ------------ | --------- |
-| 10 B      | 33 ms  | 44 ms  | 1.35 MB      | ✓         |
-| 32 B      | 19 ms  | 31 ms  | 1.35 MB      | ✓         |
-| 64 B      | 22 ms  | 24 ms  | 1.35 MB      | ✓         |
-| 128 B     | 21 ms  | 28 ms  | 1.35 MB      | ✓         |
-| 256 B     | 40 ms  | 45 ms  | 2.59 MB      | ✓         |
-
-### Data Integrity Verification
-
-All encode/decode roundtrips produce bit-exact output, verified by SHA-256:
-
-| Test Case               | PNG | WAV |
-| ----------------------- | --- | --- |
-| Empty buffer (0 B)      | ✓   | ✓   |
-| Single byte (1 B)       | ✓   | ✓   |
-| All byte values (256 B) | ✓   | ✓   |
-| 1 KB text               | ✓   | ✓   |
-| 100 KB random           | ✓   | ✓   |
-| 1 MB random             | ✓   | ✓   |
-| 5 MB random             | ✓   | ✓   |
-
-**14 / 14 integrity tests passed** across both containers.
-
-### Key Observations
-
-- **Roxify matches LZMA2 ultra-compression** on text data (18.3%) and **outperforms it on JSON** (21.4% vs 22.0%), while producing a standard PNG or WAV file instead of an archive.
-- **WAV container decode is 2–4× faster** than PNG decode at large sizes (3.9 MB/s vs 0.8 MB/s for 10 MB).
-- **WAV encode for 1 KB data completes in 2 ms** — well under the sub-second target.
-- **Lossy-resilient audio** encode/decode completes in under 50 ms for data up to 256 bytes, with full integrity.
-- **100% data integrity** across all sizes and containers — every byte is recovered exactly.
-- The CLI overhead (~400 ms Node.js startup) is amortized on larger inputs. For programmatic use, the JS API eliminates this entirely.
-- On incompressible (random) data, all tools converge to ~100% as expected. No compression algorithm can shrink truly random data.
-
-### Methodology
-
-Benchmarks were generated using `test/benchmark-detailed.cjs`. Datasets consist of procedurally generated text, JSON, and random binary data. Each tool was invoked with its maximum compression setting:
-
-| Tool     | Command / Setting           |
-| -------- | --------------------------- |
-| zip      | `zip -r -q -9`              |
-| tar/gzip | `tar -cf - \| gzip -9`      |
-| 7z       | `7z a -mx=9` (LZMA2 ultra)  |
-| Roxify   | Zstd level 19, compact mode |
-
-To reproduce:
+All measurements taken on Linux x64 (Intel i7-6700K @ 4.0 GHz, 32 GB RAM). Roxify uses its native Rust CLI (`roxify_native`) with streaming Zstd L3 + multi-threaded + LDM + window_log(30). ZIP uses `zip -r -q -9` (maximum compression).
 
-```bash
-node test/benchmark-detailed.cjs
-```
+### Real-world directory encoding: Roxify vs ZIP
+
+| Dataset | Original | ZIP -9 | Roxify PNG | ZIP time | Roxify time | Speedup |
+| --- | --- | --- | --- | --- | --- | --- |
+| Test A (19 638 files, 177 MB) | 177 MB | 87.7 MB (49.6%) | 54.9 MB (31.0%) | 17.6 s | 1.2 s | 14.7x |
+| Test B (3 936 files, 1.4 GB) | 1.4 GB | 513 MB (36.7%) | 409 MB (29.2%) | 1 min 46 s | 6.7 s | 15.9x |
+
+### Decompression
+
+| Dataset | unzip | Roxify decode | Speedup |
+| --- | --- | --- | --- |
+| Test A (177 MB) | 2.4 s | 1.3 s | 1.8x |
+| Test B (1.4 GB) | 8.4 s | 2.9 s | 2.9x |
+
+Roxify produces a valid PNG image instead of a ZIP archive. On these real-world datasets it compresses 20-37% smaller than ZIP -9 while encoding 15x faster, thanks to multi-threaded Zstd with long-distance matching.
+
+### Data integrity
+
+100% lossless roundtrip verified by byte-exact diff on all datasets. Start and end markers verified in every output PNG.
 
 ---
 
@@ -661,6 +537,8 @@ Input --> Detect Format --> Demodulate/Read Blocks --> De-interleave --> RS ECC
 | `image_utils.rs`    | Image resizing, pixel format conversion, metadata extraction           |
 | `png_utils.rs`      | Low-level PNG chunk read/write operations                              |
 | `progress.rs`       | Progress tracking for long-running compression/decompression           |
+| `streaming_encode.rs` | Streaming directory-to-PNG encoder with real-time progress           |
+| `streaming_decode.rs` | Streaming PNG-to-directory decoder with real-time progress           |
 
 ### TypeScript Modules
 

package/dist/cli.js

@@ -20,7 +20,7 @@ async function loadJsEngine() {
         VFSIndexEntry: undefined,
     };
 }
-const VERSION = '1.13.1';
+const VERSION = '1.13.2';
 function getDirectorySize(dirPath) {
     let totalSize = 0;
     try {
@@ -317,20 +317,17 @@ async function encodeCommand(args) {
             console.log(`Encoding to ${resolvedOutput} (Using native Rust encoder)\n`);
             const startTime = Date.now();
             const encodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
-            let barValue = 0;
             encodeBar.start(100, 0, { step: 'Encoding', elapsed: '0' });
-            const progressInterval = setInterval(() => {
-                barValue = Math.min(barValue + 1, 99);
+            const encryptType = parsed.encrypt === 'xor' ? 'xor' : 'aes';
+            const fileName = basename(inputPaths[0]);
+            await encodeWithRustCLI(inputPaths.length === 1 ? resolvedInputs[0] : resolvedInputs[0], resolvedOutput, 19, parsed.passphrase, encryptType, fileName, (current, total, step) => {
+                const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
                 const elapsed = Math.floor((Date.now() - startTime) / 1000);
-                encodeBar.update(barValue, {
-                    step: 'Encoding',
+                encodeBar.update(Math.min(pct, 99), {
+                    step: step || 'Encoding',
                     elapsed: String(elapsed),
                 });
-            }, 500);
-            const encryptType = parsed.encrypt === 'xor' ? 'xor' : 'aes';
-            const fileName = basename(inputPaths[0]);
-            await encodeWithRustCLI(inputPaths.length === 1 ? resolvedInputs[0] : resolvedInputs[0], resolvedOutput, 19, parsed.passphrase, encryptType, fileName);
-            clearInterval(progressInterval);
+            });
             const encodeTime = Date.now() - startTime;
             encodeBar.update(100, {
                 step: 'done',
@@ -597,17 +594,15 @@ async function decodeCommand(args) {
             console.log('Decoding... (Using native Rust decoder)\n');
             const startTime = Date.now();
             const decodeBar = new cliProgress.SingleBar({ format: ' {bar} {percentage}% | {step} | {elapsed}s' }, cliProgress.Presets.shades_classic);
-            let barValue = 0;
             decodeBar.start(100, 0, { step: 'Decoding', elapsed: '0' });
-            const progressInterval = setInterval(() => {
-                barValue = Math.min(barValue + 2, 99);
-                decodeBar.update(barValue, {
-                    step: 'Decoding',
-                    elapsed: String(Math.floor((Date.now() - startTime) / 1000)),
+            await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict, (current, total, step) => {
+                const pct = total > 0 ? Math.floor((current / total) * 100) : 0;
+                const elapsed = Math.floor((Date.now() - startTime) / 1000);
+                decodeBar.update(Math.min(pct, 99), {
+                    step: step || 'Decoding',
+                    elapsed: String(elapsed),
                 });
-            }, 300);
-            await decodeWithRustCLI(resolvedInput, resolvedOutput, parsed.passphrase, parsed.files, parsed.dict);
-            clearInterval(progressInterval);
+            });
             const decodeTime = Date.now() - startTime;
             decodeBar.update(100, { step: 'done', elapsed: String(Math.floor(decodeTime / 1000)) });
             decodeBar.stop();

package/dist/index.d.ts

@@ -12,7 +12,7 @@ export * from './utils/optimization.js';
 export * from './utils/reconstitution.js';
 export * from './utils/robust-audio.js';
 export * from './utils/robust-image.js';
-export { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI, } from './utils/rust-cli-wrapper.js';
+export { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI } from './utils/rust-cli-wrapper.js';
 export * from './utils/types.js';
 export * from './utils/zstd.js';
 export { packPaths, packPathsToParts, unpackBuffer } from './pack.js';

package/dist/index.js

@@ -12,7 +12,7 @@ export * from './utils/optimization.js';
 export * from './utils/reconstitution.js';
 export * from './utils/robust-audio.js';
 export * from './utils/robust-image.js';
-export { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI, } from './utils/rust-cli-wrapper.js';
+export { decodeWithRustCLI, encodeWithRustCLI, havepassphraseWithRustCLI, isRustBinaryAvailable, listWithRustCLI } from './utils/rust-cli-wrapper.js';
 export * from './utils/types.js';
 export * from './utils/zstd.js';
 export { packPaths, packPathsToParts, unpackBuffer } from './pack.js';

package/dist/utils/rust-cli-wrapper.d.ts

@@ -1,7 +1,8 @@
 declare function findRustBinary(): string | null;
 export { findRustBinary };
 export declare function isRustBinaryAvailable(): boolean;
-export declare function encodeWithRustCLI(inputPath: string, outputPath: string, compressionLevel?: number, passphrase?: string, encryptType?: 'aes' | 'xor', name?: string): Promise<void>;
-export declare function decodeWithRustCLI(inputPath: string, outputPath: string, passphrase?: string, files?: string[], dict?: string): Promise<void>;
+export type ProgressCallback = (current: number, total: number, step: string) => void;
+export declare function encodeWithRustCLI(inputPath: string, outputPath: string, compressionLevel?: number, passphrase?: string, encryptType?: 'aes' | 'xor', name?: string, onProgress?: ProgressCallback): Promise<void>;
+export declare function decodeWithRustCLI(inputPath: string, outputPath: string, passphrase?: string, files?: string[], dict?: string, onProgress?: ProgressCallback): Promise<void>;
 export declare function listWithRustCLI(inputPath: string): Promise<string>;
 export declare function havepassphraseWithRustCLI(inputPath: string): Promise<string>;

package/dist/utils/rust-cli-wrapper.js

@@ -162,9 +162,12 @@ function spawnRustCLI(args, options) {
         let stdout = '';
         const runSpawn = (exePath) => {
             let proc;
+            const hasProgress = !!options?.onProgress;
             const stdio = options?.collectStdout
-                ? ['pipe', 'pipe', 'inherit']
-                : 'inherit';
+                ? ['pipe', 'pipe', hasProgress ? 'pipe' : 'inherit']
+                : hasProgress
+                    ? ['pipe', 'inherit', 'pipe']
+                    : 'inherit';
             try {
                 proc = spawn(exePath, args, { stdio });
             }
@@ -184,6 +187,23 @@ function spawnRustCLI(args, options) {
             if (options?.collectStdout && proc.stdout) {
                 proc.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
             }
+            if (hasProgress && proc.stderr) {
+                let stderrBuf = '';
+                proc.stderr.on('data', (chunk) => {
+                    stderrBuf += chunk.toString();
+                    const lines = stderrBuf.split('\n');
+                    stderrBuf = lines.pop() || '';
+                    for (const line of lines) {
+                        const match = line.match(/^PROGRESS:(\d+):(\d+):(.+)$/);
+                        if (match) {
+                            options.onProgress(Number(match[1]), Number(match[2]), match[3]);
+                        }
+                        else if (line.trim()) {
+                            process.stderr.write(line + '\n');
+                        }
+                    }
+                });
+            }
             proc.on('error', (err) => {
                 if (!triedExtract) {
                     triedExtract = true;
@@ -213,7 +233,7 @@ function spawnRustCLI(args, options) {
         runSpawn(cliPath);
     });
 }
-export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name) {
+export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3, passphrase, encryptType = 'aes', name, onProgress) {
     const cliPath = findRustBinary();
     if (!cliPath)
         throw new Error('Rust CLI binary not found');
@@ -231,9 +251,9 @@ export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel
         args.push('--encrypt', encryptType);
     }
     args.push(inputPath, outputPath);
-    await spawnRustCLI(args);
+    await spawnRustCLI(args, { onProgress });
 }
-export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict) {
+export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files, dict, onProgress) {
     const args = ['decompress', inputPath, outputPath];
     if (passphrase)
         args.push('--passphrase', passphrase);
@@ -241,7 +261,7 @@ export async function decodeWithRustCLI(inputPath, outputPath, passphrase, files
         args.push('--files', JSON.stringify(files));
     if (dict)
         args.push('--dict', dict);
-    await spawnRustCLI(args);
+    await spawnRustCLI(args, { onProgress });
 }
 export async function listWithRustCLI(inputPath) {
     return spawnRustCLI(['list', inputPath], { collectStdout: true });

package/native/bench_hybrid.rs

@@ -8,7 +8,7 @@ mod pool;
 mod hybrid;
 
 fn bench_roundtrip(name: &str, data: &[u8]) {
-    let compressor = hybrid::HybridCompressor::new(false, 4);
+    let compressor = hybrid::HybridCompressor::new();
 
     let start = Instant::now();
     let (compressed, stats) = compressor.compress(data).unwrap();

package/native/core.rs

@@ -84,54 +84,9 @@ pub fn crc32_bytes(buf: &[u8]) -> u32 {
 }
 
 pub fn adler32_bytes(buf: &[u8]) -> u32 {
-    const MOD: u32 = 65521;
-    const NMAX: usize = 5552;
-
-    if buf.len() > 4 * 1024 * 1024 {
-        return adler32_parallel(buf);
-    }
-
-    let mut a: u32 = 1;
-    let mut b: u32 = 0;
-
-    for chunk in buf.chunks(NMAX) {
-        for &v in chunk {
-            a += v as u32;
-            b += a;
-        }
-        a %= MOD;
-        b %= MOD;
-    }
-
-    (b << 16) | a
-}
-
-fn adler32_parallel(buf: &[u8]) -> u32 {
-    use rayon::prelude::*;
-    const MOD: u32 = 65521;
-    const CHUNK: usize = 1024 * 1024;
-
-    let chunks: Vec<&[u8]> = buf.chunks(CHUNK).collect();
-    let partials: Vec<(u32, u32, usize)> = chunks.par_iter().map(|chunk| {
-        let mut a: u32 = 0;
-        let mut b: u32 = 0;
-        for &v in *chunk {
-            a += v as u32;
-            b += a;
-        }
-        a %= MOD;
-        b %= MOD;
-        (a, b, chunk.len())
-    }).collect();
-
-    let mut a: u64 = 1;
-    let mut b: u64 = 0;
-    for (pa, pb, len) in partials {
-        b = (b + pb as u64 + a * len as u64) % MOD as u64;
-        a = (a + pa as u64) % MOD as u64;
-    }
-
-    ((b as u32) << 16) | (a as u32)
+    let mut hasher = simd_adler32::Adler32::new();
+    hasher.write(buf);
+    hasher.finish()
 }
 
 pub fn delta_encode_bytes(buf: &[u8]) -> Vec<u8> {

package/native/crypto.rs

@@ -75,6 +75,11 @@ pub fn no_encryption(data: &[u8]) -> Vec<u8> {
     result
 }
 
+pub fn no_encryption_in_place(mut data: Vec<u8>) -> Vec<u8> {
+    data.insert(0, ENC_NONE);
+    data
+}
+
 pub fn decrypt_xor(data: &[u8], passphrase: &str) -> Result<Vec<u8>> {
     if data.is_empty() { return Err(anyhow!("Empty xor payload")); }
     if passphrase.is_empty() { return Err(anyhow!("Passphrase required")); }