roxify 1.2.8 → 1.3.0

package/README.md

@@ -13,10 +13,31 @@
 - 🎨 **Multiple modes**: Compact, chunk, pixel, and screenshot modes
 - 📦 **CLI & API**: Use as command-line tool or JavaScript library
 - 🔄 **Lossless**: Perfect roundtrip encoding/decoding
-- 📊 **Efficient**: Typically 0.01-0.05% of original size with Zstd compression
 - 📖 **Full TSDoc**: Complete TypeScript documentation
 - 🦀 **Rust Powered**: Optional native module for extreme performance (falls back to pure JS)
 
+## Real-world benchmarks 🔧
+
+**Highlights**
+
+- Practical benchmarks on large codebase datasets showing significant compression and high throughput while handling many small files efficiently.
+
+**Results**
+
+| Dataset  |   Files | Original | Compressed |     Ratio |   Time | Throughput | Notes                                       |
+| -------- | ------: | -------: | ---------: | --------: | -----: | ---------: | ------------------------------------------- |
+| 4,000 MB | 731,340 |  3.93 GB |  111.42 MB |  **2.8%** | 26.9 s | 149.4 MB/s | gzip: 2.26 GB (57.5%); 7z: 1.87 GB (47.6%)  |
+| 1,000 MB | 141,522 |  1.03 GB |     205 MB | **19.4%** | ~6.2 s |  ≈170 MB/s | shows benefits for many-small-file datasets |
+
+### Methodology
+
+- Compression: multithreaded Zstd (level 19) and Brotli (configurable).
+- Setup: parallel I/O and multithreaded compression on modern SSD-backed systems.
+- Measurements: wall-clock time; throughput = original size / time; comparisons against gzip and 7z with typical defaults.
+- Reproducibility: full benchmark details, commands and raw data are available in `docs/BENCHMARK_FINAL_REPORT.md`.
+
+These results demonstrate Roxify's strength for packaging large codebases and many-small-file archives where speed and a good compression/throughput trade-off matter.
+
 ## Documentation
 
 - 📘 **[CLI Documentation](./CLI.md)** - Complete command-line usage guide

package/dist/cli.js

@@ -1,10 +1,34 @@
 #!/usr/bin/env node
 import cliProgress from 'cli-progress';
 import { mkdirSync, readFileSync, statSync, writeFileSync } from 'fs';
+import { open } from 'fs/promises';
 import { basename, dirname, join, resolve } from 'path';
 import { DataFormatError, decodePngToBinary, encodeBinaryToPng, hasPassphraseInPng, IncorrectPassphraseError, listFilesInPng, PassphraseRequiredError, } from './index.js';
 import { packPathsGenerator, unpackBuffer } from './pack.js';
-const VERSION = '1.2.6';
+const VERSION = '1.2.9';
+async function readLargeFile(filePath) {
+    const st = statSync(filePath);
+    if (st.size <= 2 * 1024 * 1024 * 1024) {
+        return readFileSync(filePath);
+    }
+    const chunkSize = 64 * 1024 * 1024;
+    const chunks = [];
+    let position = 0;
+    const fd = await open(filePath, 'r');
+    try {
+        while (position < st.size) {
+            const currentChunkSize = Math.min(chunkSize, st.size - position);
+            const buffer = Buffer.alloc(currentChunkSize);
+            const { bytesRead } = await fd.read(buffer, 0, currentChunkSize, position);
+            chunks.push(buffer.slice(0, bytesRead));
+            position += bytesRead;
+        }
+    }
+    finally {
+        await fd.close();
+    }
+    return Buffer.concat(chunks);
+}
 function showHelp() {
     console.log(`
 ROX CLI — Encode/decode binary in PNG
@@ -187,6 +211,7 @@ async function encodeCommand(args) {
         Object.assign(options, {
             mode,
             name: parsed.outputName || 'archive',
+            skipOptimization: true,
         });
         if (parsed.verbose)
             options.verbose = true;
@@ -238,7 +263,7 @@ async function encodeCommand(args) {
                 }));
             }
             else {
-                inputData = readFileSync(resolvedInput);
+                inputData = await readLargeFile(resolvedInput);
                 inputSizeVal = inputData.length;
                 displayName = basename(resolvedInput);
                 options.includeFileList = true;
@@ -409,7 +434,7 @@ async function decodeCommand(args) {
                 currentStep = 'Done';
             }
         };
-        const inputBuffer = readFileSync(resolvedInput);
+        const inputBuffer = await readLargeFile(resolvedInput);
         const result = await decodePngToBinary(inputBuffer, options);
         const decodeTime = Date.now() - startDecode;
         clearInterval(heartbeat);

package/dist/hybrid-compression.d.ts

@@ -0,0 +1,27 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export interface CompressionStats {
+    originalSize: number;
+    compressedSize: number;
+    ratio: number;
+    entropyBits: number;
+    blocksCount: number;
+    estimatedThroughput: number;
+}
+export interface GpuInfo {
+    available: boolean;
+    adapterInfo?: string;
+}
+export declare class HybridCompressor {
+    private gpuAvailable;
+    constructor();
+    compress(data: Buffer): Promise<Buffer>;
+    decompress(data: Buffer): Promise<Buffer>;
+    getStats(data: Buffer): CompressionStats;
+    getEntropy(data: Buffer): number;
+    getGpuStatus(): GpuInfo;
+    isGpuAvailable(): boolean;
+}
+export declare function compressBuffer(data: Buffer): Promise<Buffer>;
+export declare function decompressBuffer(data: Buffer): Promise<Buffer>;
+export declare function analyzeCompression(data: Buffer): void;

package/dist/hybrid-compression.js

@@ -0,0 +1,90 @@
+let check_gpu_status;
+let entropy_estimate;
+let get_compression_stats;
+let hybrid_compress;
+let hybrid_decompress;
+try {
+    const native = require('../libroxify_native.node');
+    check_gpu_status = native.check_gpu_status;
+    entropy_estimate = native.entropy_estimate;
+    get_compression_stats = native.get_compression_stats;
+    hybrid_compress = native.hybrid_compress;
+    hybrid_decompress = native.hybrid_decompress;
+}
+catch (e) {
+    console.warn('Warning: Native module not loaded, using stubs');
+}
+export class HybridCompressor {
+    constructor() {
+        const status = check_gpu_status();
+        this.gpuAvailable = status.available;
+        if (this.gpuAvailable) {
+            console.log(`[HybridCompressor] GPU disponible`);
+        }
+        else {
+            console.log(`[HybridCompressor] GPU indisponible, fallback CPU`);
+        }
+    }
+    async compress(data) {
+        const start = performance.now();
+        const compressed = hybrid_compress(data);
+        const elapsed = (performance.now() - start) / 1000;
+        const throughput = data.length / elapsed / 1e6;
+        console.log(`[Compression] ${data.length} bytes → ${compressed.length} bytes ` +
+            `(${((compressed.length / data.length) * 100).toFixed(2)}%) ` +
+            `en ${elapsed.toFixed(3)}s (${throughput.toFixed(0)} Mo/s)`);
+        return compressed;
+    }
+    async decompress(data) {
+        const start = performance.now();
+        const decompressed = hybrid_decompress(data);
+        const elapsed = (performance.now() - start) / 1000;
+        console.log(`[Décompression] ${data.length} bytes → ${decompressed.length} bytes ` +
+            `en ${elapsed.toFixed(3)}s`);
+        return decompressed;
+    }
+    getStats(data) {
+        const start = performance.now();
+        const stats = get_compression_stats(data);
+        const elapsed = (performance.now() - start) / 1000;
+        const throughput = data.length / elapsed / 1e6;
+        return {
+            originalSize: stats.original_size,
+            compressedSize: stats.compressed_size,
+            ratio: stats.ratio,
+            entropyBits: stats.entropy_bits,
+            blocksCount: stats.blocks_count,
+            estimatedThroughput: throughput,
+        };
+    }
+    getEntropy(data) {
+        return entropy_estimate(data);
+    }
+    getGpuStatus() {
+        return check_gpu_status();
+    }
+    isGpuAvailable() {
+        return this.gpuAvailable;
+    }
+}
+export async function compressBuffer(data) {
+    const compressor = new HybridCompressor();
+    return compressor.compress(data);
+}
+export async function decompressBuffer(data) {
+    const compressor = new HybridCompressor();
+    return compressor.decompress(data);
+}
+export function analyzeCompression(data) {
+    const compressor = new HybridCompressor();
+    console.log('\n=== Analyse de Compression ===');
+    console.log(`Taille originale: ${(data.length / 1e6).toFixed(2)} Mo`);
+    console.log(`Entropie: ${compressor.getEntropy(data).toFixed(2)} bits`);
+    const stats = compressor.getStats(data);
+    console.log(`\nStats de compression:`);
+    console.log(`  Blocks: ${stats.blocksCount}`);
+    console.log(`  Ratio: ${(stats.ratio * 100).toFixed(2)}%`);
+    console.log(`  Entropie bits: ${stats.entropyBits.toFixed(2)}`);
+    console.log(`  Débit estimé: ${stats.estimatedThroughput.toFixed(0)} Mo/s`);
+    console.log(`\nGPU Status: ${compressor.isGpuAvailable() ? '✓ Disponible' : '✗ Indisponible'}`);
+}

package/dist/pack.js

@@ -1,4 +1,5 @@
 import { readFileSync, readdirSync, statSync } from 'fs';
+import { readFile } from 'fs/promises';
 import { extname, join, relative, resolve, sep } from 'path';
 function* collectFilesGenerator(paths) {
     for (const p of paths) {
@@ -158,30 +159,46 @@ export async function packPathsGenerator(paths, baseDir, onProgress) {
         indexHeader.writeUInt32BE(0x524f5849, 0);
         indexHeader.writeUInt32BE(indexBuf.length, 4);
         yield Buffer.concat([indexHeader, indexBuf]);
-        let currentBuffer = Buffer.alloc(0);
         let readSoFar = 0;
-        for (let i = 0; i < files.length; i++) {
-            const f = files[i];
-            const rel = relative(base, f).split(sep).join('/');
-            const content = readFileSync(f);
-            const nameBuf = Buffer.from(rel, 'utf8');
-            const nameLen = Buffer.alloc(2);
-            nameLen.writeUInt16BE(nameBuf.length, 0);
-            const sizeBuf = Buffer.alloc(8);
-            sizeBuf.writeBigUInt64BE(BigInt(content.length), 0);
-            const entry = Buffer.concat([nameLen, nameBuf, sizeBuf, content]);
-            if (currentBuffer.length + entry.length > BLOCK_SIZE &&
-                currentBuffer.length > 0) {
-                yield currentBuffer;
-                currentBuffer = Buffer.alloc(0);
+        const BATCH_SIZE = 1000;
+        const chunks = [];
+        let chunkSize = 0;
+        for (let batchStart = 0; batchStart < files.length; batchStart += BATCH_SIZE) {
+            const batchEnd = Math.min(batchStart + BATCH_SIZE, files.length);
+            const batchFiles = files.slice(batchStart, batchEnd);
+            const contentPromises = batchFiles.map(async (f) => {
+                try {
+                    return await readFile(f);
+                }
+                catch (e) {
+                    return Buffer.alloc(0);
+                }
+            });
+            const contents = await Promise.all(contentPromises);
+            for (let i = 0; i < batchFiles.length; i++) {
+                const f = batchFiles[i];
+                const rel = relative(base, f).split(sep).join('/');
+                const content = contents[i];
+                const nameBuf = Buffer.from(rel, 'utf8');
+                const nameLen = Buffer.alloc(2);
+                nameLen.writeUInt16BE(nameBuf.length, 0);
+                const sizeBuf = Buffer.alloc(8);
+                sizeBuf.writeBigUInt64BE(BigInt(content.length), 0);
+                const entry = Buffer.concat([nameLen, nameBuf, sizeBuf, content]);
+                chunks.push(entry);
+                chunkSize += entry.length;
+                if (chunkSize >= BLOCK_SIZE) {
+                    yield Buffer.concat(chunks);
+                    chunks.length = 0;
+                    chunkSize = 0;
+                }
+                readSoFar += content.length;
+                if (onProgress)
+                    onProgress(readSoFar, totalSize, rel);
             }
-            currentBuffer = Buffer.concat([currentBuffer, entry]);
-            readSoFar += content.length;
-            if (onProgress)
-                onProgress(readSoFar, totalSize, rel);
         }
-        if (currentBuffer.length > 0) {
-            yield currentBuffer;
+        if (chunks.length > 0) {
+            yield Buffer.concat(chunks);
         }
     }
     return { index, stream: streamGenerator(), totalSize };

package/dist/utils/encoder.js

@@ -53,7 +53,8 @@ export async function encodeBinaryToPng(input, opts = {}) {
     }
     if (opts.onProgress)
         opts.onProgress({ phase: 'compress_start', total: totalLen });
-    let payload = await parallelZstdCompress(payloadInput, 15, (loaded, total) => {
+    const compressionLevel = opts.compressionLevel ?? 7;
+    let payload = await parallelZstdCompress(payloadInput, compressionLevel, (loaded, total) => {
         if (opts.onProgress) {
             opts.onProgress({
                 phase: 'compress_progress',
@@ -253,71 +254,35 @@ export async function encodeBinaryToPng(input, opts = {}) {
         const scale = 1;
         const width = logicalWidth * scale;
         const height = logicalHeight * scale;
-        const LARGE_IMAGE_PIXELS = 50000000;
+        const LARGE_IMAGE_PIXELS = 10000000;
         const useManualPng = width * height > LARGE_IMAGE_PIXELS || !!process.env.ROX_FAST_PNG;
         let raw;
         let stride = 0;
         if (useManualPng) {
             stride = width * 3 + 1;
             raw = Buffer.alloc(height * stride);
+            const flatData = Buffer.concat(dataWithMarkers);
+            const markerEndBytes = Buffer.alloc(MARKER_END.length * 3);
+            for (let i = 0; i < MARKER_END.length; i++) {
+                markerEndBytes[i * 3] = MARKER_END[i].r;
+                markerEndBytes[i * 3 + 1] = MARKER_END[i].g;
+                markerEndBytes[i * 3 + 2] = MARKER_END[i].b;
+            }
+            const totalDataBytes = logicalWidth * logicalHeight * 3;
+            const fullData = Buffer.alloc(totalDataBytes);
+            const markerStartPos = (logicalHeight - 1) * logicalWidth * 3 +
+                (logicalWidth - MARKER_END.length) * 3;
+            flatData.copy(fullData, 0, 0, Math.min(flatData.length, markerStartPos));
+            markerEndBytes.copy(fullData, markerStartPos);
+            for (let row = 0; row < height; row++) {
+                raw[row * stride] = 0;
+                fullData.copy(raw, row * stride + 1, row * width * 3, (row + 1) * width * 3);
+            }
         }
         else {
             raw = Buffer.alloc(width * height * bytesPerPixel);
-        }
-        let currentBufIdx = 0;
-        let currentBufOffset = 0;
-        const getNextByte = () => {
-            while (currentBufIdx < dataWithMarkers.length) {
-                const buf = dataWithMarkers[currentBufIdx];
-                if (currentBufOffset < buf.length) {
-                    return buf[currentBufOffset++];
-                }
-                currentBufIdx++;
-                currentBufOffset = 0;
-            }
-            return 0;
-        };
-        for (let ly = 0; ly < logicalHeight; ly++) {
-            if (useManualPng) {
-                for (let sy = 0; sy < scale; sy++) {
-                    const py = ly * scale + sy;
-                    raw[py * stride] = 0;
-                }
-            }
-            for (let lx = 0; lx < logicalWidth; lx++) {
-                const linearIdx = ly * logicalWidth + lx;
-                let r = 0, g = 0, b = 0;
-                if (ly === logicalHeight - 1 &&
-                    lx >= logicalWidth - MARKER_END.length) {
-                    const markerIdx = lx - (logicalWidth - MARKER_END.length);
-                    r = MARKER_END[markerIdx].r;
-                    g = MARKER_END[markerIdx].g;
-                    b = MARKER_END[markerIdx].b;
-                }
-                else if (linearIdx < dataPixels) {
-                    r = getNextByte();
-                    g = getNextByte();
-                    b = getNextByte();
-                }
-                for (let sy = 0; sy < scale; sy++) {
-                    for (let sx = 0; sx < scale; sx++) {
-                        const px = lx * scale + sx;
-                        const py = ly * scale + sy;
-                        if (useManualPng) {
-                            const dstIdx = py * stride + 1 + px * 3;
-                            raw[dstIdx] = r;
-                            raw[dstIdx + 1] = g;
-                            raw[dstIdx + 2] = b;
-                        }
-                        else {
-                            const dstIdx = (py * width + px) * 3;
-                            raw[dstIdx] = r;
-                            raw[dstIdx + 1] = g;
-                            raw[dstIdx + 2] = b;
-                        }
-                    }
-                }
-            }
+            const flatData = Buffer.concat(dataWithMarkers);
+            flatData.copy(raw, 0, 0, Math.min(flatData.length, raw.length));
         }
         payload.length = 0;
         dataWithMarkers.length = 0;
@@ -344,9 +309,9 @@ export async function encodeBinaryToPng(input, opts = {}) {
             if (opts.onProgress)
                 opts.onProgress({ phase: 'png_compress', loaded: 0, total: 100 });
             const idatData = zlib.deflateSync(scanlinesData, {
-                level: 3,
+                level: 0,
                 memLevel: 8,
-                strategy: zlib.constants.Z_DEFAULT_STRATEGY,
+                strategy: zlib.constants.Z_FILTERED,
             });
             raw = Buffer.alloc(0);
             const ihdrData = Buffer.alloc(13);
@@ -405,6 +370,10 @@ export async function encodeBinaryToPng(input, opts = {}) {
         raw = Buffer.alloc(0);
         if (opts.onProgress)
             opts.onProgress({ phase: 'png_compress', loaded: 100, total: 100 });
+        if (opts.skipOptimization) {
+            progressBar?.stop();
+            return bufScr;
+        }
         if (opts.onProgress)
             opts.onProgress({ phase: 'optimizing', loaded: 0, total: 100 });
         try {

package/dist/utils/rust-cli-wrapper.d.ts

@@ -0,0 +1 @@
+export declare function encodeWithRustCLI(inputPath: string, outputPath: string, compressionLevel?: number): Promise<void>;

package/dist/utils/rust-cli-wrapper.js

@@ -0,0 +1,36 @@
+import { spawn } from 'child_process';
+import { existsSync } from 'fs';
+import { dirname, join } from 'path';
+import { fileURLToPath } from 'url';
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+export async function encodeWithRustCLI(inputPath, outputPath, compressionLevel = 3) {
+    const cliPath = join(__dirname, '..', 'dist', 'roxify-cli');
+    if (!existsSync(cliPath)) {
+        throw new Error('Rust CLI binary not found. Run: npm run build:native');
+    }
+    return new Promise((resolve, reject) => {
+        const proc = spawn(cliPath, [
+            'encode',
+            inputPath,
+            outputPath,
+            '--level',
+            String(compressionLevel),
+        ]);
+        let stderr = '';
+        proc.stderr.on('data', (data) => {
+            stderr += data.toString();
+        });
+        proc.on('error', (err) => {
+            reject(new Error(`Failed to spawn Rust CLI: ${err.message}`));
+        });
+        proc.on('close', (code) => {
+            if (code === 0) {
+                resolve();
+            }
+            else {
+                reject(new Error(`Rust CLI exited with code ${code}: ${stderr}`));
+            }
+        });
+    });
+}

package/dist/utils/types.d.ts

@@ -3,6 +3,7 @@
 import { PackedFile } from '../pack.js';
 export interface EncodeOptions {
     compression?: 'zstd';
+    compressionLevel?: number;
     passphrase?: string;
     name?: string;
     mode?: 'screenshot';
@@ -15,6 +16,7 @@ export interface EncodeOptions {
         name: string;
         size: number;
     }>;
+    skipOptimization?: boolean;
     onProgress?: (info: {
         phase: string;
         loaded?: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "roxify",
-  "version": "1.2.8",
+  "version": "1.3.0",
   "description": "Encode binary data into PNG images and decode them back. CLI and programmatic API with native Rust acceleration.",
   "type": "module",
   "main": "dist/index.js",
@@ -19,9 +19,10 @@
   "scripts": {
     "build": "tsc",
     "build:native": "cargo build --release --lib && cp target/release/libroxify_native.so libroxify_native.node",
-    "build:all": "npm run build:native && npm run build",
+    "build:cli": "cargo build --release --bin roxify_native && cp target/release/roxify_native dist/roxify-cli",
+    "build:all": "npm run build:native && npm run build && npm run build:cli",
     "prepublishOnly": "npm run build:all",
-    "test": "node test/roundtrip.js && node test/pixel-fallback-preview.js && node test/size-fallback-choice.js && node test/screenshot-roundtrip.js && node test/screenshot-fallback.js",
+    "test": "node test/test-*.js",
     "cli": "node dist/cli.js"
   },
   "keywords": [

package/CHANGELOG.md

@@ -1,36 +0,0 @@
-# Changelog
-
-## [1.2.8] - 2026-01-09
-
-### Added
-
-- 🦀 **Native Rust acceleration** via N-API for extreme performance
-  - Delta encoding/decoding with Rayon parallelization
-  - Multi-threaded Zstd compression (level 19) with `zstdmt` feature
-  - Fast CRC32 and Adler32 checksums
-  - Parallel pixel scanning for ROX1 magic and markers
-- ⚡ **Performance improvements**: Up to 1GB/s throughput on modern hardware
-  - 1GB encode: ~1.2s (863 MB/s)
-  - 1GB decode: ~1.0s (1031 MB/s)
-- 🔄 **Automatic fallback**: Pure TypeScript implementation when native module unavailable
-- 📦 **Unified repository**: Rust and TypeScript code in single npm package
-
-### Changed
-
-- Switched from `@mongodb-js/zstd` to native Rust zstd for better performance
-- Updated package description to highlight native acceleration
-- Compression ratio improved to 0.01-0.05% with Zstd level 19
-
-### Technical
-
-- Added `build:native` and `build:all` npm scripts
-- Native module compiled to `libroxify_native.node` (1.8MB)
-- Cargo workspace configured with `native/` directory
-- Updated dependencies: Rust crates (napi, rayon, zstd, crc32fast, adler)
-
-## [1.0.4] - Previous release
-
-- Initial TypeScript implementation
-- Brotli compression
-- Multiple encoding modes (compact, chunk, pixel, screenshot)
-- AES-256-GCM encryption support