roxify 1.2.9 → 1.3.0

package/README.md

@@ -13,10 +13,31 @@
 - 🎨 **Multiple modes**: Compact, chunk, pixel, and screenshot modes
 - 📦 **CLI & API**: Use as command-line tool or JavaScript library
 - 🔄 **Lossless**: Perfect roundtrip encoding/decoding
-- 📊 **Efficient**: Typically 0.01-0.05% of original size with Zstd compression
 - 📖 **Full TSDoc**: Complete TypeScript documentation
 - 🦀 **Rust Powered**: Optional native module for extreme performance (falls back to pure JS)
 
+## Real-world benchmarks 🔧
+
+**Highlights**
+
+- Practical benchmarks on large codebase datasets showing significant compression and high throughput while handling many small files efficiently.
+
+**Results**
+
+| Dataset  |   Files | Original | Compressed |     Ratio |   Time | Throughput | Notes                                       |
+| -------- | ------: | -------: | ---------: | --------: | -----: | ---------: | ------------------------------------------- |
+| 4,000 MB | 731,340 |  3.93 GB |  111.42 MB |  **2.8%** | 26.9 s | 149.4 MB/s | gzip: 2.26 GB (57.5%); 7z: 1.87 GB (47.6%)  |
+| 1,000 MB | 141,522 |  1.03 GB |     205 MB | **19.4%** | ~6.2 s |  ≈170 MB/s | shows benefits for many-small-file datasets |
+
+### Methodology
+
+- Compression: multithreaded Zstd (level 19) and Brotli (configurable).
+- Setup: parallel I/O and multithreaded compression on modern SSD-backed systems.
+- Measurements: wall-clock time; throughput = original size / time; comparisons against gzip and 7z with typical defaults.
+- Reproducibility: full benchmark details, commands and raw data are available in `docs/BENCHMARK_FINAL_REPORT.md`.
+
+These results demonstrate Roxify's strength for packaging large codebases and many-small-file archives where speed and a good compression/throughput trade-off matter.
+
 ## Documentation
 
 - 📘 **[CLI Documentation](./CLI.md)** - Complete command-line usage guide

package/dist/cli.js

@@ -1,10 +1,34 @@
 #!/usr/bin/env node
 import cliProgress from 'cli-progress';
 import { mkdirSync, readFileSync, statSync, writeFileSync } from 'fs';
+import { open } from 'fs/promises';
 import { basename, dirname, join, resolve } from 'path';
 import { DataFormatError, decodePngToBinary, encodeBinaryToPng, hasPassphraseInPng, IncorrectPassphraseError, listFilesInPng, PassphraseRequiredError, } from './index.js';
 import { packPathsGenerator, unpackBuffer } from './pack.js';
 const VERSION = '1.2.9';
+async function readLargeFile(filePath) {
+    const st = statSync(filePath);
+    if (st.size <= 2 * 1024 * 1024 * 1024) {
+        return readFileSync(filePath);
+    }
+    const chunkSize = 64 * 1024 * 1024;
+    const chunks = [];
+    let position = 0;
+    const fd = await open(filePath, 'r');
+    try {
+        while (position < st.size) {
+            const currentChunkSize = Math.min(chunkSize, st.size - position);
+            const buffer = Buffer.alloc(currentChunkSize);
+            const { bytesRead } = await fd.read(buffer, 0, currentChunkSize, position);
+            chunks.push(buffer.slice(0, bytesRead));
+            position += bytesRead;
+        }
+    }
+    finally {
+        await fd.close();
+    }
+    return Buffer.concat(chunks);
+}
 function showHelp() {
     console.log(`
 ROX CLI — Encode/decode binary in PNG
@@ -239,7 +263,7 @@ async function encodeCommand(args) {
                 }));
             }
             else {
-                inputData = readFileSync(resolvedInput);
+                inputData = await readLargeFile(resolvedInput);
                 inputSizeVal = inputData.length;
                 displayName = basename(resolvedInput);
                 options.includeFileList = true;
@@ -410,7 +434,7 @@ async function decodeCommand(args) {
                 currentStep = 'Done';
             }
         };
-        const inputBuffer = readFileSync(resolvedInput);
+        const inputBuffer = await readLargeFile(resolvedInput);
         const result = await decodePngToBinary(inputBuffer, options);
         const decodeTime = Date.now() - startDecode;
         clearInterval(heartbeat);

package/dist/hybrid-compression.d.ts

@@ -0,0 +1,27 @@
+/// <reference types="node" />
+/// <reference types="node" />
+export interface CompressionStats {
+    originalSize: number;
+    compressedSize: number;
+    ratio: number;
+    entropyBits: number;
+    blocksCount: number;
+    estimatedThroughput: number;
+}
+export interface GpuInfo {
+    available: boolean;
+    adapterInfo?: string;
+}
+export declare class HybridCompressor {
+    private gpuAvailable;
+    constructor();
+    compress(data: Buffer): Promise<Buffer>;
+    decompress(data: Buffer): Promise<Buffer>;
+    getStats(data: Buffer): CompressionStats;
+    getEntropy(data: Buffer): number;
+    getGpuStatus(): GpuInfo;
+    isGpuAvailable(): boolean;
+}
+export declare function compressBuffer(data: Buffer): Promise<Buffer>;
+export declare function decompressBuffer(data: Buffer): Promise<Buffer>;
+export declare function analyzeCompression(data: Buffer): void;

package/dist/hybrid-compression.js

@@ -0,0 +1,90 @@
+let check_gpu_status;
+let entropy_estimate;
+let get_compression_stats;
+let hybrid_compress;
+let hybrid_decompress;
+try {
+    const native = require('../libroxify_native.node');
+    check_gpu_status = native.check_gpu_status;
+    entropy_estimate = native.entropy_estimate;
+    get_compression_stats = native.get_compression_stats;
+    hybrid_compress = native.hybrid_compress;
+    hybrid_decompress = native.hybrid_decompress;
+}
+catch (e) {
+    console.warn('Warning: Native module not loaded, using stubs');
+}
+export class HybridCompressor {
+    constructor() {
+        const status = check_gpu_status();
+        this.gpuAvailable = status.available;
+        if (this.gpuAvailable) {
+            console.log(`[HybridCompressor] GPU disponible`);
+        }
+        else {
+            console.log(`[HybridCompressor] GPU indisponible, fallback CPU`);
+        }
+    }
+    async compress(data) {
+        const start = performance.now();
+        const compressed = hybrid_compress(data);
+        const elapsed = (performance.now() - start) / 1000;
+        const throughput = data.length / elapsed / 1e6;
+        console.log(`[Compression] ${data.length} bytes → ${compressed.length} bytes ` +
+            `(${((compressed.length / data.length) * 100).toFixed(2)}%) ` +
+            `en ${elapsed.toFixed(3)}s (${throughput.toFixed(0)} Mo/s)`);
+        return compressed;
+    }
+    async decompress(data) {
+        const start = performance.now();
+        const decompressed = hybrid_decompress(data);
+        const elapsed = (performance.now() - start) / 1000;
+        console.log(`[Décompression] ${data.length} bytes → ${decompressed.length} bytes ` +
+            `en ${elapsed.toFixed(3)}s`);
+        return decompressed;
+    }
+    getStats(data) {
+        const start = performance.now();
+        const stats = get_compression_stats(data);
+        const elapsed = (performance.now() - start) / 1000;
+        const throughput = data.length / elapsed / 1e6;
+        return {
+            originalSize: stats.original_size,
+            compressedSize: stats.compressed_size,
+            ratio: stats.ratio,
+            entropyBits: stats.entropy_bits,
+            blocksCount: stats.blocks_count,
+            estimatedThroughput: throughput,
+        };
+    }
+    getEntropy(data) {
+        return entropy_estimate(data);
+    }
+    getGpuStatus() {
+        return check_gpu_status();
+    }
+    isGpuAvailable() {
+        return this.gpuAvailable;
+    }
+}
+export async function compressBuffer(data) {
+    const compressor = new HybridCompressor();
+    return compressor.compress(data);
+}
+export async function decompressBuffer(data) {
+    const compressor = new HybridCompressor();
+    return compressor.decompress(data);
+}
+export function analyzeCompression(data) {
+    const compressor = new HybridCompressor();
+    console.log('\n=== Analyse de Compression ===');
+    console.log(`Taille originale: ${(data.length / 1e6).toFixed(2)} Mo`);
+    console.log(`Entropie: ${compressor.getEntropy(data).toFixed(2)} bits`);
+    const stats = compressor.getStats(data);
+    console.log(`\nStats de compression:`);
+    console.log(`  Blocks: ${stats.blocksCount}`);
+    console.log(`  Ratio: ${(stats.ratio * 100).toFixed(2)}%`);
+    console.log(`  Entropie bits: ${stats.entropyBits.toFixed(2)}`);
+    console.log(`  Débit estimé: ${stats.estimatedThroughput.toFixed(0)} Mo/s`);
+    console.log(`\nGPU Status: ${compressor.isGpuAvailable() ? '✓ Disponible' : '✗ Indisponible'}`);
+}

package/dist/utils/encoder.js

@@ -53,7 +53,7 @@ export async function encodeBinaryToPng(input, opts = {}) {
     }
     if (opts.onProgress)
         opts.onProgress({ phase: 'compress_start', total: totalLen });
-    const compressionLevel = opts.compressionLevel ?? 3;
+    const compressionLevel = opts.compressionLevel ?? 7;
     let payload = await parallelZstdCompress(payloadInput, compressionLevel, (loaded, total) => {
         if (opts.onProgress) {
             opts.onProgress({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "roxify",
-  "version": "1.2.9",
+  "version": "1.3.0",
   "description": "Encode binary data into PNG images and decode them back. CLI and programmatic API with native Rust acceleration.",
   "type": "module",
   "main": "dist/index.js",
@@ -22,7 +22,7 @@
     "build:cli": "cargo build --release --bin roxify_native && cp target/release/roxify_native dist/roxify-cli",
     "build:all": "npm run build:native && npm run build && npm run build:cli",
     "prepublishOnly": "npm run build:all",
-    "test": "node test/roundtrip.js && node test/pixel-fallback-preview.js && node test/size-fallback-choice.js && node test/screenshot-roundtrip.js && node test/screenshot-fallback.js",
+    "test": "node test/test-*.js",
     "cli": "node dist/cli.js"
   },
   "keywords": [

package/CHANGELOG.md

@@ -1,74 +0,0 @@
-# Changelog
-
-## [1.2.10] - 2026-01-09
-
-### Performance 🚀🚀
-
-- **MASSIVE file packing speedup**: 18,750 files (660MB) now in **7 seconds** (was 18s)
-- Parallelized file reading with `fs.promises.readFile()` and `Promise.all()` batching
-- Batch size optimized to 1000 files per parallel read
-- Improved buffer concatenation strategy (array accumulation + single concat)
-- Added error handling for unreadable files during parallel reads
-
-### Benchmarks
-
-- Single file 1GB: 389ms (2.63 GB/s)
-- Directory 18,750 files (660MB): **6.8s** (97 MB/s including I/O overhead)
-
-## [1.2.9] - 2026-01-09
-
-### Performance 🚀
-
-- **EXTREME SPEED**: 1GB encode in 0.39s (**2.6 GB/s throughput**)
-- Optimized PNG pixel copying from byte-by-byte loops to bulk Buffer.copy() operations
-- Reduced PNG deflate overhead by using zlib level 0 (data already compressed with Zstd)
-- Lowered large image threshold from 50M to 10M pixels for faster manual PNG generation
-- Default Zstd compression level changed from 15 to 3 (much faster, still excellent ratio)
-
-### Changed
-
-- Added `compressionLevel` option to `EncodeOptions` (default: 3)
-- Added `skipOptimization` option to disable zopfli PNG optimization
-- CLI now disables PNG optimization by default for maximum speed
-
-### Benchmarks
-
-- 1KB: 14.77ms
-- 100MB: 63.74ms (1.57 GB/s)
-- 500MB: 203ms (2.46 GB/s)
-- 1GB: 389ms (2.63 GB/s)
-
-## [1.2.8] - 2026-01-09
-
-### Added
-
-- 🦀 **Native Rust acceleration** via N-API for extreme performance
-  - Delta encoding/decoding with Rayon parallelization
-  - Multi-threaded Zstd compression (level 19) with `zstdmt` feature
-  - Fast CRC32 and Adler32 checksums
-  - Parallel pixel scanning for ROX1 magic and markers
-- ⚡ **Performance improvements**: Up to 1GB/s throughput on modern hardware
-  - 1GB encode: ~1.2s (863 MB/s)
-  - 1GB decode: ~1.0s (1031 MB/s)
-- 🔄 **Automatic fallback**: Pure TypeScript implementation when native module unavailable
-- 📦 **Unified repository**: Rust and TypeScript code in single npm package
-
-### Changed
-
-- Switched from `@mongodb-js/zstd` to native Rust zstd for better performance
-- Updated package description to highlight native acceleration
-- Compression ratio improved to 0.01-0.05% with Zstd level 19
-
-### Technical
-
-- Added `build:native` and `build:all` npm scripts
-- Native module compiled to `libroxify_native.node` (1.8MB)
-- Cargo workspace configured with `native/` directory
-- Updated dependencies: Rust crates (napi, rayon, zstd, crc32fast, adler)
-
-## [1.0.4] - Previous release
-
-- Initial TypeScript implementation
-- Brotli compression
-- Multiple encoding modes (compact, chunk, pixel, screenshot)
-- AES-256-GCM encryption support