roxify 1.10.0 → 1.10.1

package/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "roxify_native"
-version = "1.9.8"
+version = "1.10.1"
 edition = "2021"
 publish = false
 
@@ -13,6 +13,18 @@ path = "native/lib.rs"
 name = "roxify_native"
 path = "native/main.rs"
 
+[[bin]]
+name = "bench_hybrid"
+path = "native/bench_hybrid.rs"
+
+[[bin]]
+name = "test_stages"
+path = "native/test_stages.rs"
+
+[[bin]]
+name = "test_small_bwt"
+path = "native/test_small_bwt.rs"
+
 [dev-dependencies]
 
 [dependencies]
@@ -43,6 +55,7 @@ bytemuck = { version = "1.14", features = ["derive"] }
 tokio = { version = "1", features = ["sync", "rt"], optional = true }
 parking_lot = "0.12"
 pollster = { version = "0.3", optional = true }
+libsais = { version = "0.2.0", default-features = false }
 
 [features]
 # default is intentionally empty so the crate compiles fast for local checks.
@@ -79,4 +92,3 @@ codegen-units = 16
 debug = false
 incremental = true
 panic = "abort"
-

package/README.md

@@ -39,6 +39,7 @@ The core compression and image-processing logic is written in Rust and exposed t
 ## Features
 
 - **Native Rust acceleration** via N-API with automatic fallback to pure JavaScript
+- **BWT-ANS compression** -- Burrows-Wheeler Transform + Move-to-Front + RLE + rANS entropy coding via libsais O(n) SA-IS (18.1 MB/s encode, 31.2 MB/s decode)
 - **Multi-threaded Zstd compression** (level 19) with parallel chunk processing via Rayon
 - **AES-256-GCM encryption** with PBKDF2 key derivation (100,000 iterations)
 - **Lossless roundtrip** -- encoded data is recovered byte-for-byte
@@ -58,6 +59,20 @@ The core compression and image-processing logic is written in Rust and exposed t
 
 All measurements were taken on Linux x64 (Intel i7-6700K @ 4.0 GHz, 32 GB RAM) with Node.js v20. Every tool uses its **maximum compression** setting: zip -9, gzip -9, 7z LZMA2 -mx=9, and Roxify Zstd level 19. Roxify produces a valid PNG or WAV file rather than a raw archive.
 
+### BWT-ANS Native Compression (Rust, via N-API)
+
+Direct API calls through the native module (BWT → MTF → RLE0 → rANS, 1 MB blocks):
+
+| Dataset | Original | Compressed | Ratio | Encode | Decode | Enc Throughput | Dec Throughput |
+| ------- | -------- | ---------- | ----- | ------ | ------ | -------------- | -------------- |
+| Repetitive text 45 KB | 45 KB | 207 B | 0.5% | < 1 ms | < 1 ms | — | — |
+| Rust source 6 KB | 6 KB | 2.0 KB | 33.7% | < 1 ms | < 1 ms | — | — |
+| node_modules tar 175 MB | 175.5 MB | 38.4 MB | 21.9% | 9.68 s | 5.62 s | 18.1 MB/s | 31.2 MB/s |
+| Random 100 KB | 100 KB | 101 KB | 101.5% | < 1 ms | < 1 ms | — | — |
+| Zeros 100 KB | 100 KB | 51 B | 0.1% | < 1 ms | < 1 ms | — | — |
+
+> BWT-ANS achieves **21.9% on real-world node_modules** (175 MB), with **18 MB/s encode** and **31 MB/s decode** throughput. 100% lossless roundtrip verified on all datasets.
+
 ### Compression Ratio (Maximum Compression for All Tools)
 
 | Dataset     | Original | zip -9          | gzip -9         | 7z LZMA2 -9     | Roxify PNG          | Roxify WAV          |

package/dist/cli.js

@@ -66,6 +66,7 @@ Commands:
 Options:
   --image                   Use PNG container (default)
   --sound                   Use WAV audio container (smaller overhead, faster)
+  --bwt-ans                 Use BWT-ANS compression instead of Zstd
   -p, --passphrase <pass>   Use passphrase (AES-256-GCM)
   -m, --mode <mode>         Mode: screenshot (default)
   -e, --encrypt <type>      auto|aes|xor|none
@@ -135,6 +136,10 @@ function parseArgs(args) {
                 parsed.forceTs = true;
                 i++;
             }
+            else if (key === 'bwt-ans') {
+                parsed.compression = 'bwt-ans';
+                i++;
+            }
             else if (key === 'lossy-resilient') {
                 parsed.lossyResilient = true;
                 i++;
@@ -292,7 +297,7 @@ async function encodeCommand(args) {
     catch (e) {
         anyInputDir = false;
     }
-    if (isRustBinaryAvailable() && !parsed.forceTs && containerMode !== 'sound') {
+    if (isRustBinaryAvailable() && !parsed.forceTs && containerMode !== 'sound' && parsed.compression !== 'bwt-ans') {
         try {
             console.log(`Encoding to ${resolvedOutput} (Using native Rust encoder)\n`);
             const startTime = Date.now();
@@ -395,6 +400,8 @@ async function encodeCommand(args) {
             options.verbose = true;
         if (parsed.noCompress)
             options.compression = 'none';
+        if (parsed.compression === 'bwt-ans')
+            options.compression = 'bwt-ans';
         if (parsed.passphrase) {
             options.passphrase = parsed.passphrase;
             options.encrypt = parsed.encrypt || 'aes';

package/dist/utils/constants.d.ts

@@ -29,6 +29,11 @@ export declare const COMPRESSION_MARKERS: {
         g: number;
         b: number;
     }[];
+    'bwt-ans': {
+        r: number;
+        g: number;
+        b: number;
+    }[];
 };
 export declare const FORMAT_MARKERS: {
     png: {

package/dist/utils/constants.js

@@ -19,6 +19,7 @@ export const MARKER_START = MARKER_COLORS;
 export const MARKER_END = [...MARKER_COLORS].reverse();
 export const COMPRESSION_MARKERS = {
     zstd: [{ r: 0, g: 255, b: 0 }],
+    'bwt-ans': [{ r: 0, g: 128, b: 255 }],
 };
 export const FORMAT_MARKERS = {
     png: { r: 0, g: 255, b: 255 },

package/dist/utils/decoder.js

@@ -11,7 +11,7 @@ import { native } from './native.js';
 import { cropAndReconstitute } from './reconstitution.js';
 import { decodeRobustAudio, isRobustAudioWav } from './robust-audio.js';
 import { decodeRobustImage, isRobustImage } from './robust-image.js';
-import { parallelZstdDecompress, tryZstdDecompress } from './zstd.js';
+import { parallelZstdDecompress } from './zstd.js';
 function isColorMatch(r1, g1, b1, r2, g2, b2) {
     return Math.abs(r1 - r2) + Math.abs(g1 - g2) + Math.abs(b1 - b2) < 50;
 }
@@ -142,7 +142,16 @@ export function unstretchImage(rawRGB, width, height, tolerance = 0) {
     }
     return { data, width: logicalW, height: logicalH };
 }
+const RBW1_MAGIC = Buffer.from('RBW1');
 async function tryDecompress(payload, onProgress) {
+    if (payload.length >= 4 && payload.subarray(0, 4).equals(RBW1_MAGIC) && native?.hybridDecompress) {
+        if (onProgress)
+            onProgress({ phase: 'decompress_start', total: 1 });
+        const result = Buffer.from(native.hybridDecompress(payload));
+        if (onProgress)
+            onProgress({ phase: 'decompress_done', loaded: 1, total: 1 });
+        return result;
+    }
     return await parallelZstdDecompress(payload, onProgress);
 }
 function detectImageFormat(buf) {
@@ -466,7 +475,7 @@ export async function decodePngToBinary(input, opts = {}) {
         if (opts.onProgress)
             opts.onProgress({ phase: 'decompress_start' });
         try {
-            payload = await tryZstdDecompress(payload, (info) => {
+            payload = await tryDecompress(payload, (info) => {
                 if (opts.onProgress)
                     opts.onProgress(info);
             });
@@ -660,7 +669,7 @@ export async function decodePngToBinary(input, opts = {}) {
             const rawPayload = logicalData.slice(idx, idx + payloadLen);
             let payload = tryDecryptIfNeeded(rawPayload, opts.passphrase);
             try {
-                payload = await tryZstdDecompress(payload, (info) => {
+                payload = await tryDecompress(payload, (info) => {
                     if (opts.onProgress)
                         opts.onProgress(info);
                 });
@@ -1170,6 +1179,34 @@ export async function decodePngToBinary(input, opts = {}) {
             e instanceof DataFormatError) {
             throw e;
         }
+        try {
+            const rawPayload = Buffer.from(native.extractPayloadFromPng(processedBuf));
+            let payload = tryDecryptIfNeeded(rawPayload, opts.passphrase);
+            payload = await tryDecompress(payload, (info) => {
+                if (opts.onProgress)
+                    opts.onProgress(info);
+            });
+            if (!payload.slice(0, MAGIC.length).equals(MAGIC)) {
+                throw new DataFormatError('Missing ROX1 magic after native extraction');
+            }
+            payload = payload.slice(MAGIC.length);
+            const nameFromPng = native.extractNameFromPng
+                ? (() => { try {
+                    return native.extractNameFromPng(processedBuf);
+                }
+                catch {
+                    return undefined;
+                } })()
+                : undefined;
+            return { buf: payload, meta: { name: nameFromPng } };
+        }
+        catch (nativeErr) {
+            if (nativeErr instanceof PassphraseRequiredError ||
+                nativeErr instanceof IncorrectPassphraseError ||
+                nativeErr instanceof DataFormatError) {
+                throw nativeErr;
+            }
+        }
         const errMsg = e instanceof Error ? e.message : String(e);
         throw new Error('Failed to decode PNG: ' + errMsg);
     }

package/dist/utils/encoder.js

@@ -93,7 +93,8 @@ export async function encodeBinaryToPng(input, opts = {}) {
     // This must be checked BEFORE TS compression to avoid double-compression.
     if (typeof native.nativeEncodePngWithNameAndFilelist === 'function' &&
         opts.includeFileList &&
-        opts.fileList) {
+        opts.fileList &&
+        opts.compression !== 'bwt-ans') {
         const fileName = opts.name || undefined;
         const inputBuf = Array.isArray(input) ? Buffer.concat(input) : input;
         let sizeMap = null;
@@ -170,15 +171,27 @@ export async function encodeBinaryToPng(input, opts = {}) {
     }
     if (opts.onProgress)
         opts.onProgress({ phase: 'compress_start', total: totalLen });
-    let payload = await parallelZstdCompress(payloadInput, compressionLevel, (loaded, total) => {
-        if (opts.onProgress) {
-            opts.onProgress({
-                phase: 'compress_progress',
-                loaded,
-                total,
-            });
-        }
-    }, opts.dict);
+    let payload;
+    if (opts.compression === 'bwt-ans' && native?.hybridCompress) {
+        const flat = Array.isArray(payloadInput) ? Buffer.concat(payloadInput) : payloadInput;
+        if (opts.onProgress)
+            opts.onProgress({ phase: 'compress_progress', loaded: 0, total: 1 });
+        const compressed = Buffer.from(native.hybridCompress(flat));
+        payload = [compressed];
+        if (opts.onProgress)
+            opts.onProgress({ phase: 'compress_progress', loaded: 1, total: 1 });
+    }
+    else {
+        payload = await parallelZstdCompress(payloadInput, compressionLevel, (loaded, total) => {
+            if (opts.onProgress) {
+                opts.onProgress({
+                    phase: 'compress_progress',
+                    loaded,
+                    total,
+                });
+            }
+        }, opts.dict);
+    }
     if (opts.onProgress)
         opts.onProgress({ phase: 'compress_done', loaded: payload.length });
     if (Array.isArray(input)) {
@@ -392,7 +405,7 @@ export async function encodeBinaryToPng(input, opts = {}) {
             [...dataWithoutMarkers, Buffer.alloc(padding)]
             : dataWithoutMarkers;
         const markerStartBytes = colorsToBytes(MARKER_START);
-        const compressionMarkerBytes = colorsToBytes(COMPRESSION_MARKERS.zstd);
+        const compressionMarkerBytes = colorsToBytes(opts.compression === 'bwt-ans' ? COMPRESSION_MARKERS['bwt-ans'] : COMPRESSION_MARKERS.zstd);
         const dataWithMarkers = [
             markerStartBytes,
             compressionMarkerBytes,

package/dist/utils/inspection.d.ts

@@ -1,35 +1,7 @@
-/**
- * List files stored inside a ROX PNG without fully extracting it.
- * Returns `null` if no file list could be found.
- *
- * @param pngBuf - Buffer containing a PNG file.
- * @param opts - Options to include sizes.
- * @returns Promise resolving to an array of file names or objects with sizes.
- *
- * @example
- * ```js
- * import { listFilesInPng } from 'roxify';
- * const files = await listFilesInPng(fs.readFileSync('out.png'), { includeSizes: true });
- * console.log(files);
- * ```
- */
 export declare function listFilesInPng(pngBuf: Buffer, opts?: {
     includeSizes?: boolean;
 }): Promise<string[] | {
     name: string;
     size: number;
 }[] | null>;
-/**
- * Check if a PNG contains an encrypted payload requiring a passphrase.
- *
- * @param pngBuf - Buffer containing a PNG file.
- * @returns Promise resolving to `true` if the PNG requires a passphrase.
- *
- * @example
- * ```js
- * import { hasPassphraseInPng } from 'roxify';
- * const needPass = await hasPassphraseInPng(fs.readFileSync('out.png'));
- * console.log('needs passphrase?', needPass);
- * ```
- */
 export declare function hasPassphraseInPng(pngBuf: Buffer): Promise<boolean>;