zstd-stream 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 CrellinCreative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,285 @@
+<div align="center">
+
+# zstd-stream
+
+**High-performance Zstandard compression for Node.js and browsers with zero external dependencies**
+
+[![npm version](https://img.shields.io/npm/v/zstd-stream.svg)](https://www.npmjs.com/package/zstd-stream)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![TypeScript](https://img.shields.io/badge/TypeScript-Ready-brightgreen.svg)](https://www.typescriptlang.org/)
+
+</div>
+
+---
+
+## Features
+
+- 🚀 **Universal compatibility** - Works seamlessly in Node.js 18+ and modern browsers
+- 📦 **Zero external assets** - All WebAssembly code bundled internally, works out of the box
+- 🌊 **True streaming support** - Handle multi-GB files with minimal memory footprint
+- ⚡ **Backpressure handling** - Efficient memory management prevents overflow
+- 🎯 **Client-side optimization** - Reduce server load by compressing/decompressing in the browser
+- 🔧 **ESM-first** - Modern ECMAScript modules with full TypeScript support
+- 📊 **Progress tracking** - Monitor compression/decompression in real-time
+
+Built on the latest Zstandard v2 compression algorithm, `zstd-stream` is one of the only packages that embeds all WASM code internally, eliminating asset management headaches and enabling true plug-and-play compression.
+
+---
+
+## Installation
+
+```bash
+npm install zstd-stream
+```
+
+---
+
+## Usage Examples
+
+### Basic Text Compression
+
+**⚠️ Note:** This method loads all data into memory. For large files (>100MB), use streaming instead.
+
+```typescript
+import { compress, decompress } from "zstd-stream";
+
+// Compress text
+const text = "Hello, world!";
+const input = new TextEncoder().encode(text);
+const compressed = await compress(input, { level: 3 });
+
+// Decompress
+const decompressed = await decompress(compressed);
+const output = new TextDecoder().decode(decompressed);
+
+console.log(output); // "Hello, world!"
+```
+
+### Streaming Large Files (Recommended)
+
+**✅ Use this for large files** - Processes data in chunks with constant memory usage.
+
+```typescript
+import { compressStream, decompressStream } from "zstd-stream";
+
+// Compress a stream
+const textStream = new ReadableStream({
+  start(controller) {
+    controller.enqueue(new TextEncoder().encode("Large text data..."));
+    controller.close();
+  },
+});
+
+const compressedStream = await compressStream(textStream, { level: 5 });
+
+// Decompress a stream
+const decompressedStream = await decompressStream(compressedStream);
+
+// Read the result
+const reader = decompressedStream.getReader();
+let result = "";
+
+while (true) {
+  const { done, value } = await reader.read();
+  if (done) break;
+  result += new TextDecoder().decode(value);
+}
+
+console.log(result);
+```
+
+### Browser: Download Compressed File with StreamSaver.js
+
+**⚠️ Performance Note:** Handle compression via Web Workers for browser deployment to avoid degrading the application's performance!
+
+```typescript
+import { compressStream } from "zstd-stream";
+import streamSaver from "streamsaver";
+
+// User selects a file
+const file = document.querySelector('input[type="file"]').files[0];
+const fileStream = file.stream();
+
+// Compress the file
+const compressed = await compressStream(fileStream, {
+  level: 9,
+  onProgress: (bytes) => {
+    console.log(`Compressed: ${(bytes / 1024 / 1024).toFixed(2)} MB`);
+  },
+});
+
+// Save to disk as .zst
+const fileWriteStream = streamSaver.createWriteStream(`${file.name}.zst`);
+const writer = fileWriteStream.getWriter();
+
+const reader = compressed.getReader();
+while (true) {
+  const { done, value } = await reader.read();
+  if (done) break;
+  await writer.write(value);
+}
+
+await writer.close();
+```
+
+### Stream File to Server via HTTP
+
+```typescript
+import { compressStream } from "zstd-stream";
+
+// Get file from user input
+const file = document.querySelector('input[type="file"]').files[0];
+const fileStream = file.stream();
+
+// Compress and upload
+const compressed = await compressStream(fileStream, { level: 6 });
+
+const response = await fetch("https://api.example.com/upload", {
+  method: "POST",
+  headers: {
+    "Content-Type": "application/zstd",
+    "Content-Encoding": "zstd",
+  },
+  body: compressed,
+  duplex: "half", // Required for streaming request bodies
+});
+
+if (response.ok) {
+  console.log("Upload complete!");
+}
+```
+
+---
+
+## API Reference
+
+### `compress(input, options?)`
+
+Compress data in one operation. Best for small files.
+
+**Parameters:**
+
+- `input: Uint8Array` - Data to compress
+- `options?: CompressOptions`
+  - `level?: number` - Compression level (1-19, default: 3)
+  - `onProgress?: (bytesWritten: number) => void` - Progress callback
+
+**Returns:** `Promise<Uint8Array>`
+
+```typescript
+const compressed = await compress(data, { level: 9 });
+```
+
+### `compressStream(input, options?)`
+
+Compress a readable stream. Best for large files.
+
+**Parameters:**
+
+- `input: ReadableStream<Uint8Array>` - Stream to compress
+- `options?: CompressOptions`
+  - `level?: number` - Compression level (1-19, default: 3)
+  - `onProgress?: (bytesWritten: number) => void` - Progress callback
+
+**Returns:** `Promise<ReadableStream<Uint8Array>>`
+
+```typescript
+const compressed = await compressStream(fileStream, { level: 5 });
+```
+
+### `decompress(input, options?)`
+
+Decompress data in one operation.
+
+**Parameters:**
+
+- `input: Uint8Array` - Compressed data
+- `options?: DecompressOptions`
+  - `onProgress?: (bytesWritten: number) => void` - Progress callback
+
+**Returns:** `Promise<Uint8Array>`
+
+```typescript
+const decompressed = await decompress(compressed);
+```
+
+### `decompressStream(input, options?)`
+
+Decompress a readable stream with backpressure support.
+
+**Parameters:**
+
+- `input: ReadableStream<Uint8Array>` - Compressed stream
+- `options?: DecompressOptions`
+  - `onProgress?: (bytesWritten: number) => void` - Progress callback
+
+**Returns:** `Promise<ReadableStream<Uint8Array>>`
+
+```typescript
+const decompressed = await decompressStream(compressedStream);
+```
+
+### `initialize()`
+
+Pre-initialize the WASM module (optional). Call during app startup to avoid initialization delay on first use.
+
+**Returns:** `Promise<void>`
+
+```typescript
+await initialize();
+```
+
+---
+
+## Compression Levels
+
+Levels 1-19 are supported. Higher levels provide diminishing returns.
+
+| Level | Speed     | Ratio    | Use Case                           |
+| ----- | --------- | -------- | ---------------------------------- |
+| 1-3   | Fast      | Lower    | Real-time, network streaming       |
+| 3-7   | Medium    | Balanced | General purpose (recommended)      |
+| 8-15  | Slow      | Better   | File storage, archival             |
+| 16-19 | Very slow | Maximum  | One-time compression, cold storage |
+
+**Default level:** 3 (optimal balance of speed and compression)
+
+---
+
+## Browser Compatibility
+
+- Chrome/Edge 80+
+- Firefox 113+
+- Safari 16.4+
+- Node.js 18+
+
+Requires WebAssembly and ES2022 support.
+
+---
+
+## TypeScript
+
+Full type definitions included:
+
+```typescript
+import type { CompressOptions, DecompressOptions } from "zstd-stream";
+
+const options: CompressOptions = {
+  level: 9,
+  onProgress: (bytes) => console.log(`Progress: ${bytes}`),
+};
+```
+
+---
+
+## License
+
+MIT
+
+---
+
+## Credits
+
+Built with [Zstandard](https://github.com/facebook/zstd) by Meta, compiled using [Emscripten SDK](https://emscripten.org/).
+
+WebAssembly module embedded internally for zero-dependency deployment.

package/dist/compressor.d.ts

@@ -0,0 +1,27 @@
+import type { Module } from "./types.js";
+declare abstract class BaseProcessor {
+    protected destroyed: boolean;
+    protected module: Module | null;
+    destroy(): void;
+    protected abstract cleanup(): void;
+}
+export declare class ZstdCompressor extends BaseProcessor {
+    private level;
+    private ctx;
+    private buffer;
+    private bufferSize;
+    constructor(level: number);
+    init(): Promise<void>;
+    process(data: Uint8Array, isLast: boolean): Uint8Array;
+    protected cleanup(): void;
+}
+export declare class ZstdDecompressor extends BaseProcessor {
+    private ctx;
+    private buffer;
+    private bufferSize;
+    init(): Promise<void>;
+    process(data: Uint8Array): Uint8Array;
+    private concat;
+    protected cleanup(): void;
+}
+export {};

package/dist/compressor.js

@@ -0,0 +1,155 @@
+import { getModule } from "./loader.js";
+class BaseProcessor {
+    destroyed = false;
+    module = null;
+    destroy() {
+        if (!this.destroyed) {
+            this.cleanup();
+            this.destroyed = true;
+        }
+    }
+}
+export class ZstdCompressor extends BaseProcessor {
+    level;
+    ctx = 0;
+    buffer = 0;
+    bufferSize = 0;
+    constructor(level) {
+        super();
+        this.level = level;
+        if (level < 1 || level > 19) {
+            throw new Error("Compression level must be between 1 and 19");
+        }
+    }
+    async init() {
+        this.module = getModule();
+    }
+    process(data, isLast) {
+        if (!this.module)
+            throw new Error("Not initialized");
+        if (this.destroyed)
+            throw new Error("Destroyed");
+        if (!this.ctx) {
+            this.ctx = this.module._createCCtx();
+            if (!this.ctx || this.module._initCStream(this.ctx, this.level) !== 0) {
+                this.cleanup();
+                throw new Error("Failed to create compression context");
+            }
+        }
+        if (!data.length && !isLast)
+            return new Uint8Array(0);
+        const srcPtr = this.module._malloc(data.length);
+        if (!srcPtr)
+            throw new Error("Failed to allocate source buffer");
+        this.module.HEAPU8.set(data, srcPtr);
+        try {
+            if (!this.buffer) {
+                this.bufferSize = this.module._cStreamOutSize();
+                this.buffer = this.module._malloc(this.bufferSize);
+                if (!this.buffer)
+                    throw new Error("Failed to allocate output buffer");
+            }
+            const result = Number(this.module._compressStream(this.ctx, this.buffer, this.bufferSize, srcPtr, data.length, isLast ? 2 : 0));
+            if (result < 0) {
+                throw new Error(`Compression failed: ${this.module._getErrorName(-result)}`);
+            }
+            return result === 0
+                ? new Uint8Array(0)
+                : new Uint8Array(this.module.HEAPU8.subarray(this.buffer, this.buffer + result));
+        }
+        finally {
+            this.module._free(srcPtr);
+        }
+    }
+    cleanup() {
+        if (this.module) {
+            if (this.ctx)
+                this.module._freeCCtx(this.ctx);
+            if (this.buffer)
+                this.module._free(this.buffer);
+            this.ctx = 0;
+            this.buffer = 0;
+        }
+    }
+}
+export class ZstdDecompressor extends BaseProcessor {
+    ctx = 0;
+    buffer = 0;
+    bufferSize = 0;
+    async init() {
+        this.module = getModule();
+    }
+    process(data) {
+        if (!this.module)
+            throw new Error("Not initialized");
+        if (this.destroyed)
+            throw new Error("Destroyed");
+        if (!this.ctx) {
+            this.ctx = this.module._createDCtx();
+            if (!this.ctx || this.module._initDStream(this.ctx) === 0) {
+                this.cleanup();
+                throw new Error("Failed to create decompression context");
+            }
+        }
+        if (!data.length)
+            return new Uint8Array(0);
+        if (!this.buffer) {
+            this.bufferSize = this.module._dStreamOutSize();
+            this.buffer = this.module._malloc(this.bufferSize);
+            if (!this.buffer)
+                throw new Error("Failed to allocate output buffer");
+        }
+        const srcPtr = this.module._malloc(data.length);
+        if (!srcPtr)
+            throw new Error("Failed to allocate source buffer");
+        this.module.HEAPU8.set(data, srcPtr);
+        try {
+            let srcPos = 0;
+            const chunks = [];
+            while (srcPos < data.length) {
+                const result = this.module._decompressStream(this.ctx, this.buffer, this.bufferSize, srcPtr + srcPos, data.length - srcPos);
+                if (result < 0) {
+                    const errorCode = Number(BigInt(result) & 0x7fffffffffffffffn);
+                    throw new Error(`Decompression failed: ${this.module._getErrorName(errorCode)}`);
+                }
+                const consumed = Number(BigInt(result) >> 32n);
+                const outputSize = Number(BigInt(result) & 0xffffffffn);
+                if (consumed === 0 && outputSize === 0) {
+                    throw new Error("Decompression stalled");
+                }
+                if (outputSize > 0) {
+                    chunks.push(new Uint8Array(this.module.HEAPU8.subarray(this.buffer, this.buffer + outputSize)));
+                }
+                srcPos += consumed;
+            }
+            return this.concat(chunks);
+        }
+        finally {
+            this.module._free(srcPtr);
+        }
+    }
+    concat(chunks) {
+        if (chunks.length === 0)
+            return new Uint8Array(0);
+        if (chunks.length === 1)
+            return chunks[0];
+        const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+        const result = new Uint8Array(total);
+        let offset = 0;
+        for (const chunk of chunks) {
+            result.set(chunk, offset);
+            offset += chunk.length;
+        }
+        return result;
+    }
+    cleanup() {
+        if (this.module) {
+            if (this.ctx)
+                this.module._freeDCtx(this.ctx);
+            if (this.buffer)
+                this.module._free(this.buffer);
+            this.ctx = 0;
+            this.buffer = 0;
+        }
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,12 @@
+export interface CompressOptions {
+    level?: number;
+    onProgress?: (bytesWritten: number) => void;
+}
+export interface DecompressOptions {
+    onProgress?: (bytesWritten: number) => void;
+}
+export declare function compress(input: Uint8Array, options?: CompressOptions): Promise<Uint8Array>;
+export declare function compressStream(input: ReadableStream<Uint8Array>, options?: CompressOptions): Promise<ReadableStream<Uint8Array>>;
+export declare function decompress(input: Uint8Array, options?: DecompressOptions): Promise<Uint8Array>;
+export declare function decompressStream(input: ReadableStream<Uint8Array>, options?: DecompressOptions): Promise<ReadableStream<Uint8Array>>;
+export declare function initialize(): Promise<void>;

package/dist/index.js

@@ -0,0 +1,148 @@
+import { ZstdCompressor, ZstdDecompressor } from "./compressor.js";
+import { initZstd } from "./loader.js";
+let initialized = false;
+async function ensureInit() {
+    if (!initialized) {
+        await initZstd();
+        initialized = true;
+    }
+}
+// Compress Uint8Array -> Uint8Array
+export async function compress(input, options) {
+    await ensureInit();
+    const { level = 3, onProgress } = options || {};
+    const compressor = new ZstdCompressor(level);
+    await compressor.init();
+    try {
+        const result = compressor.process(input, true);
+        onProgress?.(result.length);
+        return result;
+    }
+    finally {
+        compressor.destroy();
+    }
+}
+export async function compressStream(input, options) {
+    await ensureInit();
+    const { level = 3, onProgress } = options || {};
+    const reader = input.getReader();
+    let compressor = null;
+    let totalWritten = 0;
+    let inputDone = false;
+    return new ReadableStream({
+        async start() {
+            compressor = new ZstdCompressor(level);
+            await compressor.init();
+        },
+        async pull(controller) {
+            try {
+                if (inputDone)
+                    return;
+                let loopIterations = 0;
+                // Keep reading until we have data to enqueue or input is exhausted
+                while (!inputDone) {
+                    loopIterations++;
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        // Final flush
+                        const chunk = compressor.process(new Uint8Array(0), true);
+                        if (chunk.length > 0) {
+                            controller.enqueue(chunk);
+                            totalWritten += chunk.length;
+                            onProgress?.(totalWritten);
+                        }
+                        controller.close();
+                        reader.releaseLock();
+                        compressor?.destroy();
+                        inputDone = true;
+                        return;
+                    }
+                    // Process chunk
+                    const chunk = compressor.process(value, false);
+                    if (chunk.length > 0) {
+                        controller.enqueue(chunk);
+                        totalWritten += chunk.length;
+                        onProgress?.(totalWritten);
+                        return; // Exit after enqueuing data
+                    }
+                    // If chunk is empty, continue loop to read next input chunk
+                }
+            }
+            catch (error) {
+                inputDone = true;
+                reader.releaseLock();
+                compressor?.destroy();
+                controller.error(error);
+            }
+        },
+        cancel() {
+            reader.releaseLock();
+            compressor?.destroy();
+        },
+    });
+}
+// Decompress Uint8Array -> Uint8Array
+export async function decompress(input, options) {
+    await ensureInit();
+    const { onProgress } = options || {};
+    const decompressor = new ZstdDecompressor();
+    await decompressor.init();
+    try {
+        const result = decompressor.process(input);
+        onProgress?.(result.length);
+        return result;
+    }
+    finally {
+        decompressor.destroy();
+    }
+}
+// Decompress ReadableStream -> ReadableStream with backpressure
+export async function decompressStream(input, options) {
+    await ensureInit();
+    const { onProgress } = options || {};
+    const reader = input.getReader();
+    let decompressor = null;
+    let totalWritten = 0;
+    let inputDone = false;
+    return new ReadableStream({
+        async start() {
+            decompressor = new ZstdDecompressor();
+            await decompressor.init();
+        },
+        async pull(controller) {
+            try {
+                if (inputDone) {
+                    return; // Already finished
+                }
+                const { done, value } = await reader.read();
+                if (done) {
+                    inputDone = true;
+                    controller.close();
+                    reader.releaseLock();
+                    decompressor?.destroy();
+                    return;
+                }
+                const chunk = decompressor.process(value);
+                if (chunk.length > 0) {
+                    controller.enqueue(chunk);
+                    totalWritten += chunk.length;
+                    onProgress?.(totalWritten);
+                }
+            }
+            catch (error) {
+                inputDone = true;
+                reader.releaseLock();
+                decompressor?.destroy();
+                controller.error(error);
+                throw error;
+            }
+        },
+        cancel() {
+            reader.releaseLock();
+            decompressor?.destroy();
+        },
+    });
+}
+export async function initialize() {
+    await ensureInit();
+}

package/dist/lib.d.ts

@@ -0,0 +1 @@
+export declare const lib: string;