zstd-stream 1.0.0 → 1.1.0

package/README.md

@@ -1,285 +1,273 @@
-<div align="center">
-
-# zstd-stream
-
-**High-performance Zstandard compression for Node.js and browsers with zero external dependencies**
-
-[![npm version](https://img.shields.io/npm/v/zstd-stream.svg)](https://www.npmjs.com/package/zstd-stream)
-[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
-[![TypeScript](https://img.shields.io/badge/TypeScript-Ready-brightgreen.svg)](https://www.typescriptlang.org/)
-
-</div>
-
----
-
-## Features
-
-- 🚀 **Universal compatibility** - Works seamlessly in Node.js 18+ and modern browsers
-- 📦 **Zero external assets** - All WebAssembly code bundled internally, works out of the box
-- 🌊 **True streaming support** - Handle multi-GB files with minimal memory footprint
-- ⚡ **Backpressure handling** - Efficient memory management prevents overflow
-- 🎯 **Client-side optimization** - Reduce server load by compressing/decompressing in the browser
-- 🔧 **ESM-first** - Modern ECMAScript modules with full TypeScript support
-- 📊 **Progress tracking** - Monitor compression/decompression in real-time
-
-Built on the latest Zstandard v2 compression algorithm, `zstd-stream` is one of the only packages that embeds all WASM code internally, eliminating asset management headaches and enabling true plug-and-play compression.
-
----
-
-## Installation
-
-```bash
-npm install zstd-stream
-```
-
----
-
-## Usage Examples
-
-### Basic Text Compression
-
-**⚠️ Note:** This method loads all data into memory. For large files (>100MB), use streaming instead.
-
-```typescript
-import { compress, decompress } from "zstd-stream";
-
-// Compress text
-const text = "Hello, world!";
-const input = new TextEncoder().encode(text);
-const compressed = await compress(input, { level: 3 });
-
-// Decompress
-const decompressed = await decompress(compressed);
-const output = new TextDecoder().decode(decompressed);
-
-console.log(output); // "Hello, world!"
-```
-
-### Streaming Large Files (Recommended)
-
-**✅ Use this for large files** - Processes data in chunks with constant memory usage.
-
-```typescript
-import { compressStream, decompressStream } from "zstd-stream";
-
-// Compress a stream
-const textStream = new ReadableStream({
-  start(controller) {
-    controller.enqueue(new TextEncoder().encode("Large text data..."));
-    controller.close();
-  },
-});
-
-const compressedStream = await compressStream(textStream, { level: 5 });
-
-// Decompress a stream
-const decompressedStream = await decompressStream(compressedStream);
-
-// Read the result
-const reader = decompressedStream.getReader();
-let result = "";
-
-while (true) {
-  const { done, value } = await reader.read();
-  if (done) break;
-  result += new TextDecoder().decode(value);
-}
-
-console.log(result);
-```
-
-### Browser: Download Compressed File with StreamSaver.js
-
-**⚠️ Performance Note:** Handle compression via Web Workers for browser deployment to avoid degrading the application's performance!
-
-```typescript
-import { compressStream } from "zstd-stream";
-import streamSaver from "streamsaver";
-
-// User selects a file
-const file = document.querySelector('input[type="file"]').files[0];
-const fileStream = file.stream();
-
-// Compress the file
-const compressed = await compressStream(fileStream, {
-  level: 9,
-  onProgress: (bytes) => {
-    console.log(`Compressed: ${(bytes / 1024 / 1024).toFixed(2)} MB`);
-  },
-});
-
-// Save to disk as .zst
-const fileWriteStream = streamSaver.createWriteStream(`${file.name}.zst`);
-const writer = fileWriteStream.getWriter();
-
-const reader = compressed.getReader();
-while (true) {
-  const { done, value } = await reader.read();
-  if (done) break;
-  await writer.write(value);
-}
-
-await writer.close();
-```
-
-### Stream File to Server via HTTP
-
-```typescript
-import { compressStream } from "zstd-stream";
-
-// Get file from user input
-const file = document.querySelector('input[type="file"]').files[0];
-const fileStream = file.stream();
-
-// Compress and upload
-const compressed = await compressStream(fileStream, { level: 6 });
-
-const response = await fetch("https://api.example.com/upload", {
-  method: "POST",
-  headers: {
-    "Content-Type": "application/zstd",
-    "Content-Encoding": "zstd",
-  },
-  body: compressed,
-  duplex: "half", // Required for streaming request bodies
-});
-
-if (response.ok) {
-  console.log("Upload complete!");
-}
-```
-
----
-
-## API Reference
-
-### `compress(input, options?)`
-
-Compress data in one operation. Best for small files.
-
-**Parameters:**
-
-- `input: Uint8Array` - Data to compress
-- `options?: CompressOptions`
-  - `level?: number` - Compression level (1-19, default: 3)
-  - `onProgress?: (bytesWritten: number) => void` - Progress callback
-
-**Returns:** `Promise<Uint8Array>`
-
-```typescript
-const compressed = await compress(data, { level: 9 });
-```
-
-### `compressStream(input, options?)`
-
-Compress a readable stream. Best for large files.
-
-**Parameters:**
-
-- `input: ReadableStream<Uint8Array>` - Stream to compress
-- `options?: CompressOptions`
-  - `level?: number` - Compression level (1-19, default: 3)
-  - `onProgress?: (bytesWritten: number) => void` - Progress callback
-
-**Returns:** `Promise<ReadableStream<Uint8Array>>`
-
-```typescript
-const compressed = await compressStream(fileStream, { level: 5 });
-```
-
-### `decompress(input, options?)`
-
-Decompress data in one operation.
-
-**Parameters:**
-
-- `input: Uint8Array` - Compressed data
-- `options?: DecompressOptions`
-  - `onProgress?: (bytesWritten: number) => void` - Progress callback
-
-**Returns:** `Promise<Uint8Array>`
-
-```typescript
-const decompressed = await decompress(compressed);
-```
-
-### `decompressStream(input, options?)`
-
-Decompress a readable stream with backpressure support.
-
-**Parameters:**
-
-- `input: ReadableStream<Uint8Array>` - Compressed stream
-- `options?: DecompressOptions`
-  - `onProgress?: (bytesWritten: number) => void` - Progress callback
-
-**Returns:** `Promise<ReadableStream<Uint8Array>>`
-
-```typescript
-const decompressed = await decompressStream(compressedStream);
-```
-
-### `initialize()`
-
-Pre-initialize the WASM module (optional). Call during app startup to avoid initialization delay on first use.
-
-**Returns:** `Promise<void>`
-
-```typescript
-await initialize();
-```
-
----
-
-## Compression Levels
-
-Levels 1-19 are supported. Higher levels provide diminishing returns.
-
-| Level | Speed     | Ratio    | Use Case                           |
-| ----- | --------- | -------- | ---------------------------------- |
-| 1-3   | Fast      | Lower    | Real-time, network streaming       |
-| 3-7   | Medium    | Balanced | General purpose (recommended)      |
-| 8-15  | Slow      | Better   | File storage, archival             |
-| 16-19 | Very slow | Maximum  | One-time compression, cold storage |
-
-**Default level:** 3 (optimal balance of speed and compression)
-
----
-
-## Browser Compatibility
-
-- Chrome/Edge 80+
-- Firefox 113+
-- Safari 16.4+
-- Node.js 18+
-
-Requires WebAssembly and ES2022 support.
-
----
-
-## TypeScript
-
-Full type definitions included:
-
-```typescript
-import type { CompressOptions, DecompressOptions } from "zstd-stream";
-
-const options: CompressOptions = {
-  level: 9,
-  onProgress: (bytes) => console.log(`Progress: ${bytes}`),
-};
-```
-
----
-
-## License
-
-MIT
-
----
-
-## Credits
-
-Built with [Zstandard](https://github.com/facebook/zstd) by Meta, compiled using [Emscripten SDK](https://emscripten.org/).
-
-WebAssembly module embedded internally for zero-dependency deployment.
+<div align="center">
+
+# zstd-stream
+
+**Streaming Zstandard compression for Node.js and browsers — zero external dependencies, bounded memory.**
+
+[![npm version](https://img.shields.io/npm/v/zstd-stream.svg)](https://www.npmjs.com/package/zstd-stream)
+[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
+[![TypeScript](https://img.shields.io/badge/TypeScript-Ready-brightgreen.svg)](https://www.typescriptlang.org/)
+[![Browser Demo](https://img.shields.io/badge/demo-live-brightgreen)](https://crellincreative.github.io/zstd-stream/)
+
+</div>
+
+---
+
+`zstd-stream` compresses and decompresses [Zstandard](https://github.com/facebook/zstd)
+data through the Web Streams API. It streams files larger than available RAM in
+**constant memory**, propagates **backpressure** end to end, and ships the
+WebAssembly build **embedded** — no `.wasm` asset to host or configure.
+
+- 🌊 **Streaming first** — process multi-GB data in fixed ~128 KB buffers
+- ⚡ **Backpressure built in** — a slow writer automatically throttles the reader, so memory never runs away
+- 📦 **Zero external assets** — WASM is bundled in; `npm install` and import
+- 🌍 **Universal** — the same code runs in Node.js 18+ and modern browsers
+- 🔧 **ESM + TypeScript** — full type definitions included
+- 📊 **Progress callbacks** — observe bytes processed in real time
+
+---
+
+## Installation
+
+```bash
+npm install zstd-stream
+```
+
+---
+
+## Quick start
+
+Everything operates on `ReadableStream<Uint8Array>` — the same stream type
+returned by `fetch`, `Blob.stream()`, `File.stream()`, and Node's
+`Readable.toWeb()`.
+
+```typescript
+import { compressStream, decompressStream } from "zstd-stream";
+
+// Compress any byte stream...
+const compressed = await compressStream(source, { level: 3 });
+
+// ...and decompress it back.
+const restored = await decompressStream(compressed);
+
+// Collect a stream into bytes (or use .text() for strings):
+const bytes = new Uint8Array(await new Response(restored).arrayBuffer());
+```
+
+Prefer streaming wherever the data is large or arrives incrementally. For small,
+fully in-memory buffers there are one-shot [`compress`](#compressinput-options) /
+[`decompress`](#decompressinput-options) helpers.
+
+---
+
+## Recipes
+
+### Compress a file and upload it
+
+Pipe the compressed stream straight into a `fetch` request body — nothing is
+buffered in full.
+
+```typescript
+import { compressStream } from "zstd-stream";
+
+const file = document.querySelector<HTMLInputElement>("input[type=file]")!.files![0];
+const compressed = await compressStream(file.stream(), { level: 6 });
+
+await fetch("/upload", {
+  method: "POST",
+  headers: { "Content-Encoding": "zstd" },
+  body: compressed,
+  duplex: "half", // required when streaming a request body
+});
+```
+
+### Download and decompress
+
+```typescript
+import { decompressStream } from "zstd-stream";
+
+const res = await fetch("/data.zst");
+const decompressed = await decompressStream(res.body!);
+
+const text = await new Response(decompressed).text();
+```
+
+### Save a compressed file to disk (browser)
+
+`pipeTo` drives the whole transfer and applies backpressure for you.
+
+```typescript
+import { compressStream } from "zstd-stream";
+import streamSaver from "streamsaver";
+
+const compressed = await compressStream(file.stream(), {
+  level: 9,
+  onProgress: (bytes) => console.log(`${(bytes / 1e6).toFixed(1)} MB written`),
+});
+
+await compressed.pipeTo(streamSaver.createWriteStream(`${file.name}.zst`));
+```
+
+> **Tip:** compression is CPU-intensive. In the browser, run it inside a Web
+> Worker to keep the UI responsive.
+
+### Compress a file on disk (Node.js)
+
+```typescript
+import { createReadStream, createWriteStream } from "node:fs";
+import { Readable, Writable } from "node:stream";
+import { compressStream } from "zstd-stream";
+
+const source = Readable.toWeb(createReadStream("big.log")) as ReadableStream<Uint8Array>;
+const compressed = await compressStream(source, { level: 6 });
+
+await compressed.pipeTo(Writable.toWeb(createWriteStream("big.log.zst")));
+```
+
+### Small, in-memory data
+
+When you already hold the whole payload, skip the streams:
+
+```typescript
+import { compress, decompress } from "zstd-stream";
+
+const data = new TextEncoder().encode("Hello, world!");
+const compressed = await compress(data, { level: 3 });
+const restored = await decompress(compressed);
+
+console.log(new TextDecoder().decode(restored)); // "Hello, world!"
+```
+
+---
+
+## Memory & backpressure
+
+The streaming API is built to handle data far larger than available RAM:
+
+- **Bounded memory** — data flows through fixed ~128 KB working buffers and is
+  emitted one slice at a time. A compressed chunk that expands to gigabytes is
+  never decoded into a single allocation.
+- **Backpressure** — output is produced lazily, one slice per read, so a slow
+  consumer (disk, network) throttles reads from the source. A fast producer
+  cannot outrun a slow consumer and pile up in memory.
+- **Tunable buffering** — `highWaterMark` (bytes) sets how much output may queue
+  before reads pause. Larger values trade memory for throughput.
+
+The one-shot `compress` / `decompress` helpers, by contrast, hold the entire
+input and output in memory — use them only for small data.
+
+---
+
+## API reference
+
+All functions are async and lazily initialize the WASM module on first use.
+
+### `compressStream(input, options?)`
+
+Compress a stream. The primary API.
+
+- `input: ReadableStream<Uint8Array>`
+- `options?: CompressOptions`
+  - `level?: number` — compression level 1–19 (default: `3`)
+  - `onProgress?: (bytesWritten: number) => void` — cumulative compressed bytes
+  - `highWaterMark?: number` — output bytes buffered before backpressure pauses the source (default: 1 MiB)
+
+**Returns:** `Promise<ReadableStream<Uint8Array>>`
+
+### `decompressStream(input, options?)`
+
+Decompress a stream.
+
+- `input: ReadableStream<Uint8Array>`
+- `options?: DecompressOptions`
+  - `onProgress?: (bytesWritten: number) => void` — cumulative decompressed bytes
+  - `highWaterMark?: number` — output bytes buffered before backpressure pauses the source (default: 1 MiB)
+
+**Returns:** `Promise<ReadableStream<Uint8Array>>`
+
+### `compress(input, options?)`
+
+One-shot compression of an in-memory buffer. Holds all data in memory.
+
+- `input: Uint8Array`
+- `options?: CompressOptions` — `level`, `onProgress` (as above)
+
+**Returns:** `Promise<Uint8Array>`
+
+### `decompress(input, options?)`
+
+One-shot decompression of an in-memory buffer. Holds all data in memory.
+
+- `input: Uint8Array`
+- `options?: DecompressOptions` — `onProgress` (as above)
+
+**Returns:** `Promise<Uint8Array>`
+
+### `initialize()`
+
+Optional. Pre-loads the WASM module so the first compress/decompress call has no
+startup cost. Safe to call multiple times.
+
+**Returns:** `Promise<void>`
+
+```typescript
+import { initialize } from "zstd-stream";
+
+await initialize(); // e.g. during app startup
+```
+
+---
+
+## Compression levels
+
+Levels **1–19** are supported (default: `3`). Higher levels compress more but
+cost more time and memory; "ultra" levels 20–22 are intentionally excluded
+because, in streaming mode, they force every decompressor to allocate a 128 MB
+window — even for tiny inputs.
+
+Out-of-range values are clamped to the nearest valid level (with a
+`console.warn`) rather than throwing — e.g. `level: 22` runs as `19`.
+
+| Level | Speed     | Ratio    | Typical use                   |
+| ----- | --------- | -------- | ----------------------------- |
+| 1–3   | Fast      | Lower    | Real-time, network streaming  |
+| 4–7   | Balanced  | Good     | General purpose (recommended) |
+| 8–15  | Slow      | Better   | File storage, archival        |
+| 16–19 | Very slow | Maximum  | One-time / cold storage       |
+
+---
+
+## TypeScript
+
+```typescript
+import type { CompressOptions, DecompressOptions } from "zstd-stream";
+
+const options: CompressOptions = {
+  level: 9,
+  highWaterMark: 4 * 1024 * 1024,
+  onProgress: (bytes) => console.log(`Progress: ${bytes}`),
+};
+```
+
+---
+
+## Compatibility
+
+| Environment | Minimum |
+| ----------- | ------- |
+| Node.js     | 18      |
+| Chrome/Edge | 80      |
+| Firefox     | 113     |
+| Safari      | 16.4    |
+
+Requires WebAssembly and ES2022 support.
+
+---
+
+## License
+
+MIT
+
+Built with [Zstandard](https://github.com/facebook/zstd) by Meta, compiled with
+the [Emscripten SDK](https://emscripten.org/) and embedded for zero-dependency
+deployment.

package/dist/compressor.d.ts

@@ -5,23 +5,76 @@ declare abstract class BaseProcessor {
     destroy(): void;
     protected abstract cleanup(): void;
 }
+/**
+ * Incremental Zstandard compressor.
+ *
+ * The engine works on two fixed WASM buffers: an input buffer sized to
+ * `_cStreamInSize()` and an output buffer sized to `_cStreamOutSize()` (which
+ * zstd guarantees is large enough to flush one full block). Feeding no more
+ * than `inputBufferSize` bytes per call keeps every call's output within a
+ * single output buffer, so memory never scales with the total stream length.
+ */
 export declare class ZstdCompressor extends BaseProcessor {
-    private level;
     private ctx;
-    private buffer;
-    private bufferSize;
+    private inBuffer;
+    private outBuffer;
+    private outBufferSize;
+    inputBufferSize: number;
+    private readonly level;
     constructor(level: number);
     init(): Promise<void>;
+    /**
+     * Compress one input slice (`length` must be <= `inputBufferSize`). The
+     * slice is fully consumed; returns the bytes produced this call, which may
+     * be empty when the engine is still buffering toward a block boundary.
+     */
+    compressStep(data: Uint8Array, offset: number, length: number): Uint8Array;
+    /**
+     * Emit one output buffer of the closing frame. `done` is true once the frame
+     * has been fully flushed (i.e. the engine returned less than a full buffer).
+     */
+    compressFinish(): {
+        output: Uint8Array;
+        done: boolean;
+    };
+    /** Single-shot compression of an in-memory buffer. */
     process(data: Uint8Array, isLast: boolean): Uint8Array;
+    /** Run one `_compressStream` call and copy out any produced bytes. */
+    private run;
+    private requireModule;
     protected cleanup(): void;
 }
+/**
+ * Incremental Zstandard decompressor.
+ *
+ * Input is loaded into a fixed WASM buffer one `inputBufferSize`-sized slice at
+ * a time; `decompressStep()` then drains it one output buffer at a time. A
+ * single compressed slice can expand to far more than memory, so the caller is
+ * expected to enqueue each produced slice before requesting the next one.
+ */
 export declare class ZstdDecompressor extends BaseProcessor {
     private ctx;
-    private buffer;
-    private bufferSize;
+    private inBuffer;
+    private outBuffer;
+    private outBufferSize;
+    inputBufferSize: number;
+    private inLen;
+    private inPos;
     init(): Promise<void>;
+    /** Load one input slice (`length` must be <= `inputBufferSize`) for decoding. */
+    setInput(data: Uint8Array, offset: number, length: number): void;
+    /** True while the resident input slice still has bytes to decode. */
+    hasInput(): boolean;
+    /**
+     * Decode one output buffer's worth of data from the resident input slice.
+     * Returns the produced bytes (possibly empty). Advances the internal input
+     * position; throws if the engine can neither consume nor produce (corrupt
+     * input).
+     */
+    decompressStep(): Uint8Array;
+    /** Single-shot decompression of an in-memory buffer. */
     process(data: Uint8Array): Uint8Array;
-    private concat;
+    private requireModule;
     protected cleanup(): void;
 }
 export {};

package/dist/compressor.js

@@ -1,4 +1,44 @@
 import { getModule } from "./loader.js";
+const EMPTY = new Uint8Array(0);
+// ZSTD operation modes for the streaming wrapper.
+const ZSTD_E_CONTINUE = 0;
+const ZSTD_E_END = 2;
+// Supported compression levels. 20-22 ("ultra") are excluded: in streaming mode
+// they pin the window to 128 MB, forcing every decompressor to allocate 128 MB
+// even for tiny files, and cost hundreds of MB to compress.
+const MIN_LEVEL = 1;
+const MAX_LEVEL = 19;
+const DEFAULT_LEVEL = 3;
+/**
+ * Coerce a requested level into the supported 1-19 range, warning (rather than
+ * throwing) so a misconfigured caller still gets a working stream.
+ */
+function normalizeLevel(level) {
+    if (!Number.isFinite(level)) {
+        console.warn(`zstd-stream: invalid compression level ${level}; using default ${DEFAULT_LEVEL}.`);
+        return DEFAULT_LEVEL;
+    }
+    const valid = Math.min(MAX_LEVEL, Math.max(MIN_LEVEL, Math.round(level)));
+    if (valid !== level) {
+        console.warn(`zstd-stream: compression level ${level} is outside ${MIN_LEVEL}-${MAX_LEVEL}; using ${valid}.`);
+    }
+    return valid;
+}
+/** Join an array of chunks into a single contiguous Uint8Array. */
+function concat(chunks) {
+    if (chunks.length === 0)
+        return EMPTY;
+    if (chunks.length === 1)
+        return chunks[0];
+    const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+    const result = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return result;
+}
 class BaseProcessor {
     destroyed = false;
     module = null;
@@ -9,147 +49,221 @@ class BaseProcessor {
         }
     }
 }
+/**
+ * Incremental Zstandard compressor.
+ *
+ * The engine works on two fixed WASM buffers: an input buffer sized to
+ * `_cStreamInSize()` and an output buffer sized to `_cStreamOutSize()` (which
+ * zstd guarantees is large enough to flush one full block). Feeding no more
+ * than `inputBufferSize` bytes per call keeps every call's output within a
+ * single output buffer, so memory never scales with the total stream length.
+ */
 export class ZstdCompressor extends BaseProcessor {
-    level;
     ctx = 0;
-    buffer = 0;
-    bufferSize = 0;
+    inBuffer = 0;
+    outBuffer = 0;
+    outBufferSize = 0;
+    inputBufferSize = 0;
+    level;
     constructor(level) {
         super();
-        this.level = level;
-        if (level < 1 || level > 19) {
-            throw new Error("Compression level must be between 1 and 19");
-        }
+        this.level = normalizeLevel(level);
     }
     async init() {
-        this.module = getModule();
+        const m = (this.module = getModule());
+        this.ctx = m._createCCtx();
+        if (!this.ctx || m._initCStream(this.ctx, this.level) !== 0) {
+            this.cleanup();
+            throw new Error("Failed to create compression context");
+        }
+        this.outBufferSize = m._cStreamOutSize();
+        this.inputBufferSize = m._cStreamInSize();
+        this.outBuffer = m._malloc(this.outBufferSize);
+        this.inBuffer = m._malloc(this.inputBufferSize);
+        if (!this.outBuffer || !this.inBuffer) {
+            this.cleanup();
+            throw new Error("Failed to allocate compression buffers");
+        }
+    }
+    /**
+     * Compress one input slice (`length` must be <= `inputBufferSize`). The
+     * slice is fully consumed; returns the bytes produced this call, which may
+     * be empty when the engine is still buffering toward a block boundary.
+     */
+    compressStep(data, offset, length) {
+        const m = this.requireModule();
+        m.HEAPU8.set(data.subarray(offset, offset + length), this.inBuffer);
+        return this.run(length, ZSTD_E_CONTINUE);
     }
+    /**
+     * Emit one output buffer of the closing frame. `done` is true once the frame
+     * has been fully flushed (i.e. the engine returned less than a full buffer).
+     */
+    compressFinish() {
+        this.requireModule();
+        const output = this.run(0, ZSTD_E_END);
+        return { output, done: output.length < this.outBufferSize };
+    }
+    /** Single-shot compression of an in-memory buffer. */
     process(data, isLast) {
+        this.requireModule();
+        if (!data.length && !isLast)
+            return EMPTY;
+        const chunks = [];
+        let pos = 0;
+        while (pos < data.length) {
+            const n = Math.min(this.inputBufferSize, data.length - pos);
+            const out = this.compressStep(data, pos, n);
+            if (out.length)
+                chunks.push(out);
+            pos += n;
+        }
+        if (isLast) {
+            let result;
+            do {
+                result = this.compressFinish();
+                if (result.output.length)
+                    chunks.push(result.output);
+            } while (!result.done);
+        }
+        return concat(chunks);
+    }
+    /** Run one `_compressStream` call and copy out any produced bytes. */
+    run(length, mode) {
+        const m = this.module;
+        const result = Number(m._compressStream(this.ctx, this.outBuffer, this.outBufferSize, this.inBuffer, length, mode));
+        if (result < 0) {
+            throw new Error(`Compression failed: ${m._getErrorName(-result)}`);
+        }
+        return result === 0
+            ? EMPTY
+            : new Uint8Array(m.HEAPU8.subarray(this.outBuffer, this.outBuffer + result));
+    }
+    requireModule() {
         if (!this.module)
             throw new Error("Not initialized");
         if (this.destroyed)
             throw new Error("Destroyed");
-        if (!this.ctx) {
-            this.ctx = this.module._createCCtx();
-            if (!this.ctx || this.module._initCStream(this.ctx, this.level) !== 0) {
-                this.cleanup();
-                throw new Error("Failed to create compression context");
-            }
-        }
-        if (!data.length && !isLast)
-            return new Uint8Array(0);
-        const srcPtr = this.module._malloc(data.length);
-        if (!srcPtr)
-            throw new Error("Failed to allocate source buffer");
-        this.module.HEAPU8.set(data, srcPtr);
-        try {
-            if (!this.buffer) {
-                this.bufferSize = this.module._cStreamOutSize();
-                this.buffer = this.module._malloc(this.bufferSize);
-                if (!this.buffer)
-                    throw new Error("Failed to allocate output buffer");
-            }
-            const result = Number(this.module._compressStream(this.ctx, this.buffer, this.bufferSize, srcPtr, data.length, isLast ? 2 : 0));
-            if (result < 0) {
-                throw new Error(`Compression failed: ${this.module._getErrorName(-result)}`);
-            }
-            return result === 0
-                ? new Uint8Array(0)
-                : new Uint8Array(this.module.HEAPU8.subarray(this.buffer, this.buffer + result));
-        }
-        finally {
-            this.module._free(srcPtr);
-        }
+        return this.module;
     }
     cleanup() {
         if (this.module) {
             if (this.ctx)
                 this.module._freeCCtx(this.ctx);
-            if (this.buffer)
-                this.module._free(this.buffer);
+            if (this.outBuffer)
+                this.module._free(this.outBuffer);
+            if (this.inBuffer)
+                this.module._free(this.inBuffer);
             this.ctx = 0;
-            this.buffer = 0;
+            this.outBuffer = 0;
+            this.inBuffer = 0;
         }
     }
 }
+/**
+ * Incremental Zstandard decompressor.
+ *
+ * Input is loaded into a fixed WASM buffer one `inputBufferSize`-sized slice at
+ * a time; `decompressStep()` then drains it one output buffer at a time. A
+ * single compressed slice can expand to far more than memory, so the caller is
+ * expected to enqueue each produced slice before requesting the next one.
+ */
 export class ZstdDecompressor extends BaseProcessor {
     ctx = 0;
-    buffer = 0;
-    bufferSize = 0;
+    inBuffer = 0;
+    outBuffer = 0;
+    outBufferSize = 0;
+    inputBufferSize = 0;
+    // Position within the slice currently resident in the WASM input buffer.
+    inLen = 0;
+    inPos = 0;
     async init() {
-        this.module = getModule();
-    }
-    process(data) {
-        if (!this.module)
-            throw new Error("Not initialized");
-        if (this.destroyed)
-            throw new Error("Destroyed");
-        if (!this.ctx) {
-            this.ctx = this.module._createDCtx();
-            if (!this.ctx || this.module._initDStream(this.ctx) === 0) {
-                this.cleanup();
-                throw new Error("Failed to create decompression context");
-            }
+        const m = (this.module = getModule());
+        this.ctx = m._createDCtx();
+        if (!this.ctx || m._initDStream(this.ctx) === 0) {
+            this.cleanup();
+            throw new Error("Failed to create decompression context");
         }
-        if (!data.length)
-            return new Uint8Array(0);
-        if (!this.buffer) {
-            this.bufferSize = this.module._dStreamOutSize();
-            this.buffer = this.module._malloc(this.bufferSize);
-            if (!this.buffer)
-                throw new Error("Failed to allocate output buffer");
+        this.outBufferSize = m._dStreamOutSize();
+        this.inputBufferSize = m._dStreamInSize();
+        this.outBuffer = m._malloc(this.outBufferSize);
+        this.inBuffer = m._malloc(this.inputBufferSize);
+        if (!this.outBuffer || !this.inBuffer) {
+            this.cleanup();
+            throw new Error("Failed to allocate decompression buffers");
         }
-        const srcPtr = this.module._malloc(data.length);
-        if (!srcPtr)
-            throw new Error("Failed to allocate source buffer");
-        this.module.HEAPU8.set(data, srcPtr);
-        try {
-            let srcPos = 0;
-            const chunks = [];
-            while (srcPos < data.length) {
-                const result = this.module._decompressStream(this.ctx, this.buffer, this.bufferSize, srcPtr + srcPos, data.length - srcPos);
-                if (result < 0) {
-                    const errorCode = Number(BigInt(result) & 0x7fffffffffffffffn);
-                    throw new Error(`Decompression failed: ${this.module._getErrorName(errorCode)}`);
-                }
-                const consumed = Number(BigInt(result) >> 32n);
-                const outputSize = Number(BigInt(result) & 0xffffffffn);
-                if (consumed === 0 && outputSize === 0) {
-                    throw new Error("Decompression stalled");
-                }
-                if (outputSize > 0) {
-                    chunks.push(new Uint8Array(this.module.HEAPU8.subarray(this.buffer, this.buffer + outputSize)));
-                }
-                srcPos += consumed;
-            }
-            return this.concat(chunks);
+    }
+    /** Load one input slice (`length` must be <= `inputBufferSize`) for decoding. */
+    setInput(data, offset, length) {
+        const m = this.requireModule();
+        m.HEAPU8.set(data.subarray(offset, offset + length), this.inBuffer);
+        this.inLen = length;
+        this.inPos = 0;
+    }
+    /** True while the resident input slice still has bytes to decode. */
+    hasInput() {
+        return this.inPos < this.inLen;
+    }
+    /**
+     * Decode one output buffer's worth of data from the resident input slice.
+     * Returns the produced bytes (possibly empty). Advances the internal input
+     * position; throws if the engine can neither consume nor produce (corrupt
+     * input).
+     */
+    decompressStep() {
+        const m = this.requireModule();
+        const result = m._decompressStream(this.ctx, this.outBuffer, this.outBufferSize, this.inBuffer + this.inPos, this.inLen - this.inPos);
+        if (result < 0) {
+            const errorCode = Number(BigInt(result) & 0x7fffffffffffffffn);
+            throw new Error(`Decompression failed: ${m._getErrorName(errorCode)}`);
         }
-        finally {
-            this.module._free(srcPtr);
+        const consumed = Number(BigInt(result) >> 32n);
+        const outputSize = Number(BigInt(result) & 0xffffffffn);
+        if (consumed === 0 && outputSize === 0) {
+            throw new Error("Decompression stalled");
         }
+        this.inPos += consumed;
+        return outputSize === 0
+            ? EMPTY
+            : new Uint8Array(m.HEAPU8.subarray(this.outBuffer, this.outBuffer + outputSize));
     }
-    concat(chunks) {
-        if (chunks.length === 0)
-            return new Uint8Array(0);
-        if (chunks.length === 1)
-            return chunks[0];
-        const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
-        const result = new Uint8Array(total);
-        let offset = 0;
-        for (const chunk of chunks) {
-            result.set(chunk, offset);
-            offset += chunk.length;
+    /** Single-shot decompression of an in-memory buffer. */
+    process(data) {
+        this.requireModule();
+        if (!data.length)
+            return EMPTY;
+        const chunks = [];
+        let pos = 0;
+        while (pos < data.length) {
+            const n = Math.min(this.inputBufferSize, data.length - pos);
+            this.setInput(data, pos, n);
+            while (this.hasInput()) {
+                const out = this.decompressStep();
+                if (out.length)
+                    chunks.push(out);
+            }
+            pos += n;
         }
-        return result;
+        return concat(chunks);
+    }
+    requireModule() {
+        if (!this.module)
+            throw new Error("Not initialized");
+        if (this.destroyed)
+            throw new Error("Destroyed");
+        return this.module;
     }
     cleanup() {
         if (this.module) {
             if (this.ctx)
                 this.module._freeDCtx(this.ctx);
-            if (this.buffer)
-                this.module._free(this.buffer);
+            if (this.outBuffer)
+                this.module._free(this.outBuffer);
+            if (this.inBuffer)
+                this.module._free(this.inBuffer);
             this.ctx = 0;
-            this.buffer = 0;
+            this.outBuffer = 0;
+            this.inBuffer = 0;
         }
     }
 }

package/dist/index.d.ts

@@ -1,9 +1,16 @@
 export interface CompressOptions {
     level?: number;
     onProgress?: (bytesWritten: number) => void;
+    /**
+     * Max bytes of output buffered before backpressure pauses reading the
+     * source. Larger values trade memory for throughput. Default: 1 MiB.
+     */
+    highWaterMark?: number;
 }
 export interface DecompressOptions {
     onProgress?: (bytesWritten: number) => void;
+    /** See {@link CompressOptions.highWaterMark}. Default: 1 MiB. */
+    highWaterMark?: number;
 }
 export declare function compress(input: Uint8Array, options?: CompressOptions): Promise<Uint8Array>;
 export declare function compressStream(input: ReadableStream<Uint8Array>, options?: CompressOptions): Promise<ReadableStream<Uint8Array>>;

package/dist/index.js

@@ -1,5 +1,6 @@
 import { ZstdCompressor, ZstdDecompressor } from "./compressor.js";
 import { initZstd } from "./loader.js";
+const DEFAULT_HIGH_WATER_MARK = 1024 * 1024;
 let initialized = false;
 async function ensureInit() {
     if (!initialized) {
@@ -24,62 +25,71 @@ export async function compress(input, options) {
 }
 export async function compressStream(input, options) {
     await ensureInit();
-    const { level = 3, onProgress } = options || {};
+    const { level = 3, onProgress, highWaterMark = DEFAULT_HIGH_WATER_MARK } = options || {};
     const reader = input.getReader();
-    let compressor = null;
+    const compressor = new ZstdCompressor(level);
+    await compressor.init();
+    let chunk = null; // current source chunk
+    let chunkPos = 0; // bytes of `chunk` already fed
+    let finishing = false; // source drained, flushing the frame
     let totalWritten = 0;
-    let inputDone = false;
+    const release = () => {
+        reader.releaseLock();
+        compressor.destroy();
+    };
     return new ReadableStream({
-        async start() {
-            compressor = new ZstdCompressor(level);
-            await compressor.init();
-        },
+        // Produce exactly one output slice per pull so the returned stream's
+        // queue (and thus how far we read ahead of the consumer) stays bounded.
         async pull(controller) {
             try {
-                if (inputDone)
-                    return;
-                let loopIterations = 0;
-                // Keep reading until we have data to enqueue or input is exhausted
-                while (!inputDone) {
-                    loopIterations++;
-                    const { done, value } = await reader.read();
-                    if (done) {
-                        // Final flush
-                        const chunk = compressor.process(new Uint8Array(0), true);
-                        if (chunk.length > 0) {
-                            controller.enqueue(chunk);
-                            totalWritten += chunk.length;
+                while (true) {
+                    if (chunk && chunkPos < chunk.length) {
+                        const n = Math.min(compressor.inputBufferSize, chunk.length - chunkPos);
+                        const out = compressor.compressStep(chunk, chunkPos, n);
+                        chunkPos += n;
+                        if (out.length) {
+                            controller.enqueue(out);
+                            totalWritten += out.length;
                             onProgress?.(totalWritten);
+                            return;
                         }
-                        controller.close();
-                        reader.releaseLock();
-                        compressor?.destroy();
-                        inputDone = true;
-                        return;
+                        continue; // buffered internally; keep feeding
+                    }
+                    if (!finishing) {
+                        const { done, value } = await reader.read();
+                        if (done) {
+                            finishing = true;
+                            continue;
+                        }
+                        if (value.length === 0)
+                            continue;
+                        chunk = value;
+                        chunkPos = 0;
+                        continue;
                     }
-                    // Process chunk
-                    const chunk = compressor.process(value, false);
-                    if (chunk.length > 0) {
-                        controller.enqueue(chunk);
-                        totalWritten += chunk.length;
+                    // Flushing the final frame, one output buffer at a time.
+                    const { output, done } = compressor.compressFinish();
+                    if (output.length) {
+                        controller.enqueue(output);
+                        totalWritten += output.length;
                         onProgress?.(totalWritten);
-                        return; // Exit after enqueuing data
                     }
-                    // If chunk is empty, continue loop to read next input chunk
+                    if (done) {
+                        controller.close();
+                        release();
+                    }
+                    return;
                 }
             }
             catch (error) {
-                inputDone = true;
-                reader.releaseLock();
-                compressor?.destroy();
+                release();
                 controller.error(error);
             }
         },
         cancel() {
-            reader.releaseLock();
-            compressor?.destroy();
+            release();
         },
-    });
+    }, new ByteLengthQueuingStrategy({ highWaterMark }));
 }
 // Decompress Uint8Array -> Uint8Array
 export async function decompress(input, options) {
@@ -99,49 +109,61 @@ export async function decompress(input, options) {
 // Decompress ReadableStream -> ReadableStream with backpressure
 export async function decompressStream(input, options) {
     await ensureInit();
-    const { onProgress } = options || {};
+    const { onProgress, highWaterMark = DEFAULT_HIGH_WATER_MARK } = options || {};
     const reader = input.getReader();
-    let decompressor = null;
+    const decompressor = new ZstdDecompressor();
+    await decompressor.init();
+    let chunk = null; // current source chunk
+    let chunkPos = 0; // bytes of `chunk` already loaded into WASM
     let totalWritten = 0;
-    let inputDone = false;
+    const release = () => {
+        reader.releaseLock();
+        decompressor.destroy();
+    };
     return new ReadableStream({
-        async start() {
-            decompressor = new ZstdDecompressor();
-            await decompressor.init();
-        },
+        // One output slice per pull. A small compressed slice can expand to many
+        // gigabytes, so we never decode more than one output buffer ahead of the
+        // consumer.
         async pull(controller) {
             try {
-                if (inputDone) {
-                    return; // Already finished
-                }
-                const { done, value } = await reader.read();
-                if (done) {
-                    inputDone = true;
-                    controller.close();
-                    reader.releaseLock();
-                    decompressor?.destroy();
-                    return;
-                }
-                const chunk = decompressor.process(value);
-                if (chunk.length > 0) {
-                    controller.enqueue(chunk);
-                    totalWritten += chunk.length;
-                    onProgress?.(totalWritten);
+                while (true) {
+                    if (decompressor.hasInput()) {
+                        const out = decompressor.decompressStep();
+                        if (out.length) {
+                            controller.enqueue(out);
+                            totalWritten += out.length;
+                            onProgress?.(totalWritten);
+                            return;
+                        }
+                        continue; // consumed input without output yet
+                    }
+                    if (chunk && chunkPos < chunk.length) {
+                        const n = Math.min(decompressor.inputBufferSize, chunk.length - chunkPos);
+                        decompressor.setInput(chunk, chunkPos, n);
+                        chunkPos += n;
+                        continue;
+                    }
+                    const { done, value } = await reader.read();
+                    if (done) {
+                        controller.close();
+                        release();
+                        return;
+                    }
+                    if (value.length === 0)
+                        continue;
+                    chunk = value;
+                    chunkPos = 0;
                 }
             }
             catch (error) {
-                inputDone = true;
-                reader.releaseLock();
-                decompressor?.destroy();
+                release();
                 controller.error(error);
-                throw error;
             }
         },
         cancel() {
-            reader.releaseLock();
-            decompressor?.destroy();
+            release();
         },
-    });
+    }, new ByteLengthQueuingStrategy({ highWaterMark }));
 }
 export async function initialize() {
     await ensureInit();

package/dist/loader.d.ts

@@ -1,3 +1,3 @@
-import type { Module } from "./types.ts";
+import type { Module } from "./types.js";
 export declare function initZstd(): Promise<void>;
 export declare function getModule(): Module;

package/dist/loader.js

@@ -28,10 +28,8 @@ async function evaluateCode(code) {
             const mod = await import(/* @vite-ignore */ url);
             return mod.default || mod.ZstdWasm || mod;
         }
-        catch (e) {
-            if (e.code !== "ERR_UNKNOWN_FILE_EXTENSION") {
-                console.debug("Data URL import failed:", e?.message);
-            }
+        catch {
+            // Older Node (pre-22) can't import a data: URL — fall back to vm.
             const vm = await import("node:vm");
             // Create a context with required globals
             const context = vm.createContext({
@@ -51,8 +49,7 @@ async function evaluateCode(code) {
                     try {
                         return await import(/* @vite-ignore */ specifier);
                     }
-                    catch (e) {
-                        console.warn(`⚠️ Dynamic import failed: ${specifier}`, e.message);
+                    catch {
                         throw new Error(`Dynamic import not supported: ${specifier}`);
                     }
                 },

package/dist/types.d.ts

@@ -9,6 +9,8 @@ export interface Module {
     _decompressStream(ctx: number, dst: number, dstSize: number, src: number, srcSize: number): number;
     _dStreamOutSize(): number;
     _cStreamOutSize(): number;
+    _cStreamInSize(): number;
+    _dStreamInSize(): number;
     _getErrorName(error: number): string;
     _malloc(size: number): number;
     _free(ptr: number): void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "zstd-stream",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Simple and efficient Zstandard compression/decompression for Node.js and browsers",
   "type": "module",
   "main": "./dist/index.js",