zlib-streams 1.2.3 → 1.4.0

package/Makefile

@@ -298,6 +298,9 @@ run_all_tests: dist/zlib-streams-dev.wasm
 	@node src/wasm/tests/test_process_error_encoding.js dist/zlib-streams-dev.wasm
 	# Buffers and zlib state released whichever end ends the stream, with no transformer hook
 	@node src/wasm/tests/test_stream_abort_releases_memory.js dist/zlib-streams-dev.wasm
+	# Bytes after the end of the stream rejected, as the platform inflaters do
+	@node src/wasm/tests/test_trailing_data.js dist/zlib-streams-dev.wasm
+	@node src/wasm/tests/test_unknown_format.js dist/zlib-streams-dev.wasm
 	@echo "Completed run_all_tests"
 
 dist/zlib-streams-dev.wasm: $(WASM_SRCS)

package/README.md

@@ -42,5 +42,45 @@ Output: `dist/zip-module.wasm`
 
 The files in `dist/` are tracked, so a checkout leaves them with the same timestamp as the sources and `make dist/<file>` reports them up to date even when the sources changed. The three targets above always rebuild their file; the test targets depend on the file rules and rebuild only when a source is newer. Rebuild and commit `dist/` in the same commit as a change to `src/`.
 
+## API
+
+The npm package exports the JavaScript driver as `zlib-streams/zlib-streams.js` and the modules as `zlib-streams/zlib-streams.wasm` and `zlib-streams/zip-module.wasm`. The module imports nothing, so it is instantiated bare, and its exports are handed to the driver before any stream is created.
+
+```js
+import { setWasmExports, CompressionStreamZlib, DecompressionStreamZlib } from "zlib-streams/zlib-streams.js";
+
+const wasmBytes = await (await fetch(import.meta.resolve("zlib-streams/zlib-streams.wasm"))).arrayBuffer();
+const { instance } = await WebAssembly.instantiate(wasmBytes);
+setWasmExports(instance.exports);
+
+const compressed = new Blob(["Hello World"]).stream().pipeThrough(new CompressionStreamZlib("gzip", { level: 9 }));
+const text = await new Response(compressed.pipeThrough(new DecompressionStreamZlib("gzip"))).text();
+```
+
+### Driver functions
+
+- `setWasmExports(exports)`: registers the exports of an instantiated module (`malloc`, `free`, `memory` and the codec functions); throws `"Invalid WASM module"` when they are missing.
+- `resetWasmExports()`: forgets the exports; the next stream throws `"WASM module not loaded"`.
+- `setInitError(error)`: records why the module could not be loaded; the error is the `cause` of `"WASM module not loaded"`.
+
+### Streams
+
+```js
+new CompressionStreamZlib(format = "deflate", { level, outBuffer, inBufferSize })
+new DecompressionStreamZlib(format = "deflate", { outBuffer, inBufferSize })
+```
+
+Both classes return a `{ readable, writable }` pair usable with `pipeThrough()`. The formats are `"deflate"` (the zlib wrapper, RFC 1950), `"deflate-raw"` (RFC 1951, no wrapper), `"gzip"` and, for decompression only, `"deflate64-raw"`. `level` is the zlib compression level, 0 to 9, -1 (the default) for the zlib default. `outBuffer` and `inBufferSize` are the sizes in bytes of the output and input buffers allocated in the module heap, 64 KB each by default; input chunks are processed in slices of at most 32 KB.
+
+The static properties `supportedFormats` (an array) and `requiresModule` (`true`) let a host check the formats and know that the classes are unusable until `setWasmExports()` has run.
+
+### Errors
+
+- Bytes following the end of a stream are rejected with `"trailing data after the end of the stream"`, whatever the format, as the `DecompressionStream` of the browsers does; concatenated gzip members are not decoded.
+- A corrupted stream fails with `"process error:<code>"` carrying the zlib return code; a stream whose end is missing fails on close with `"end error:<code>"`.
+- A failure to allocate memory, on construction or while processing, carries the property `code` set to `"Z_MEM_ERROR"`: `"allocation failed"` when the heap is exhausted, `"init failed:-4"` or `"process error:-4"` when zlib reports it.
+
+The heap of the module is fixed, so the number of streams alive at the same time is bounded by their buffers; a stream releases its buffers when it closes, errors or is cancelled.
+
 ## License
 See LICENSE for details.

package/dist/zlib-streams.js

@@ -32,6 +32,10 @@ const FORMAT_DEFLATE = "deflate";
 const FORMAT_DEFLATE_RAW = "deflate-raw";
 const FORMAT_DEFLATE64_RAW = "deflate64-raw";
 const FORMAT_GZIP = "gzip";
+const Z_MEM_ERROR = -4;
+const Z_MEM_ERROR_CODE = "Z_MEM_ERROR";
+const COMPRESSION_FORMATS = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP];
+const DECOMPRESSION_FORMATS = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP, FORMAT_DEFLATE64_RAW];
 
 let wasm, malloc, free, memory, initError;
 
@@ -52,7 +56,17 @@ export function resetWasmExports() {
 	wasm = malloc = free = memory = initError = null;
 }
 
+function setZlibCode(error, result) {
+	if (result === Z_MEM_ERROR) {
+		error.code = Z_MEM_ERROR_CODE;
+	}
+	return error;
+}
+
 function _make(isCompress, type, options = {}) {
+	if (!(isCompress ? COMPRESSION_FORMATS : DECOMPRESSION_FORMATS).includes(type)) {
+		throw new TypeError("Unsupported format: " + type);
+	}
 	if (!wasm) {
 		const error = new Error("WASM module not loaded");
 		error.cause = initError;
@@ -61,7 +75,7 @@ function _make(isCompress, type, options = {}) {
 	const level = (typeof options.level === "number") ? options.level : -1;
 	const outBufferSize = (typeof options.outBuffer === "number") ? options.outBuffer : 64 * 1024;
 	const inBufferSize = (typeof options.inBufferSize === "number") ? options.inBufferSize : 64 * 1024;
-	const state = { out: 0, in: 0, inBufferSize: 0, streamHandle: 0 };
+	const state = { out: 0, in: 0, inBufferSize: 0, streamHandle: 0, streamEnded: false };
 	let readableController, writableController, resolveWrite, cancelReason;
 	let backpressure = true;
 	let cancelled = false;
@@ -114,7 +128,7 @@ function _make(isCompress, type, options = {}) {
 			}
 			const result = disposeStream(state);
 			if (result !== 0) {
-				const error = new Error("end error:" + result);
+				const error = setZlibCode(new Error("end error:" + result), result);
 				readableController.error(error);
 				throw error;
 			}
@@ -134,7 +148,7 @@ function _make(isCompress, type, options = {}) {
 			state.in = malloc(inBufferSize);
 			state.inBufferSize = inBufferSize;
 			if (!state.out || !state.in) {
-				throw new Error("allocation failed");
+				throw setZlibCode(new Error("allocation failed"), Z_MEM_ERROR);
 			}
 			if (isCompress) {
 				state._process = wasm.deflate_process;
@@ -170,7 +184,7 @@ function _make(isCompress, type, options = {}) {
 				}
 			}
 			if (result !== 0) {
-				throw new Error("init failed:" + result);
+				throw setZlibCode(new Error("init failed:" + result), result);
 			}
 		} catch (error) {
 			disposeStream(state);
@@ -185,6 +199,9 @@ function _make(isCompress, type, options = {}) {
 		const out = state.out;
 		let offset = 0;
 		while (offset < chunk.length) {
+			if (state.streamEnded) {
+				throw new Error("trailing data after the end of the stream");
+			}
 			const toRead = Math.min(chunk.length - offset, 32 * 1024);
 			if (!state.in || state.inBufferSize < toRead) {
 				if (state.in && free) {
@@ -194,7 +211,7 @@ function _make(isCompress, type, options = {}) {
 				state.in = malloc(toRead);
 				state.inBufferSize = toRead;
 				if (!state.in) {
-					throw new Error("allocation failed");
+					throw setZlibCode(new Error("allocation failed"), Z_MEM_ERROR);
 				}
 			}
 			heap.set(chunk.subarray(offset, offset + toRead), state.in);
@@ -203,14 +220,16 @@ function _make(isCompress, type, options = {}) {
 			const code = (result >> 24) & 0xff;
 			const signedCode = (code & 0x80) ? code - 256 : code;
 			if (signedCode < 0) {
-				throw new Error("process error:" + signedCode);
+				throw setZlibCode(new Error("process error:" + signedCode), signedCode);
 			}
 			const prod = result & 0x00ffffff;
 			if (prod) {
 				enqueue(heap.slice(out, out + prod));
 			}
 			const consumed = last_consumed(state.streamHandle);
-			if (consumed === 0 && prod === 0) {
+			if (code === 1) {
+				state.streamEnded = true;
+			} else if (consumed === 0 && prod === 0) {
 				break;
 			}
 			offset += consumed;
@@ -226,7 +245,7 @@ function _make(isCompress, type, options = {}) {
 			const code = (result >> 24) & 0xff;
 			const signedCode = (code & 0x80) ? code - 256 : code;
 			if (signedCode < 0) {
-				throw new Error("process error:" + signedCode);
+				throw setZlibCode(new Error("process error:" + signedCode), signedCode);
 			}
 			const produced = result & 0x00ffffff;
 			if (produced) {
@@ -285,7 +304,7 @@ export class DecompressionStreamZlib {
 CompressionStreamZlib.requiresModule = true;
 DecompressionStreamZlib.requiresModule = true;
 // Constructing these classes before the module is loaded throws, so capability probes cannot rely
-// on trying the constructor; the formats are declared instead, next to the branches implementing
-// them in _make().
-CompressionStreamZlib.supportedFormats = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP];
-DecompressionStreamZlib.supportedFormats = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP, FORMAT_DEFLATE64_RAW];
+// on trying the constructor; the formats are declared instead, the lists _make() checks a format
+// against before anything else.
+CompressionStreamZlib.supportedFormats = COMPRESSION_FORMATS;
+DecompressionStreamZlib.supportedFormats = DECOMPRESSION_FORMATS;

package/package.json

@@ -3,7 +3,7 @@
   "description": "WASM-based Compression Streams API implementation using zlib, with support for deflate64 decompression.",
   "author": "Gildas Lormeau",
   "license": "BSD-3-Clause",
-  "version": "1.2.3",
+  "version": "1.4.0",
   "type": "module",
   "keywords": [
     "deflate",

package/src/wasm/api/zlib-streams.js

@@ -32,6 +32,10 @@ const FORMAT_DEFLATE = "deflate";
 const FORMAT_DEFLATE_RAW = "deflate-raw";
 const FORMAT_DEFLATE64_RAW = "deflate64-raw";
 const FORMAT_GZIP = "gzip";
+const Z_MEM_ERROR = -4;
+const Z_MEM_ERROR_CODE = "Z_MEM_ERROR";
+const COMPRESSION_FORMATS = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP];
+const DECOMPRESSION_FORMATS = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP, FORMAT_DEFLATE64_RAW];
 
 let wasm, malloc, free, memory, initError;
 
@@ -52,7 +56,17 @@ export function resetWasmExports() {
 	wasm = malloc = free = memory = initError = null;
 }
 
+function setZlibCode(error, result) {
+	if (result === Z_MEM_ERROR) {
+		error.code = Z_MEM_ERROR_CODE;
+	}
+	return error;
+}
+
 function _make(isCompress, type, options = {}) {
+	if (!(isCompress ? COMPRESSION_FORMATS : DECOMPRESSION_FORMATS).includes(type)) {
+		throw new TypeError("Unsupported format: " + type);
+	}
 	if (!wasm) {
 		const error = new Error("WASM module not loaded");
 		error.cause = initError;
@@ -61,7 +75,7 @@ function _make(isCompress, type, options = {}) {
 	const level = (typeof options.level === "number") ? options.level : -1;
 	const outBufferSize = (typeof options.outBuffer === "number") ? options.outBuffer : 64 * 1024;
 	const inBufferSize = (typeof options.inBufferSize === "number") ? options.inBufferSize : 64 * 1024;
-	const state = { out: 0, in: 0, inBufferSize: 0, streamHandle: 0 };
+	const state = { out: 0, in: 0, inBufferSize: 0, streamHandle: 0, streamEnded: false };
 	let readableController, writableController, resolveWrite, cancelReason;
 	let backpressure = true;
 	let cancelled = false;
@@ -114,7 +128,7 @@ function _make(isCompress, type, options = {}) {
 			}
 			const result = disposeStream(state);
 			if (result !== 0) {
-				const error = new Error("end error:" + result);
+				const error = setZlibCode(new Error("end error:" + result), result);
 				readableController.error(error);
 				throw error;
 			}
@@ -134,7 +148,7 @@ function _make(isCompress, type, options = {}) {
 			state.in = malloc(inBufferSize);
 			state.inBufferSize = inBufferSize;
 			if (!state.out || !state.in) {
-				throw new Error("allocation failed");
+				throw setZlibCode(new Error("allocation failed"), Z_MEM_ERROR);
 			}
 			if (isCompress) {
 				state._process = wasm.deflate_process;
@@ -170,7 +184,7 @@ function _make(isCompress, type, options = {}) {
 				}
 			}
 			if (result !== 0) {
-				throw new Error("init failed:" + result);
+				throw setZlibCode(new Error("init failed:" + result), result);
 			}
 		} catch (error) {
 			disposeStream(state);
@@ -185,6 +199,9 @@ function _make(isCompress, type, options = {}) {
 		const out = state.out;
 		let offset = 0;
 		while (offset < chunk.length) {
+			if (state.streamEnded) {
+				throw new Error("trailing data after the end of the stream");
+			}
 			const toRead = Math.min(chunk.length - offset, 32 * 1024);
 			if (!state.in || state.inBufferSize < toRead) {
 				if (state.in && free) {
@@ -194,7 +211,7 @@ function _make(isCompress, type, options = {}) {
 				state.in = malloc(toRead);
 				state.inBufferSize = toRead;
 				if (!state.in) {
-					throw new Error("allocation failed");
+					throw setZlibCode(new Error("allocation failed"), Z_MEM_ERROR);
 				}
 			}
 			heap.set(chunk.subarray(offset, offset + toRead), state.in);
@@ -203,14 +220,16 @@ function _make(isCompress, type, options = {}) {
 			const code = (result >> 24) & 0xff;
 			const signedCode = (code & 0x80) ? code - 256 : code;
 			if (signedCode < 0) {
-				throw new Error("process error:" + signedCode);
+				throw setZlibCode(new Error("process error:" + signedCode), signedCode);
 			}
 			const prod = result & 0x00ffffff;
 			if (prod) {
 				enqueue(heap.slice(out, out + prod));
 			}
 			const consumed = last_consumed(state.streamHandle);
-			if (consumed === 0 && prod === 0) {
+			if (code === 1) {
+				state.streamEnded = true;
+			} else if (consumed === 0 && prod === 0) {
 				break;
 			}
 			offset += consumed;
@@ -226,7 +245,7 @@ function _make(isCompress, type, options = {}) {
 			const code = (result >> 24) & 0xff;
 			const signedCode = (code & 0x80) ? code - 256 : code;
 			if (signedCode < 0) {
-				throw new Error("process error:" + signedCode);
+				throw setZlibCode(new Error("process error:" + signedCode), signedCode);
 			}
 			const produced = result & 0x00ffffff;
 			if (produced) {
@@ -285,7 +304,7 @@ export class DecompressionStreamZlib {
 CompressionStreamZlib.requiresModule = true;
 DecompressionStreamZlib.requiresModule = true;
 // Constructing these classes before the module is loaded throws, so capability probes cannot rely
-// on trying the constructor; the formats are declared instead, next to the branches implementing
-// them in _make().
-CompressionStreamZlib.supportedFormats = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP];
-DecompressionStreamZlib.supportedFormats = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP, FORMAT_DEFLATE64_RAW];
+// on trying the constructor; the formats are declared instead, the lists _make() checks a format
+// against before anything else.
+CompressionStreamZlib.supportedFormats = COMPRESSION_FORMATS;
+DecompressionStreamZlib.supportedFormats = DECOMPRESSION_FORMATS;