zlib-streams 1.2.1 → 1.2.3

package/Makefile

@@ -296,6 +296,8 @@ run_all_tests: dist/zlib-streams-dev.wasm
 	@node src/wasm/tests/test_output_buffer_boundary.js dist/zlib-streams-dev.wasm
 	# Error codes packed the same way as a success, on both directions
 	@node src/wasm/tests/test_process_error_encoding.js dist/zlib-streams-dev.wasm
+	# Buffers and zlib state released whichever end ends the stream, with no transformer hook
+	@node src/wasm/tests/test_stream_abort_releases_memory.js dist/zlib-streams-dev.wasm
 	@echo "Completed run_all_tests"
 
 dist/zlib-streams-dev.wasm: $(WASM_SRCS)

package/dist/zlib-streams.js

@@ -26,7 +26,7 @@
  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-/* global TransformStream */
+/* global ReadableStream, WritableStream */
 
 const FORMAT_DEFLATE = "deflate";
 const FORMAT_DEFLATE_RAW = "deflate-raw";
@@ -61,142 +61,195 @@ function _make(isCompress, type, options = {}) {
 	const level = (typeof options.level === "number") ? options.level : -1;
 	const outBufferSize = (typeof options.outBuffer === "number") ? options.outBuffer : 64 * 1024;
 	const inBufferSize = (typeof options.inBufferSize === "number") ? options.inBufferSize : 64 * 1024;
-
-	return new TransformStream({
-		start() {
+	const state = { out: 0, in: 0, inBufferSize: 0, streamHandle: 0 };
+	let readableController, writableController, resolveWrite, cancelReason;
+	let backpressure = true;
+	let cancelled = false;
+	initStream();
+	const readable = new ReadableStream({
+		start(controller) {
+			readableController = controller;
+		},
+		pull() {
+			backpressure = false;
+			resumeWrite();
+		},
+		cancel(reason) {
+			cancelled = true;
+			cancelReason = reason;
+			disposeStream(state);
+			writableController.error(reason);
+			resumeWrite();
+		}
+	}, { highWaterMark: 0 });
+	const writable = new WritableStream({
+		start(controller) {
+			writableController = controller;
+		},
+		async write(chunk) {
+			if (backpressure) {
+				await new Promise(resolve => resolveWrite = resolve);
+			}
+			if (cancelled) {
+				throw cancelReason;
+			}
 			try {
-				let result;
-				this.out = malloc(outBufferSize);
-				this.in = malloc(inBufferSize);
-				this.inBufferSize = inBufferSize;
-				if (!this.out || !this.in) {
-					throw new Error("allocation failed");
-				}
-				this._scratch = new Uint8Array(outBufferSize);
-				if (isCompress) {
-					this._process = wasm.deflate_process;
-					this._last_consumed = wasm.deflate_last_consumed;
-					this._end = wasm.deflate_end;
-					this.streamHandle = wasm.deflate_new();
-					if (type === FORMAT_GZIP) {
-						result = wasm.deflate_init_gzip(this.streamHandle, level);
-					} else if (type === FORMAT_DEFLATE_RAW) {
-						result = wasm.deflate_init_raw(this.streamHandle, level);
-					} else {
-						result = wasm.deflate_init(this.streamHandle, level);
-					}
-				} else {
-					if (type === FORMAT_DEFLATE64_RAW) {
-						this._process = wasm.inflate9_process;
-						this._last_consumed = wasm.inflate9_last_consumed;
-						this._end = wasm.inflate9_end;
-						this.streamHandle = wasm.inflate9_new();
-						result = wasm.inflate9_init_raw(this.streamHandle);
-					} else {
-						this._process = wasm.inflate_process;
-						this._last_consumed = wasm.inflate_last_consumed;
-						this._end = wasm.inflate_end;
-						this.streamHandle = wasm.inflate_new();
-						if (type === FORMAT_DEFLATE_RAW) {
-							result = wasm.inflate_init_raw(this.streamHandle);
-						} else if (type === FORMAT_GZIP) {
-							result = wasm.inflate_init_gzip(this.streamHandle);
-						} else {
-							result = wasm.inflate_init(this.streamHandle);
-						}
-					}
-				}
-				if (result !== 0) {
-					throw new Error("init failed:" + result);
-				}
+				processChunk(chunk);
 			} catch (error) {
-				disposeStream(this);
+				disposeStream(state);
+				readableController.error(error);
 				throw error;
 			}
 		},
-		transform(chunk, controller) {
+		close() {
+			if (cancelled) {
+				return;
+			}
 			try {
-				const buffer = chunk;
-				const heap = new Uint8Array(memory.buffer);
-				const process = this._process;
-				const last_consumed = this._last_consumed;
-				const out = this.out;
-				const scratch = this._scratch;
-				let offset = 0;
-				while (offset < buffer.length) {
-					const toRead = Math.min(buffer.length - offset, 32 * 1024);
-					if (!this.in || this.inBufferSize < toRead) {
-						if (this.in && free) {
-							free(this.in);
-							this.in = 0;
-						}
-						this.in = malloc(toRead);
-						this.inBufferSize = toRead;
-						if (!this.in) {
-							throw new Error("allocation failed");
-						}
-					}
-					heap.set(buffer.subarray(offset, offset + toRead), this.in);
-					const result = process(this.streamHandle, this.in, toRead, out, outBufferSize, 0);
-					// checked before the byte count is used, so a status code can never be read as one
-					const code = (result >> 24) & 0xff;
-					const signedCode = (code & 0x80) ? code - 256 : code;
-					if (signedCode < 0) {
-						throw new Error("process error:" + signedCode);
-					}
-					const prod = result & 0x00ffffff;
-					if (prod) {
-						scratch.set(heap.subarray(out, out + prod), 0);
-						controller.enqueue(scratch.slice(0, prod));
-					}
-					const consumed = last_consumed(this.streamHandle);
-					if (consumed === 0 && prod === 0) {
-						break;
-					}
-					offset += consumed;
-				}
+				flushStream();
 			} catch (error) {
-				disposeStream(this);
-				controller.error(error);
+				disposeStream(state);
+				readableController.error(error);
+				throw error;
 			}
+			const result = disposeStream(state);
+			if (result !== 0) {
+				const error = new Error("end error:" + result);
+				readableController.error(error);
+				throw error;
+			}
+			readableController.close();
 		},
-		flush(controller) {
-			try {
-				const heap = new Uint8Array(memory.buffer);
-				const process = this._process;
-				const out = this.out;
-				const scratch = this._scratch;
-				while (true) {
-					const result = process(this.streamHandle, 0, 0, out, outBufferSize, 4);
-					const code = (result >> 24) & 0xff;
-					const signedCode = (code & 0x80) ? code - 256 : code;
-					if (signedCode < 0) {
-						throw new Error("process error:" + signedCode);
-					}
-					const produced = result & 0x00ffffff;
-					if (produced) {
-						scratch.set(heap.subarray(out, out + produced), 0);
-						controller.enqueue(scratch.slice(0, produced));
-					}
-					if (code === 1 || produced === 0) {
-						break;
+		abort(reason) {
+			disposeStream(state);
+			readableController.error(reason);
+		}
+	}, { highWaterMark: 1 });
+	return { readable, writable };
+
+	function initStream() {
+		try {
+			let result;
+			state.out = malloc(outBufferSize);
+			state.in = malloc(inBufferSize);
+			state.inBufferSize = inBufferSize;
+			if (!state.out || !state.in) {
+				throw new Error("allocation failed");
+			}
+			if (isCompress) {
+				state._process = wasm.deflate_process;
+				state._last_consumed = wasm.deflate_last_consumed;
+				state._end = wasm.deflate_end;
+				state.streamHandle = wasm.deflate_new();
+				if (type === FORMAT_GZIP) {
+					result = wasm.deflate_init_gzip(state.streamHandle, level);
+				} else if (type === FORMAT_DEFLATE_RAW) {
+					result = wasm.deflate_init_raw(state.streamHandle, level);
+				} else {
+					result = wasm.deflate_init(state.streamHandle, level);
+				}
+			} else {
+				if (type === FORMAT_DEFLATE64_RAW) {
+					state._process = wasm.inflate9_process;
+					state._last_consumed = wasm.inflate9_last_consumed;
+					state._end = wasm.inflate9_end;
+					state.streamHandle = wasm.inflate9_new();
+					result = wasm.inflate9_init_raw(state.streamHandle);
+				} else {
+					state._process = wasm.inflate_process;
+					state._last_consumed = wasm.inflate_last_consumed;
+					state._end = wasm.inflate_end;
+					state.streamHandle = wasm.inflate_new();
+					if (type === FORMAT_DEFLATE_RAW) {
+						result = wasm.inflate_init_raw(state.streamHandle);
+					} else if (type === FORMAT_GZIP) {
+						result = wasm.inflate_init_gzip(state.streamHandle);
+					} else {
+						result = wasm.inflate_init(state.streamHandle);
 					}
 				}
-			} catch (error) {
-				controller.error(error);
-			} finally {
-				const result = disposeStream(this);
-				if (result !== 0) {
-					controller.error(new Error("end error:" + result));
+			}
+			if (result !== 0) {
+				throw new Error("init failed:" + result);
+			}
+		} catch (error) {
+			disposeStream(state);
+			throw error;
+		}
+	}
+
+	function processChunk(chunk) {
+		const heap = new Uint8Array(memory.buffer);
+		const process = state._process;
+		const last_consumed = state._last_consumed;
+		const out = state.out;
+		let offset = 0;
+		while (offset < chunk.length) {
+			const toRead = Math.min(chunk.length - offset, 32 * 1024);
+			if (!state.in || state.inBufferSize < toRead) {
+				if (state.in && free) {
+					free(state.in);
+					state.in = 0;
+				}
+				state.in = malloc(toRead);
+				state.inBufferSize = toRead;
+				if (!state.in) {
+					throw new Error("allocation failed");
 				}
 			}
-		},
-		cancel() {
-			// release the stream handle and buffers when the pipeline is aborted,
-			// they would be leaked in the process-lifetime wasm heap otherwise
-			disposeStream(this);
+			heap.set(chunk.subarray(offset, offset + toRead), state.in);
+			const result = process(state.streamHandle, state.in, toRead, out, outBufferSize, 0);
+			// checked before the byte count is used, so a status code can never be read as one
+			const code = (result >> 24) & 0xff;
+			const signedCode = (code & 0x80) ? code - 256 : code;
+			if (signedCode < 0) {
+				throw new Error("process error:" + signedCode);
+			}
+			const prod = result & 0x00ffffff;
+			if (prod) {
+				enqueue(heap.slice(out, out + prod));
+			}
+			const consumed = last_consumed(state.streamHandle);
+			if (consumed === 0 && prod === 0) {
+				break;
+			}
+			offset += consumed;
+		}
+	}
+
+	function flushStream() {
+		const heap = new Uint8Array(memory.buffer);
+		const process = state._process;
+		const out = state.out;
+		while (true) {
+			const result = process(state.streamHandle, 0, 0, out, outBufferSize, 4);
+			const code = (result >> 24) & 0xff;
+			const signedCode = (code & 0x80) ? code - 256 : code;
+			if (signedCode < 0) {
+				throw new Error("process error:" + signedCode);
+			}
+			const produced = result & 0x00ffffff;
+			if (produced) {
+				enqueue(heap.slice(out, out + produced));
+			}
+			if (code === 1 || produced === 0) {
+				break;
+			}
 		}
-	});
+	}
+
+	function enqueue(chunk) {
+		backpressure = true;
+		readableController.enqueue(chunk);
+	}
+
+	function resumeWrite() {
+		if (resolveWrite) {
+			const resolve = resolveWrite;
+			resolveWrite = null;
+			resolve();
+		}
+	}
 
 	function disposeStream(state) {
 		let endResult = 0;

package/package.json

@@ -3,7 +3,7 @@
   "description": "WASM-based Compression Streams API implementation using zlib, with support for deflate64 decompression.",
   "author": "Gildas Lormeau",
   "license": "BSD-3-Clause",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "type": "module",
   "keywords": [
     "deflate",

package/src/wasm/api/zlib-streams.js

@@ -26,7 +26,7 @@
  EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
-/* global TransformStream */
+/* global ReadableStream, WritableStream */
 
 const FORMAT_DEFLATE = "deflate";
 const FORMAT_DEFLATE_RAW = "deflate-raw";
@@ -61,142 +61,195 @@ function _make(isCompress, type, options = {}) {
 	const level = (typeof options.level === "number") ? options.level : -1;
 	const outBufferSize = (typeof options.outBuffer === "number") ? options.outBuffer : 64 * 1024;
 	const inBufferSize = (typeof options.inBufferSize === "number") ? options.inBufferSize : 64 * 1024;
-
-	return new TransformStream({
-		start() {
+	const state = { out: 0, in: 0, inBufferSize: 0, streamHandle: 0 };
+	let readableController, writableController, resolveWrite, cancelReason;
+	let backpressure = true;
+	let cancelled = false;
+	initStream();
+	const readable = new ReadableStream({
+		start(controller) {
+			readableController = controller;
+		},
+		pull() {
+			backpressure = false;
+			resumeWrite();
+		},
+		cancel(reason) {
+			cancelled = true;
+			cancelReason = reason;
+			disposeStream(state);
+			writableController.error(reason);
+			resumeWrite();
+		}
+	}, { highWaterMark: 0 });
+	const writable = new WritableStream({
+		start(controller) {
+			writableController = controller;
+		},
+		async write(chunk) {
+			if (backpressure) {
+				await new Promise(resolve => resolveWrite = resolve);
+			}
+			if (cancelled) {
+				throw cancelReason;
+			}
 			try {
-				let result;
-				this.out = malloc(outBufferSize);
-				this.in = malloc(inBufferSize);
-				this.inBufferSize = inBufferSize;
-				if (!this.out || !this.in) {
-					throw new Error("allocation failed");
-				}
-				this._scratch = new Uint8Array(outBufferSize);
-				if (isCompress) {
-					this._process = wasm.deflate_process;
-					this._last_consumed = wasm.deflate_last_consumed;
-					this._end = wasm.deflate_end;
-					this.streamHandle = wasm.deflate_new();
-					if (type === FORMAT_GZIP) {
-						result = wasm.deflate_init_gzip(this.streamHandle, level);
-					} else if (type === FORMAT_DEFLATE_RAW) {
-						result = wasm.deflate_init_raw(this.streamHandle, level);
-					} else {
-						result = wasm.deflate_init(this.streamHandle, level);
-					}
-				} else {
-					if (type === FORMAT_DEFLATE64_RAW) {
-						this._process = wasm.inflate9_process;
-						this._last_consumed = wasm.inflate9_last_consumed;
-						this._end = wasm.inflate9_end;
-						this.streamHandle = wasm.inflate9_new();
-						result = wasm.inflate9_init_raw(this.streamHandle);
-					} else {
-						this._process = wasm.inflate_process;
-						this._last_consumed = wasm.inflate_last_consumed;
-						this._end = wasm.inflate_end;
-						this.streamHandle = wasm.inflate_new();
-						if (type === FORMAT_DEFLATE_RAW) {
-							result = wasm.inflate_init_raw(this.streamHandle);
-						} else if (type === FORMAT_GZIP) {
-							result = wasm.inflate_init_gzip(this.streamHandle);
-						} else {
-							result = wasm.inflate_init(this.streamHandle);
-						}
-					}
-				}
-				if (result !== 0) {
-					throw new Error("init failed:" + result);
-				}
+				processChunk(chunk);
 			} catch (error) {
-				disposeStream(this);
+				disposeStream(state);
+				readableController.error(error);
 				throw error;
 			}
 		},
-		transform(chunk, controller) {
+		close() {
+			if (cancelled) {
+				return;
+			}
 			try {
-				const buffer = chunk;
-				const heap = new Uint8Array(memory.buffer);
-				const process = this._process;
-				const last_consumed = this._last_consumed;
-				const out = this.out;
-				const scratch = this._scratch;
-				let offset = 0;
-				while (offset < buffer.length) {
-					const toRead = Math.min(buffer.length - offset, 32 * 1024);
-					if (!this.in || this.inBufferSize < toRead) {
-						if (this.in && free) {
-							free(this.in);
-							this.in = 0;
-						}
-						this.in = malloc(toRead);
-						this.inBufferSize = toRead;
-						if (!this.in) {
-							throw new Error("allocation failed");
-						}
-					}
-					heap.set(buffer.subarray(offset, offset + toRead), this.in);
-					const result = process(this.streamHandle, this.in, toRead, out, outBufferSize, 0);
-					// checked before the byte count is used, so a status code can never be read as one
-					const code = (result >> 24) & 0xff;
-					const signedCode = (code & 0x80) ? code - 256 : code;
-					if (signedCode < 0) {
-						throw new Error("process error:" + signedCode);
-					}
-					const prod = result & 0x00ffffff;
-					if (prod) {
-						scratch.set(heap.subarray(out, out + prod), 0);
-						controller.enqueue(scratch.slice(0, prod));
-					}
-					const consumed = last_consumed(this.streamHandle);
-					if (consumed === 0 && prod === 0) {
-						break;
-					}
-					offset += consumed;
-				}
+				flushStream();
 			} catch (error) {
-				disposeStream(this);
-				controller.error(error);
+				disposeStream(state);
+				readableController.error(error);
+				throw error;
 			}
+			const result = disposeStream(state);
+			if (result !== 0) {
+				const error = new Error("end error:" + result);
+				readableController.error(error);
+				throw error;
+			}
+			readableController.close();
 		},
-		flush(controller) {
-			try {
-				const heap = new Uint8Array(memory.buffer);
-				const process = this._process;
-				const out = this.out;
-				const scratch = this._scratch;
-				while (true) {
-					const result = process(this.streamHandle, 0, 0, out, outBufferSize, 4);
-					const code = (result >> 24) & 0xff;
-					const signedCode = (code & 0x80) ? code - 256 : code;
-					if (signedCode < 0) {
-						throw new Error("process error:" + signedCode);
-					}
-					const produced = result & 0x00ffffff;
-					if (produced) {
-						scratch.set(heap.subarray(out, out + produced), 0);
-						controller.enqueue(scratch.slice(0, produced));
-					}
-					if (code === 1 || produced === 0) {
-						break;
+		abort(reason) {
+			disposeStream(state);
+			readableController.error(reason);
+		}
+	}, { highWaterMark: 1 });
+	return { readable, writable };
+
+	function initStream() {
+		try {
+			let result;
+			state.out = malloc(outBufferSize);
+			state.in = malloc(inBufferSize);
+			state.inBufferSize = inBufferSize;
+			if (!state.out || !state.in) {
+				throw new Error("allocation failed");
+			}
+			if (isCompress) {
+				state._process = wasm.deflate_process;
+				state._last_consumed = wasm.deflate_last_consumed;
+				state._end = wasm.deflate_end;
+				state.streamHandle = wasm.deflate_new();
+				if (type === FORMAT_GZIP) {
+					result = wasm.deflate_init_gzip(state.streamHandle, level);
+				} else if (type === FORMAT_DEFLATE_RAW) {
+					result = wasm.deflate_init_raw(state.streamHandle, level);
+				} else {
+					result = wasm.deflate_init(state.streamHandle, level);
+				}
+			} else {
+				if (type === FORMAT_DEFLATE64_RAW) {
+					state._process = wasm.inflate9_process;
+					state._last_consumed = wasm.inflate9_last_consumed;
+					state._end = wasm.inflate9_end;
+					state.streamHandle = wasm.inflate9_new();
+					result = wasm.inflate9_init_raw(state.streamHandle);
+				} else {
+					state._process = wasm.inflate_process;
+					state._last_consumed = wasm.inflate_last_consumed;
+					state._end = wasm.inflate_end;
+					state.streamHandle = wasm.inflate_new();
+					if (type === FORMAT_DEFLATE_RAW) {
+						result = wasm.inflate_init_raw(state.streamHandle);
+					} else if (type === FORMAT_GZIP) {
+						result = wasm.inflate_init_gzip(state.streamHandle);
+					} else {
+						result = wasm.inflate_init(state.streamHandle);
 					}
 				}
-			} catch (error) {
-				controller.error(error);
-			} finally {
-				const result = disposeStream(this);
-				if (result !== 0) {
-					controller.error(new Error("end error:" + result));
+			}
+			if (result !== 0) {
+				throw new Error("init failed:" + result);
+			}
+		} catch (error) {
+			disposeStream(state);
+			throw error;
+		}
+	}
+
+	function processChunk(chunk) {
+		const heap = new Uint8Array(memory.buffer);
+		const process = state._process;
+		const last_consumed = state._last_consumed;
+		const out = state.out;
+		let offset = 0;
+		while (offset < chunk.length) {
+			const toRead = Math.min(chunk.length - offset, 32 * 1024);
+			if (!state.in || state.inBufferSize < toRead) {
+				if (state.in && free) {
+					free(state.in);
+					state.in = 0;
+				}
+				state.in = malloc(toRead);
+				state.inBufferSize = toRead;
+				if (!state.in) {
+					throw new Error("allocation failed");
 				}
 			}
-		},
-		cancel() {
-			// release the stream handle and buffers when the pipeline is aborted,
-			// they would be leaked in the process-lifetime wasm heap otherwise
-			disposeStream(this);
+			heap.set(chunk.subarray(offset, offset + toRead), state.in);
+			const result = process(state.streamHandle, state.in, toRead, out, outBufferSize, 0);
+			// checked before the byte count is used, so a status code can never be read as one
+			const code = (result >> 24) & 0xff;
+			const signedCode = (code & 0x80) ? code - 256 : code;
+			if (signedCode < 0) {
+				throw new Error("process error:" + signedCode);
+			}
+			const prod = result & 0x00ffffff;
+			if (prod) {
+				enqueue(heap.slice(out, out + prod));
+			}
+			const consumed = last_consumed(state.streamHandle);
+			if (consumed === 0 && prod === 0) {
+				break;
+			}
+			offset += consumed;
+		}
+	}
+
+	function flushStream() {
+		const heap = new Uint8Array(memory.buffer);
+		const process = state._process;
+		const out = state.out;
+		while (true) {
+			const result = process(state.streamHandle, 0, 0, out, outBufferSize, 4);
+			const code = (result >> 24) & 0xff;
+			const signedCode = (code & 0x80) ? code - 256 : code;
+			if (signedCode < 0) {
+				throw new Error("process error:" + signedCode);
+			}
+			const produced = result & 0x00ffffff;
+			if (produced) {
+				enqueue(heap.slice(out, out + produced));
+			}
+			if (code === 1 || produced === 0) {
+				break;
+			}
 		}
-	});
+	}
+
+	function enqueue(chunk) {
+		backpressure = true;
+		readableController.enqueue(chunk);
+	}
+
+	function resumeWrite() {
+		if (resolveWrite) {
+			const resolve = resolveWrite;
+			resolveWrite = null;
+			resolve();
+		}
+	}
 
 	function disposeStream(state) {
 		let endResult = 0;