zlib-streams 1.0.6 → 1.1.0

package/Makefile

@@ -1,6 +1,11 @@
 CC=gcc
-CFLAGS=-I./src/zlib -I./src -I./test -I./src/zlib/contrib/infback9 -O2 -w -MMD -MP
-DEBUG_CFLAGS=-I./src/zlib -I./src -I./test -I./src/zlib/contrib/infback9 -O0 -g3 -ggdb3 -fno-inline -fno-omit-frame-pointer -w -MMD -MP
+# INFLATE_CHUNK_GENERIC selects the portable chunk copies in chunkcopy.h, which has
+# no wasm SIMD path and would #error without one of its three arch macros.
+# INFLATE_CHUNK_READ_64LE buffers the input bits in a uint64_t inside inffast_chunk.c
+# only; state->hold stays unsigned long. Both are needed wherever inflate.c is built.
+INFLATE_CHUNK_CFLAGS = -DINFLATE_CHUNK_GENERIC -DINFLATE_CHUNK_READ_64LE
+CFLAGS=-I./src/zlib -I./src -I./test -I./src/zlib/contrib/infback9 -O2 -w -MMD -MP $(INFLATE_CHUNK_CFLAGS)
+DEBUG_CFLAGS=-I./src/zlib -I./src -I./test -I./src/zlib/contrib/infback9 -O0 -g3 -ggdb3 -fno-inline -fno-omit-frame-pointer -w -MMD -MP $(INFLATE_CHUNK_CFLAGS)
 ## By default we don't enable the INF9 tracing macros so debug binaries
 ## don't embed large diagnostic strings. To enable tracing for local
 ## debugging, uncomment and set DEBUG_DEFINES accordingly.
@@ -9,18 +14,18 @@ DEBUG_CFLAGS=-I./src/zlib -I./src -I./test -I./src/zlib/contrib/infback9 -O0 -g3
 ## Debug defines are disabled by default for a clean commit-ready Makefile.
 DEBUG_DEFINES=
 DEBUG_DEFINES_TRACED=
-SRC=src/inflate9.c src/zlib/contrib/infback9/inftree9.c src/zlib/zutil.c src/zlib/inflate.c src/zlib/inftrees.c
+SRC=src/inflate.c src/inffast_chunk.c src/inftrees.c src/zlib/zutil.c
 TESTSRC=zlib/crc32.c
 LIBS=-lz
 
 ## Build the payload decompressor test binary used for verification (deduplicated sources)
 PD_SRCS = test/payload_decompress.c \
-	src/inflate9.c src/zlib/inffast.c src/zlib/inftrees.c src/zlib/zutil.c src/zlib/inflate.c src/zlib/infback.c \
-	src/zlib/contrib/infback9/infback9.c src/zlib/contrib/infback9/inftree9.c src/zlib/crc32.c src/zlib/adler32.c src/zlib/trees.c
+	src/inflate.c src/inffast_chunk.c src/inftrees.c src/zlib/zutil.c \
+	src/zlib/crc32.c src/zlib/adler32.c
 
 PD_NOWINDOW_SRCS = test/payload_decompress_nowindow.c \
-	src/inflate9.c src/zlib/inffast.c src/zlib/inftrees.c src/zlib/zutil.c src/zlib/inflate.c src/zlib/infback.c \
-	src/zlib/contrib/infback9/infback9.c src/zlib/contrib/infback9/inftree9.c src/zlib/crc32.c src/zlib/adler32.c src/zlib/trees.c
+	src/inflate.c src/inffast_chunk.c src/inftrees.c src/zlib/zutil.c \
+	src/zlib/crc32.c src/zlib/adler32.c
 
 PD_NOWINDOW_DEBUG_OBJS = $(PD_NOWINDOW_SRCS:%.c=build/debug/%.o)
 
@@ -128,17 +133,22 @@ ci:
 # WASM build target (convenience target to produce dist/zlib-streams-dev.wasm)
 # -----------------------------------------------------------------------------
 EMCC ?= emsdk/upstream/emscripten/emcc
+WASM_OPT ?= emsdk/upstream/bin/wasm-opt
 
-WASM_SRCS = src/wasm/inflate9_stream_wasm.c src/wasm/inflate_stream_wasm.c src/wasm/deflate_stream_wasm.c src/wasm/wasm_stream_common.c src/wasm/allocator.c src/inflate9.c \
-	src/zlib/contrib/infback9/inftree9.c \
-	src/zlib/inffast.c src/zlib/inflate.c src/zlib/inftrees.c src/zlib/zutil.c \
-	src/zlib/crc32.c src/zlib/adler32.c src/zlib/trees.c src/zlib/deflate.c
+WASM_SRCS = src/wasm/inflate9_stream_wasm.c src/wasm/inflate_stream_wasm.c src/wasm/deflate_stream_wasm.c src/wasm/wasm_stream_common.c src/wasm/allocator.c \
+	src/inflate.c src/inffast_chunk.c src/inftrees.c src/zlib/zutil.c \
+	src/zlib/crc32.c src/zlib/adler32.c src/trees.c src/deflate.c src/deflate_crc32.c
 # CRC-32: -DZ_SOLO suppresses zlib's Z_U4/Z_U8 word types, which makes crc32.c's braid
 # path (#elif defined(Z_U4)) fall back to a byte-at-a-time loop. Restore the types and
 # force the 8-byte braid (Z_TESTW=8) so the slicing/braided CRC compiles for wasm:
 # ~12x faster CRC (342 -> ~4100 MB/s), bit-identical output. wasm32 has native i64.
 WASM_CRC_CFLAGS = -DZ_U4=unsigned -DZ_U8='unsigned long long' -DZ_TESTW=8
-WASM_CFLAGS = -Isrc -Isrc/zlib -Isrc/zlib/contrib/infback9 -O2 -flto -DDYNAMIC_CRC_TABLE -DBUILDFIXED -DZ_SOLO $(WASM_CRC_CFLAGS)
+# DEFLATE_COMPARE256_64LE extends matches 8 bytes at a time in longest_match, which is
+# the same match length computed faster: ~7% at every level, byte-identical output.
+# USE_ZLIB_RABIN_KARP_ROLLING_HASH turns Chromium's own hash OFF so the deflate stream
+# stays bit-for-bit what madler's deflate.c produces.
+WASM_DEFLATE_CFLAGS = -DDEFLATE_COMPARE256_64LE -DUSE_ZLIB_RABIN_KARP_ROLLING_HASH
+WASM_CFLAGS = -Isrc -Isrc/zlib -Isrc/zlib/contrib/infback9 -O2 -flto -DDYNAMIC_CRC_TABLE -DBUILDFIXED -DZ_SOLO $(WASM_CRC_CFLAGS) $(WASM_DEFLATE_CFLAGS) $(INFLATE_CHUNK_CFLAGS)
 
 .PHONY: wasm
 wasm: dist/zlib-streams-dev.wasm
@@ -279,6 +289,9 @@ run_all_tests: dist/zlib-streams-dev.wasm
 	@node src/wasm/tests/run_roundtrip_cli.js deflate 24000 dist/zlib-streams-dev.wasm
 	@$(MAKE) run_transform_roundtrip
 	@$(MAKE) test_decompressionstream_inflate9
+	# Output buffer filling before any input is consumed, including a deflate64
+	# match longer than the buffer, which no encoder we have can produce
+	@node src/wasm/tests/test_output_buffer_boundary.js dist/zlib-streams-dev.wasm
 	@echo "Completed run_all_tests"
 
 dist/zlib-streams-dev.wasm: $(WASM_SRCS)
@@ -301,7 +314,7 @@ dist/zlib-streams.wasm: $(WASM_SRCS)
 		-s EXPORTED_FUNCTIONS='["_inflate9_new","_inflate9_init","_inflate9_init_raw","_inflate9_process","_inflate9_end","_inflate9_last_consumed","_inflate_new","_inflate_init","_inflate_init_raw","_inflate_init_gzip","_inflate_process","_inflate_end","_inflate_last_consumed","_deflate_new","_deflate_init","_deflate_init_raw","_deflate_init_gzip","_deflate_process","_deflate_end","_deflate_last_consumed","_malloc","_free"]' \
 		-o $@
 	cp src/wasm/api/zlib-streams.js dist/zlib-streams.js
-	@which wasm-opt >/dev/null 2>&1 && { echo "Running wasm-opt -Oz --enable-bulk-memory-opt"; wasm-opt -Oz --enable-bulk-memory-opt -o $@ $@ || true; } || true
+	@test -x $(WASM_OPT) && { echo "Running wasm-opt -Oz --enable-bulk-memory-opt"; $(WASM_OPT) -Oz --enable-bulk-memory-opt -o $@ $@ || true; } || true
 
 # -----------------------------------------------------------------------------
 # Optional generator: build a C++ tool that creates Deflate64 ZIPs using the

package/dist/zlib-streams.js

@@ -1,7 +1,39 @@
-/* eslint-disable no-unused-vars */
-/* global Buffer, process, TransformStream */
+/*
+ Copyright (c) 2025 Gildas Lormeau. All rights reserved.
 
-let wasm, malloc, free, memory;
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ 1. Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+
+ 2. Redistributions in binary form must reproduce the above copyright 
+ notice, this list of conditions and the following disclaimer in 
+ the documentation and/or other materials provided with the distribution.
+
+ 3. The names of the authors may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED ''AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
+ INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL JCRAFT,
+ INC. OR ANY CONTRIBUTORS TO THIS SOFTWARE BE LIABLE FOR ANY DIRECT, INDIRECT,
+ INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+ OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* global TransformStream */
+
+const FORMAT_DEFLATE = "deflate";
+const FORMAT_DEFLATE_RAW = "deflate-raw";
+const FORMAT_DEFLATE64_RAW = "deflate64-raw";
+const FORMAT_GZIP = "gzip";
+
+let wasm, malloc, free, memory, initError;
 
 export function setWasmExports(wasmAPI) {
 	wasm = wasmAPI;
@@ -12,53 +44,74 @@ export function setWasmExports(wasmAPI) {
 	}
 }
 
+export function setInitError(error) {
+	initError = error;
+}
+
+export function resetWasmExports() {
+	wasm = malloc = free = memory = initError = null;
+}
+
 function _make(isCompress, type, options = {}) {
+	if (!wasm) {
+		const error = new Error("WASM module not loaded");
+		error.cause = initError;
+		throw error;
+	}
 	const level = (typeof options.level === "number") ? options.level : -1;
 	const outBufferSize = (typeof options.outBuffer === "number") ? options.outBuffer : 64 * 1024;
 	const inBufferSize = (typeof options.inBufferSize === "number") ? options.inBufferSize : 64 * 1024;
 
 	return new TransformStream({
 		start() {
-			let result;
-			this.out = malloc(outBufferSize);
-			this.in = malloc(inBufferSize);
-			this.inBufferSize = inBufferSize;
-			this._scratch = new Uint8Array(outBufferSize);
-			if (isCompress) {
-				this._process = wasm.deflate_process;
-				this._last_consumed = wasm.deflate_last_consumed;
-				this._end = wasm.deflate_end;
-				this.streamHandle = wasm.deflate_new();
-				if (type === "gzip") {
-					result = wasm.deflate_init_gzip(this.streamHandle, level);
-				} else if (type === "deflate-raw") {
-					result = wasm.deflate_init_raw(this.streamHandle, level);
-				} else {
-					result = wasm.deflate_init(this.streamHandle, level);
+			try {
+				let result;
+				this.out = malloc(outBufferSize);
+				this.in = malloc(inBufferSize);
+				this.inBufferSize = inBufferSize;
+				if (!this.out || !this.in) {
+					throw new Error("allocation failed");
 				}
-			} else {
-				if (type === "deflate64-raw") {
-					this._process = wasm.inflate9_process;
-					this._last_consumed = wasm.inflate9_last_consumed;
-					this._end = wasm.inflate9_end;
-					this.streamHandle = wasm.inflate9_new();
-					result = wasm.inflate9_init_raw(this.streamHandle);
+				this._scratch = new Uint8Array(outBufferSize);
+				if (isCompress) {
+					this._process = wasm.deflate_process;
+					this._last_consumed = wasm.deflate_last_consumed;
+					this._end = wasm.deflate_end;
+					this.streamHandle = wasm.deflate_new();
+					if (type === FORMAT_GZIP) {
+						result = wasm.deflate_init_gzip(this.streamHandle, level);
+					} else if (type === FORMAT_DEFLATE_RAW) {
+						result = wasm.deflate_init_raw(this.streamHandle, level);
+					} else {
+						result = wasm.deflate_init(this.streamHandle, level);
+					}
 				} else {
-					this._process = wasm.inflate_process;
-					this._last_consumed = wasm.inflate_last_consumed;
-					this._end = wasm.inflate_end;
-					this.streamHandle = wasm.inflate_new();
-					if (type === "deflate-raw") {
-						result = wasm.inflate_init_raw(this.streamHandle);
-					} else if (type === "gzip") {
-						result = wasm.inflate_init_gzip(this.streamHandle);
+					if (type === FORMAT_DEFLATE64_RAW) {
+						this._process = wasm.inflate9_process;
+						this._last_consumed = wasm.inflate9_last_consumed;
+						this._end = wasm.inflate9_end;
+						this.streamHandle = wasm.inflate9_new();
+						result = wasm.inflate9_init_raw(this.streamHandle);
 					} else {
-						result = wasm.inflate_init(this.streamHandle);
+						this._process = wasm.inflate_process;
+						this._last_consumed = wasm.inflate_last_consumed;
+						this._end = wasm.inflate_end;
+						this.streamHandle = wasm.inflate_new();
+						if (type === FORMAT_DEFLATE_RAW) {
+							result = wasm.inflate_init_raw(this.streamHandle);
+						} else if (type === FORMAT_GZIP) {
+							result = wasm.inflate_init_gzip(this.streamHandle);
+						} else {
+							result = wasm.inflate_init(this.streamHandle);
+						}
 					}
 				}
-			}
-			if (result !== 0) {
-				throw new Error("init failed:" + result);
+				if (result !== 0) {
+					throw new Error("init failed:" + result);
+				}
+			} catch (error) {
+				disposeStream(this);
+				throw error;
 			}
 		},
 		transform(chunk, controller) {
@@ -75,9 +128,13 @@ function _make(isCompress, type, options = {}) {
 					if (!this.in || this.inBufferSize < toRead) {
 						if (this.in && free) {
 							free(this.in);
+							this.in = 0;
 						}
 						this.in = malloc(toRead);
 						this.inBufferSize = toRead;
+						if (!this.in) {
+							throw new Error("allocation failed");
+						}
 					}
 					heap.set(buffer.subarray(offset, offset + toRead), this.in);
 					const result = process(this.streamHandle, this.in, toRead, out, outBufferSize, 0);
@@ -94,21 +151,13 @@ function _make(isCompress, type, options = {}) {
 						}
 					}
 					const consumed = last_consumed(this.streamHandle);
-					if (consumed === 0) {
+					if (consumed === 0 && prod === 0) {
 						break;
 					}
 					offset += consumed;
 				}
 			} catch (error) {
-				if (this._end && this.streamHandle) {
-					this._end(this.streamHandle);
-				}
-				if (this.in && free) {
-					free(this.in);
-				}
-				if (this.out && free) {
-					free(this.out);
-				}
+				disposeStream(this);
 				controller.error(error);
 			}
 		},
@@ -139,30 +188,54 @@ function _make(isCompress, type, options = {}) {
 			} catch (error) {
 				controller.error(error);
 			} finally {
-				if (this._end && this.streamHandle) {
-					const result = this._end(this.streamHandle);
-					if (result !== 0) {
-						controller.error(new Error("end error:" + result));
-					}
-				}
-				if (this.in && free) {
-					free(this.in);
-				}
-				if (this.out && free) {
-					free(this.out);
+				const result = disposeStream(this);
+				if (result !== 0) {
+					controller.error(new Error("end error:" + result));
 				}
 			}
+		},
+		cancel() {
+			// release the stream handle and buffers when the pipeline is aborted,
+			// they would be leaked in the process-lifetime wasm heap otherwise
+			disposeStream(this);
 		}
 	});
+
+	function disposeStream(state) {
+		let endResult = 0;
+		if (state.streamHandle && state._end) {
+			endResult = state._end(state.streamHandle);
+		}
+		state.streamHandle = 0;
+		if (state.in && free) {
+			free(state.in);
+		}
+		state.in = 0;
+		if (state.out && free) {
+			free(state.out);
+		}
+		state.out = 0;
+		return endResult;
+	}
 }
 
 export class CompressionStreamZlib {
-	constructor(type = "deflate", options) {
+	constructor(type = FORMAT_DEFLATE, options) {
 		return _make(true, type, options);
 	}
 }
 export class DecompressionStreamZlib {
-	constructor(type = "deflate", options) {
+	constructor(type = FORMAT_DEFLATE, options) {
 		return _make(false, type, options);
 	}
-}
+}
+// These codecs are backed by the WASM module; they are unusable until setWasmExports() has run.
+// The worker uses this flag to know it must fall back to the native CompressionStream when the
+// module fails to load, rather than discarding a self-contained codec supplied through config.
+CompressionStreamZlib.requiresModule = true;
+DecompressionStreamZlib.requiresModule = true;
+// Constructing these classes before the module is loaded throws, so capability probes cannot rely
+// on trying the constructor; the formats are declared instead, next to the branches implementing
+// them in _make().
+CompressionStreamZlib.supportedFormats = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP];
+DecompressionStreamZlib.supportedFormats = [FORMAT_DEFLATE, FORMAT_DEFLATE_RAW, FORMAT_GZIP, FORMAT_DEFLATE64_RAW];

package/package.json

@@ -3,7 +3,7 @@
   "description": "WASM-based Compression Streams API implementation using zlib, with support for deflate64 decompression.",
   "author": "Gildas Lormeau",
   "license": "BSD-3-Clause",
-  "version": "1.0.6",
+  "version": "1.1.0",
   "type": "module",
   "keywords": [
     "deflate",
@@ -28,4 +28,4 @@
   "publishConfig": {
     "access": "public"
   }
-}
+}

package/src/README.md

@@ -1 +1,61 @@
-See https://github.com/gildas-lormeau/inflate9.c
+# Sources
+
+`src/zlib` is a submodule of [madler/zlib](https://github.com/madler/zlib) and is never edited. The
+files directly under `src/` are forked copies, compiled instead of their submodule counterparts. The
+deflate64 work is described in <https://github.com/gildas-lormeau/inflate9.c>.
+
+## Forked from madler/zlib
+
+| file | why it is forked |
+|---|---|
+| `inftrees.c`, `inftrees.h` | deflate64 length code 285 and distance codes 30 and 31 |
+| `trees.c` | builds the static deflate trees at run time instead of embedding them |
+
+## Forked from Chromium's zlib
+
+`deflate.c`, `deflate.h`, `compare256.h`, `insert_string.h`, `cpu_features.h`, `inflate.c`,
+`inffast.h`, `inffast_chunk.c`, `inffast_chunk.h` and `chunkcopy.h` come from
+<https://chromium.googlesource.com/chromium/src/third_party/zlib>, revision
+`285e94b8fa95ad3b7d16b80798ec8dce6febb8c8`. They are BSD-licensed, see the headers in the files.
+`inflate.c`, `inffast_chunk.*` and `chunkcopy.h` are their `contrib/optimizations` versions.
+
+`inflate.c` still carries the deflate64 support on top. That merge is small because the two are
+orthogonal: deflate64 never enters `inflate_fast`, so the chunked fast path does not know about it.
+
+### Build options
+
+Deflate, in `WASM_DEFLATE_CFLAGS`:
+
+- `DEFLATE_COMPARE256_64LE` extends matches 8 bytes at a time in `longest_match`. Same match length,
+  computed faster. About 7% at every level.
+- `USE_ZLIB_RABIN_KARP_ROLLING_HASH` turns Chromium's own hash off, so the deflate stream stays
+  bit-for-bit what madler's `deflate.c` produces. Dropping this flag enables the faster ANZAC hash,
+  which is valid DEFLATE but different bytes.
+
+Inflate, in `INFLATE_CHUNK_CFLAGS`, needed by every target that builds `inflate.c`, not only the
+wasm one:
+
+- `INFLATE_CHUNK_GENERIC` selects the portable chunk copies. `chunkcopy.h` has no wasm SIMD path and
+  `#error`s without one of its three arch macros. `-msimd128` was measured and buys nothing.
+- `INFLATE_CHUNK_READ_64LE` buffers the input bits in a `uint64_t` inside `inffast_chunk.c` only.
+  `state->hold` stays `unsigned long`. Together they are worth about 20%.
+
+### Deliberate edits, to re-apply when the Chromium files are updated
+
+1. `deflate.c`, `inflate.c` and `inffast_chunk.c` include `"insert_string.h"`, `"inffast_chunk.h"`
+   and `"chunkcopy.h"` directly, not through `contrib/optimizations/`, because this repo keeps
+   forked zlib sources flat.
+2. `deflate.h` leaves `LIT_MEM` off, as madler does. Chromium turns it on, which costs 16 KB of
+   `pending_buf` per deflate stream for about 1.5% of speed. The wasm heap is fixed at 16.9 MB, so
+   that is 62 concurrent deflate streams instead of 59.
+3. `deflate.h` casts the arguments of `zmemcpy` and `zmemzero`. Chromium's `zutil.h` declares them
+   taking `void *`, madler's takes `Bytef *`, and this build keeps madler's.
+4. `inflate.c` keeps the 9-bit and 6-bit root tables instead of Chromium's 10 and 9. Theirs need
+   `ENOUGH_LENS` raised from 852 to 1332, which is 1920 more bytes per inflate state and a fifth of
+   the concurrent inflate streams, for about 1% of speed. Without that change to `inftrees.h` they
+   fail every dynamic block with "invalid literal/lengths set".
+5. `chunkcopy.h` includes `<stddef.h>` for `ptrdiff_t`, which `-DZ_SOLO` keeps `zutil.h` from
+   pulling in.
+
+`deflate_crc32.c` supplies `crc_reset`, `crc_finalize` and `copy_with_crc`. Chromium's `crc32.c` owns
+them and `deflate.c` calls them on the gzip path, but this build keeps madler's `crc32.c`.