webpack 5.39.1 → 5.42.0

package/lib/serialization/FileMiddleware.js

@@ -5,6 +5,14 @@
 "use strict";
 
 const { constants } = require("buffer");
+const { pipeline } = require("stream");
+const {
+	createBrotliCompress,
+	createBrotliDecompress,
+	createGzip,
+	createGunzip,
+	constants: zConstants
+} = require("zlib");
 const createHash = require("../util/createHash");
 const { dirname, join, mkdirp } = require("../util/fs");
 const memoize = require("../util/memoize");
@@ -37,6 +45,9 @@ const hashForName = buffers => {
 	return /** @type {string} */ (hash.digest("hex"));
 };
 
+const COMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
+const DECOMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
+
 const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
 	? (buf, value, offset) => {
 			buf.writeBigUInt64LE(BigInt(value), offset);
@@ -69,7 +80,7 @@ const readUInt64LE = Buffer.prototype.readBigUInt64LE
  * @param {FileMiddleware} middleware this
  * @param {BufferSerializableType[] | Promise<BufferSerializableType[]>} data data to be serialized
  * @param {string | boolean} name file base name
- * @param {function(string | false, Buffer[]): Promise} writeFile writes a file
+ * @param {function(string | false, Buffer[]): Promise<void>} writeFile writes a file
  * @returns {Promise<SerializeResult>} resulting file pointer and promise
  */
 const serialize = async (middleware, data, name, writeFile) => {
@@ -280,8 +291,16 @@ const deserialize = async (middleware, name, readFile) => {
 	}
 	const sectionCount = readUInt32LE();
 	const lengths = [];
+	let lastLengthPositive = false;
 	for (let i = 0; i < sectionCount; i++) {
-		lengths.push(readInt32LE());
+		const value = readInt32LE();
+		const valuePositive = value >= 0;
+		if (lastLengthPositive && valuePositive) {
+			lengths[lengths.length - 1] += value;
+		} else {
+			lengths.push(value);
+			lastLengthPositive = valuePositive;
+		}
 	}
 	const result = [];
 	for (let length of lengths) {
@@ -307,13 +326,24 @@ const deserialize = async (middleware, name, readFile) => {
 			} else if (contentPosition !== 0) {
 				if (length <= contentItemLength - contentPosition) {
 					result.push(
-						contentItem.slice(contentPosition, contentPosition + length)
+						Buffer.from(
+							contentItem.buffer,
+							contentItem.byteOffset + contentPosition,
+							length
+						)
 					);
 					contentPosition += length;
 					length = 0;
 				} else {
-					result.push(contentItem.slice(contentPosition));
-					length -= contentItemLength - contentPosition;
+					const l = contentItemLength - contentPosition;
+					result.push(
+						Buffer.from(
+							contentItem.buffer,
+							contentItem.byteOffset + contentPosition,
+							l
+						)
+					);
+					length -= l;
 					contentPosition = contentItemLength;
 				}
 			} else {
@@ -322,7 +352,9 @@ const deserialize = async (middleware, name, readFile) => {
 					length -= contentItemLength;
 					contentPosition = contentItemLength;
 				} else {
-					result.push(contentItem.slice(0, length));
+					result.push(
+						Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
+					);
 					contentPosition += length;
 					length = 0;
 				}
@@ -334,7 +366,9 @@ const deserialize = async (middleware, name, readFile) => {
 					length -= contentItemLength;
 					contentPosition = contentItemLength;
 				} else {
-					result.push(contentItem.slice(0, length));
+					result.push(
+						Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
+					);
 					contentPosition += length;
 					length = 0;
 				}
@@ -376,11 +410,37 @@ class FileMiddleware extends SerializerMiddleware {
 						? join(this.fs, filename, `../${name}${extension}`)
 						: filename;
 					await new Promise((resolve, reject) => {
-						const stream = this.fs.createWriteStream(file + "_");
+						let stream = this.fs.createWriteStream(file + "_");
+						let compression;
+						if (file.endsWith(".gz")) {
+							compression = createGzip({
+								chunkSize: COMPRESSION_CHUNK_SIZE,
+								level: zConstants.Z_BEST_SPEED
+							});
+						} else if (file.endsWith(".br")) {
+							compression = createBrotliCompress({
+								chunkSize: COMPRESSION_CHUNK_SIZE,
+								params: {
+									[zConstants.BROTLI_PARAM_MODE]: zConstants.BROTLI_MODE_TEXT,
+									[zConstants.BROTLI_PARAM_QUALITY]: 2,
+									[zConstants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: true,
+									[zConstants.BROTLI_PARAM_SIZE_HINT]: content.reduce(
+										(size, b) => size + b.length,
+										0
+									)
+								}
+							});
+						}
+						if (compression) {
+							pipeline(compression, stream, reject);
+							stream = compression;
+							stream.on("finish", () => resolve());
+						} else {
+							stream.on("error", err => reject(err));
+							stream.on("finish", () => resolve());
+						}
 						for (const b of content) stream.write(b);
 						stream.end();
-						stream.on("error", err => reject(err));
-						stream.on("finish", () => resolve());
 					});
 					if (name) allWrittenFiles.add(file);
 				};
@@ -447,6 +507,34 @@ class FileMiddleware extends SerializerMiddleware {
 					let currentBuffer;
 					let currentBufferUsed;
 					const buf = [];
+					let decompression;
+					if (file.endsWith(".gz")) {
+						decompression = createGunzip({
+							chunkSize: DECOMPRESSION_CHUNK_SIZE
+						});
+					} else if (file.endsWith(".br")) {
+						decompression = createBrotliDecompress({
+							chunkSize: DECOMPRESSION_CHUNK_SIZE
+						});
+					}
+					if (decompression) {
+						let newResolve, newReject;
+						resolve(
+							Promise.all([
+								new Promise((rs, rj) => {
+									newResolve = rs;
+									newReject = rj;
+								}),
+								new Promise((resolve, reject) => {
+									decompression.on("data", chunk => buf.push(chunk));
+									decompression.on("end", () => resolve());
+									decompression.on("error", err => reject(err));
+								})
+							]).then(() => buf)
+						);
+						resolve = newResolve;
+						reject = newReject;
+					}
 					this.fs.open(file, "r", (err, fd) => {
 						if (err) {
 							reject(err);
@@ -455,13 +543,18 @@ class FileMiddleware extends SerializerMiddleware {
 						const read = () => {
 							if (currentBuffer === undefined) {
 								currentBuffer = Buffer.allocUnsafeSlow(
-									Math.min(constants.MAX_LENGTH, remaining)
+									Math.min(
+										constants.MAX_LENGTH,
+										remaining,
+										decompression ? DECOMPRESSION_CHUNK_SIZE : Infinity
+									)
 								);
 								currentBufferUsed = 0;
 							}
 							let readBuffer = currentBuffer;
 							let readOffset = currentBufferUsed;
 							let readLength = currentBuffer.length - currentBufferUsed;
+							// values passed to fs.read must be valid int32 values
 							if (readOffset > 0x7fffffff) {
 								readBuffer = currentBuffer.slice(readOffset);
 								readOffset = 0;
@@ -485,9 +578,16 @@ class FileMiddleware extends SerializerMiddleware {
 									currentBufferUsed += bytesRead;
 									remaining -= bytesRead;
 									if (currentBufferUsed === currentBuffer.length) {
-										buf.push(currentBuffer);
+										if (decompression) {
+											decompression.write(currentBuffer);
+										} else {
+											buf.push(currentBuffer);
+										}
 										currentBuffer = undefined;
 										if (remaining === 0) {
+											if (decompression) {
+												decompression.end();
+											}
 											this.fs.close(fd, err => {
 												if (err) {
 													reject(err);

package/lib/serialization/PlainObjectSerializer.js

@@ -7,19 +7,21 @@
 const cache = new WeakMap();
 
 class ObjectStructure {
-	constructor(keys) {
-		this.keys = keys;
-		this.children = new Map();
+	constructor() {
+		this.keys = undefined;
+		this.children = undefined;
 	}
 
-	getKeys() {
+	getKeys(keys) {
+		if (this.keys === undefined) this.keys = keys;
 		return this.keys;
 	}
 
 	key(key) {
+		if (this.children === undefined) this.children = new Map();
 		const child = this.children.get(key);
 		if (child !== undefined) return child;
-		const newChild = new ObjectStructure(this.keys.concat(key));
+		const newChild = new ObjectStructure();
 		this.children.set(key, newChild);
 		return newChild;
 	}
@@ -28,20 +30,27 @@ class ObjectStructure {
 const getCachedKeys = (keys, cacheAssoc) => {
 	let root = cache.get(cacheAssoc);
 	if (root === undefined) {
-		root = new ObjectStructure([]);
+		root = new ObjectStructure();
 		cache.set(cacheAssoc, root);
 	}
 	let current = root;
 	for (const key of keys) {
 		current = current.key(key);
 	}
-	return current.getKeys();
+	return current.getKeys(keys);
 };
 
 class PlainObjectSerializer {
 	serialize(obj, { write }) {
 		const keys = Object.keys(obj);
-		if (keys.length > 1) {
+		if (keys.length > 128) {
+			// Objects with so many keys are unlikely to share structure
+			// with other objects
+			write(keys);
+			for (const key of keys) {
+				write(obj[key]);
+			}
+		} else if (keys.length > 1) {
 			write(getCachedKeys(keys, write));
 			for (const key of keys) {
 				write(obj[key]);

package/lib/serialization/Serializer.js

@@ -15,7 +15,7 @@ class Serializer {
 		const ctx = { ...context, ...this.context };
 		let current = obj;
 		for (const middleware of this.serializeMiddlewares) {
-			if (current instanceof Promise) {
+			if (current && typeof current.then === "function") {
 				current = current.then(
 					data => data && middleware.serialize(data, context)
 				);
@@ -35,7 +35,7 @@ class Serializer {
 		/** @type {any} */
 		let current = value;
 		for (const middleware of this.deserializeMiddlewares) {
-			if (current instanceof Promise) {
+			if (current && typeof current.then === "function") {
 				current = current.then(data => middleware.deserialize(data, context));
 			} else {
 				current = middleware.deserialize(current, ctx);

package/lib/serialization/SerializerMiddleware.js

@@ -100,9 +100,10 @@ class SerializerMiddleware {
 	static serializeLazy(lazy, serialize) {
 		const fn = memoize(() => {
 			const r = lazy();
-			if (r instanceof Promise) return r.then(data => data && serialize(data));
-			if (r) return serialize(r);
-			return null;
+			if (r && typeof r.then === "function") {
+				return r.then(data => data && serialize(data));
+			}
+			return serialize(r);
 		});
 		fn[LAZY_TARGET] = lazy[LAZY_TARGET];
 		/** @type {any} */ (fn).options = /** @type {any} */ (lazy).options;
@@ -118,7 +119,9 @@ class SerializerMiddleware {
 	static deserializeLazy(lazy, deserialize) {
 		const fn = memoize(() => {
 			const r = lazy();
-			if (r instanceof Promise) return r.then(data => deserialize(data));
+			if (r && typeof r.then === "function") {
+				return r.then(data => deserialize(data));
+			}
 			return deserialize(r);
 		});
 		fn[LAZY_TARGET] = lazy[LAZY_TARGET];

package/lib/util/LazySet.js

@@ -24,21 +24,17 @@ const merge = (targetSet, toMerge) => {
 /**
  * @template T
  * @param {Set<Iterable<T>>} targetSet set where iterables should be added
- * @param {Array<Iterable<T> | LazySet<T>>} toDeepMerge iterables or lazy set to be flattened
+ * @param {Array<LazySet<T>>} toDeepMerge lazy sets to be flattened
  * @returns {void}
  */
 const flatten = (targetSet, toDeepMerge) => {
 	for (const set of toDeepMerge) {
-		if (set instanceof LazySet) {
-			if (set._set.size > 0) targetSet.add(set._set);
-			if (set._needMerge) {
-				for (const mergedSet of set._toMerge) {
-					targetSet.add(mergedSet);
-				}
-				flatten(targetSet, set._toDeepMerge);
+		if (set._set.size > 0) targetSet.add(set._set);
+		if (set._needMerge) {
+			for (const mergedSet of set._toMerge) {
+				targetSet.add(mergedSet);
 			}
-		} else {
-			targetSet.add(set);
+			flatten(targetSet, set._toDeepMerge);
 		}
 	}
 };
@@ -58,7 +54,7 @@ class LazySet {
 		this._set = new Set(iterable);
 		/** @type {Set<Iterable<T>>} */
 		this._toMerge = new Set();
-		/** @type {Array<Iterable<T> | LazySet<T>>} */
+		/** @type {Array<LazySet<T>>} */
 		this._toDeepMerge = [];
 		this._needMerge = false;
 		this._deopt = false;
@@ -76,6 +72,14 @@ class LazySet {
 		this._needMerge = false;
 	}
 
+	_isEmpty() {
+		return (
+			this._set.size === 0 &&
+			this._toMerge.size === 0 &&
+			this._toDeepMerge.length === 0
+		);
+	}
+
 	get size() {
 		if (this._needMerge) this._merge();
 		return this._set.size;
@@ -101,13 +105,18 @@ class LazySet {
 				_set.add(item);
 			}
 		} else {
-			this._toDeepMerge.push(iterable);
-			this._needMerge = true;
-			// Avoid being too memory hungry
-			if (this._toDeepMerge.length > 100000) {
-				this._flatten();
-				if (this._toMerge.size > 100000) this._merge();
+			if (iterable instanceof LazySet) {
+				if (iterable._isEmpty()) return this;
+				this._toDeepMerge.push(iterable);
+				this._needMerge = true;
+				if (this._toDeepMerge.length > 100000) {
+					this._flatten();
+				}
+			} else {
+				this._toMerge.add(iterable);
+				this._needMerge = true;
 			}
+			if (this._toMerge.size > 100000) this._merge();
 		}
 		return this;
 	}

package/lib/wasm/EnableWasmLoadingPlugin.js

@@ -84,11 +84,20 @@ class EnableWasmLoadingPlugin {
 				case "async-node": {
 					// TODO webpack 6 remove ReadFileCompileWasmPlugin
 					const ReadFileCompileWasmPlugin = require("../node/ReadFileCompileWasmPlugin");
+					// @ts-expect-error typescript bug for duplicate require
 					const ReadFileCompileAsyncWasmPlugin = require("../node/ReadFileCompileAsyncWasmPlugin");
 					new ReadFileCompileWasmPlugin({
 						mangleImports: compiler.options.optimization.mangleWasmImports
 					}).apply(compiler);
-					new ReadFileCompileAsyncWasmPlugin().apply(compiler);
+					new ReadFileCompileAsyncWasmPlugin({ type }).apply(compiler);
+					break;
+				}
+				case "async-node-module": {
+					// @ts-expect-error typescript bug for duplicate require
+					const ReadFileCompileAsyncWasmPlugin = require("../node/ReadFileCompileAsyncWasmPlugin");
+					new ReadFileCompileAsyncWasmPlugin({ type, import: true }).apply(
+						compiler
+					);
 					break;
 				}
 				case "universal":

package/lib/wasm-sync/WasmChunkLoadingRuntimeModule.js

@@ -291,7 +291,7 @@ class WasmChunkLoadingRuntimeModule extends RuntimeModule {
 						"var promise;",
 						this.supportsStreaming
 							? Template.asString([
-									"if(importObject instanceof Promise && typeof WebAssembly.compileStreaming === 'function') {",
+									"if(importObject && typeof importObject.then === 'function' && typeof WebAssembly.compileStreaming === 'function') {",
 									Template.indent([
 										"promise = Promise.all([WebAssembly.compileStreaming(req), importObject]).then(function(items) {",
 										Template.indent([
@@ -309,7 +309,7 @@ class WasmChunkLoadingRuntimeModule extends RuntimeModule {
 									])
 							  ])
 							: Template.asString([
-									"if(importObject instanceof Promise) {",
+									"if(importObject && typeof importObject.then === 'function') {",
 									Template.indent([
 										"var bytesPromise = req.then(function(x) { return x.arrayBuffer(); });",
 										"promise = Promise.all([",

package/lib/wasm-sync/WebAssemblyModulesPlugin.js

@@ -16,7 +16,7 @@ const WebAssemblyInInitialChunkError = require("./WebAssemblyInInitialChunkError
 /** @typedef {import("../Compiler")} Compiler */
 /** @typedef {import("../Module")} Module */
 /** @typedef {import("../ModuleTemplate")} ModuleTemplate */
-/** @typedef {import("../ModuleTemplate").RenderContext} RenderContext */
+/** @typedef {import("../javascript/JavascriptModulesPlugin").RenderContext} RenderContext */
 
 const getWebAssemblyGenerator = memoize(() =>
 	require("./WebAssemblyGenerator")

package/package.json

@@ -1,20 +1,20 @@
 {
   "name": "webpack",
-  "version": "5.39.1",
+  "version": "5.42.0",
   "author": "Tobias Koppers @sokra",
   "description": "Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.",
   "license": "MIT",
   "dependencies": {
     "@types/eslint-scope": "^3.7.0",
-    "@types/estree": "^0.0.47",
+    "@types/estree": "^0.0.48",
     "@webassemblyjs/ast": "1.11.0",
     "@webassemblyjs/wasm-edit": "1.11.0",
     "@webassemblyjs/wasm-parser": "1.11.0",
-    "acorn": "^8.2.1",
+    "acorn": "^8.4.1",
     "browserslist": "^4.14.5",
     "chrome-trace-event": "^1.0.2",
     "enhanced-resolve": "^5.8.0",
-    "es-module-lexer": "^0.4.0",
+    "es-module-lexer": "^0.6.0",
     "eslint-scope": "5.1.1",
     "events": "^3.2.0",
     "glob-to-regexp": "^0.4.1",
@@ -25,7 +25,7 @@
     "neo-async": "^2.6.2",
     "schema-utils": "^3.0.0",
     "tapable": "^2.1.1",
-    "terser-webpack-plugin": "^5.1.1",
+    "terser-webpack-plugin": "^5.1.3",
     "watchpack": "^2.2.0",
     "webpack-sources": "^2.3.0"
   },
@@ -37,7 +37,7 @@
   "devDependencies": {
     "@babel/core": "^7.11.1",
     "@babel/preset-react": "^7.10.4",
-    "@types/es-module-lexer": "^0.3.0",
+    "@types/es-module-lexer": "^0.4.1",
     "@types/jest": "^26.0.15",
     "@types/node": "^15.0.1",
     "babel-loader": "^8.1.0",
@@ -64,7 +64,7 @@
     "is-ci": "^3.0.0",
     "istanbul": "^0.4.5",
     "jest": "^26.6.3",
-    "jest-diff": "^26.6.2",
+    "jest-diff": "^27.0.2",
     "jest-junit": "^12.0.0",
     "json-loader": "^0.5.7",
     "json5": "^2.1.3",
@@ -80,7 +80,7 @@
     "nyc": "^15.1.0",
     "open-cli": "^6.0.1",
     "prettier": "^2.2.0",
-    "pretty-format": "^26.3.0",
+    "pretty-format": "^27.0.2",
     "pug": "^3.0.0",
     "pug-loader": "^2.4.0",
     "raw-loader": "^4.0.1",
@@ -91,7 +91,7 @@
     "simple-git": "^2.17.0",
     "strip-ansi": "^6.0.0",
     "style-loader": "^2.0.0",
-    "terser": "^5.5.0",
+    "terser": "^5.7.0",
     "toml": "^3.0.0",
     "tooling": "webpack/tooling#v1.19.0",
     "ts-loader": "^8.0.2",
@@ -131,11 +131,11 @@
   ],
   "scripts": {
     "setup": "node ./setup/setup.js",
-    "test": "node --max-old-space-size=4096 --trace-deprecation node_modules/jest-cli/bin/jest",
+    "test": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest",
     "test:update-snapshots": "yarn jest -u",
-    "test:integration": "node --max-old-space-size=4096 --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.test.js\"",
-    "test:basic": "node --max-old-space-size=4096 --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/te{st/TestCasesNormal,st/StatsTestCases,st/ConfigTestCases}.test.js\"",
-    "test:unit": "node --max-old-space-size=4096 --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.unittest.js\"",
+    "test:integration": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.test.js\"",
+    "test:basic": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/te{st/TestCasesNormal,st/StatsTestCases,st/ConfigTestCases}.test.js\"",
+    "test:unit": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.unittest.js\"",
     "travis:integration": "yarn cover:integration --ci $JEST",
     "travis:basic": "yarn cover:basic --ci $JEST",
     "travis:lintunit": "yarn lint && yarn cover:unit --ci $JEST",
@@ -162,13 +162,13 @@
     "pretty-lint": "yarn pretty-lint-base --check",
     "yarn-lint": "yarn-deduplicate --fail --list -s highest yarn.lock",
     "yarn-lint-fix": "yarn-deduplicate -s highest yarn.lock",
-    "benchmark": "node --max-old-space-size=4096 --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.benchmark.js\" --runInBand",
+    "benchmark": "node --max-old-space-size=4096 --experimental-vm-modules --trace-deprecation node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.benchmark.js\" --runInBand",
     "cover": "yarn cover:all && yarn cover:report",
     "cover:clean": "rimraf .nyc_output coverage",
-    "cover:all": "node --max-old-space-size=4096 node_modules/jest-cli/bin/jest --coverage",
-    "cover:basic": "node --max-old-space-size=4096 node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/te{st/TestCasesNormal,st/StatsTestCases,st/ConfigTestCases}.test.js\" --coverage",
-    "cover:integration": "node --max-old-space-size=4096 node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.test.js\" --coverage",
-    "cover:unit": "node --max-old-space-size=4096 node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.unittest.js\" --coverage",
+    "cover:all": "node --max-old-space-size=4096 --experimental-vm-modules node_modules/jest-cli/bin/jest --coverage",
+    "cover:basic": "node --max-old-space-size=4096 --experimental-vm-modules node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/te{st/TestCasesNormal,st/StatsTestCases,st/ConfigTestCases}.test.js\" --coverage",
+    "cover:integration": "node --max-old-space-size=4096 --experimental-vm-modules node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.test.js\" --coverage",
+    "cover:unit": "node --max-old-space-size=4096 --experimental-vm-modules node_modules/jest-cli/bin/jest --testMatch \"<rootDir>/test/*.unittest.js\" --coverage",
     "cover:types": "node node_modules/tooling/type-coverage",
     "cover:merge": "nyc merge .nyc_output coverage/coverage-nyc.json && rimraf .nyc_output",
     "cover:report": "nyc report -t coverage"