webpack 5.41.1 → 5.42.0

package/lib/Compiler.js

@@ -260,6 +260,8 @@ class Compiler {
 
 		/** @type {Compilation} */
 		this._lastCompilation = undefined;
+		/** @type {NormalModuleFactory} */
+		this._lastNormalModuleFactory = undefined;
 
 		/** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
 		this._assetEmittingSourceCache = new WeakMap();
@@ -375,6 +377,14 @@ class Compiler {
 		}
 	}
 
+	// TODO webpack 6: solve this in a better way
+	_cleanupLastNormalModuleFactory() {
+		if (this._lastNormalModuleFactory !== undefined) {
+			this._lastNormalModuleFactory.cleanupForCache();
+			this._lastNormalModuleFactory = undefined;
+		}
+	}
+
 	/**
 	 * @param {WatchOptions} watchOptions the watcher's options
 	 * @param {Callback<Stats>} handler signals when the call finishes
@@ -1036,6 +1046,7 @@ ${other}`);
 	}
 
 	createNormalModuleFactory() {
+		this._cleanupLastNormalModuleFactory();
 		const normalModuleFactory = new NormalModuleFactory({
 			context: this.options.context,
 			fs: this.inputFileSystem,
@@ -1044,6 +1055,7 @@ ${other}`);
 			associatedObjectForCache: this.root,
 			layers: this.options.experiments.layers
 		});
+		this._lastNormalModuleFactory = normalModuleFactory;
 		this.hooks.normalModuleFactory.call(normalModuleFactory);
 		return normalModuleFactory;
 	}
@@ -1122,8 +1134,9 @@ ${other}`);
 			if (err) return callback(err);
 			// Get rid of reference to last compilation to avoid leaking memory
 			// We can't run this._cleanupLastCompilation() as the Stats to this compilation
-			// might be still in use. We try to get rid for the reference to the cache instead.
+			// might be still in use. We try to get rid of the reference to the cache instead.
 			this._lastCompilation = undefined;
+			this._lastNormalModuleFactory = undefined;
 			this.cache.shutdown(callback);
 		});
 	}

package/lib/ExternalModule.js

@@ -429,6 +429,11 @@ class ExternalModule extends Module {
 				this.buildInfo.strict = false;
 				break;
 			case "system":
+				if (!Array.isArray(request) || request.length === 1) {
+					this.buildMeta.exportsType = "namespace";
+					canMangle = true;
+				}
+				break;
 			case "module":
 				if (this.buildInfo.module) {
 					if (!Array.isArray(request) || request.length === 1) {

package/lib/NormalModuleFactory.js

@@ -13,8 +13,10 @@ const {
 	SyncHook,
 	HookMap
 } = require("tapable");
+const ChunkGraph = require("./ChunkGraph");
 const Module = require("./Module");
 const ModuleFactory = require("./ModuleFactory");
+const ModuleGraph = require("./ModuleGraph");
 const NormalModule = require("./NormalModule");
 const BasicEffectRulePlugin = require("./rules/BasicEffectRulePlugin");
 const BasicMatcherRulePlugin = require("./rules/BasicMatcherRulePlugin");
@@ -257,8 +259,8 @@ class NormalModuleFactory extends ModuleFactory {
 		this.parserCache = new Map();
 		/** @type {Map<string, WeakMap<Object, Generator>>} */
 		this.generatorCache = new Map();
-		/** @type {WeakSet<Module>} */
-		this._restoredUnsafeCacheEntries = new WeakSet();
+		/** @type {Set<Module>} */
+		this._restoredUnsafeCacheEntries = new Set();
 
 		const cacheParseResource = parseResource.bindCache(
 			associatedObjectForCache
@@ -661,6 +663,14 @@ class NormalModuleFactory extends ModuleFactory {
 		);
 	}
 
+	cleanupForCache() {
+		for (const module of this._restoredUnsafeCacheEntries) {
+			ChunkGraph.clearChunkGraphForModule(module);
+			ModuleGraph.clearModuleGraphForModule(module);
+			module.cleanupForCache();
+		}
+	}
+
 	/**
 	 * @param {ModuleFactoryCreateData} data data object
 	 * @param {function(Error=, ModuleFactoryResult=): void} callback callback

package/lib/WebpackOptionsApply.js

@@ -572,7 +572,8 @@ class WebpackOptionsApply extends OptionsApply {
 									snapshot: options.snapshot,
 									maxAge: cacheOptions.maxAge,
 									profile: cacheOptions.profile,
-									allowCollectingMemory: cacheOptions.allowCollectingMemory
+									allowCollectingMemory: cacheOptions.allowCollectingMemory,
+									compression: cacheOptions.compression
 								}),
 								cacheOptions.idleTimeout,
 								cacheOptions.idleTimeoutForInitialStore,

package/lib/cache/PackFileCacheStrategy.js

@@ -848,6 +848,7 @@ class PackFileCacheStrategy {
 	 * @param {number} options.maxAge max age of cache items
 	 * @param {boolean} options.profile track and log detailed timing information for individual cache items
 	 * @param {boolean} options.allowCollectingMemory allow to collect unused memory created during deserialization
+	 * @param {false | "gzip" | "brotli"} options.compression compression used
 	 */
 	constructor({
 		compiler,
@@ -859,7 +860,8 @@ class PackFileCacheStrategy {
 		snapshot,
 		maxAge,
 		profile,
-		allowCollectingMemory
+		allowCollectingMemory,
+		compression
 	}) {
 		this.fileSerializer = createFileSerializer(fs);
 		this.fileSystemInfo = new FileSystemInfo(fs, {
@@ -875,6 +877,13 @@ class PackFileCacheStrategy {
 		this.maxAge = maxAge;
 		this.profile = profile;
 		this.allowCollectingMemory = allowCollectingMemory;
+		this.compression = compression;
+		this._extension =
+			compression === "brotli"
+				? ".pack.br"
+				: compression === "gzip"
+				? ".pack.gz"
+				: ".pack";
 		this.snapshot = snapshot;
 		/** @type {Set<string>} */
 		this.buildDependencies = new Set();
@@ -916,8 +925,8 @@ class PackFileCacheStrategy {
 		logger.time("restore cache container");
 		return this.fileSerializer
 			.deserialize(null, {
-				filename: `${cacheLocation}/index.pack`,
-				extension: ".pack",
+				filename: `${cacheLocation}/index${this._extension}`,
+				extension: `${this._extension}`,
 				logger,
 				profile,
 				retainedBuffer: this.allowCollectingMemory
@@ -927,11 +936,13 @@ class PackFileCacheStrategy {
 			.catch(err => {
 				if (err.code !== "ENOENT") {
 					logger.warn(
-						`Restoring pack failed from ${cacheLocation}.pack: ${err}`
+						`Restoring pack failed from ${cacheLocation}${this._extension}: ${err}`
 					);
 					logger.debug(err.stack);
 				} else {
-					logger.debug(`No pack exists at ${cacheLocation}.pack: ${err}`);
+					logger.debug(
+						`No pack exists at ${cacheLocation}${this._extension}: ${err}`
+					);
 				}
 				return undefined;
 			})
@@ -940,14 +951,14 @@ class PackFileCacheStrategy {
 				if (!packContainer) return undefined;
 				if (!(packContainer instanceof PackContainer)) {
 					logger.warn(
-						`Restored pack from ${cacheLocation}.pack, but contained content is unexpected.`,
+						`Restored pack from ${cacheLocation}${this._extension}, but contained content is unexpected.`,
 						packContainer
 					);
 					return undefined;
 				}
 				if (packContainer.version !== version) {
 					logger.log(
-						`Restored pack from ${cacheLocation}.pack, but version doesn't match.`
+						`Restored pack from ${cacheLocation}${this._extension}, but version doesn't match.`
 					);
 					return undefined;
 				}
@@ -959,14 +970,14 @@ class PackFileCacheStrategy {
 							(err, valid) => {
 								if (err) {
 									logger.log(
-										`Restored pack from ${cacheLocation}.pack, but checking snapshot of build dependencies errored: ${err}.`
+										`Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of build dependencies errored: ${err}.`
 									);
 									logger.debug(err.stack);
 									return resolve(false);
 								}
 								if (!valid) {
 									logger.log(
-										`Restored pack from ${cacheLocation}.pack, but build dependencies have changed.`
+										`Restored pack from ${cacheLocation}${this._extension}, but build dependencies have changed.`
 									);
 									return resolve(false);
 								}
@@ -981,7 +992,7 @@ class PackFileCacheStrategy {
 							(err, valid) => {
 								if (err) {
 									logger.log(
-										`Restored pack from ${cacheLocation}.pack, but checking snapshot of resolving of build dependencies errored: ${err}.`
+										`Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of resolving of build dependencies errored: ${err}.`
 									);
 									logger.debug(err.stack);
 									return resolve(false);
@@ -1001,7 +1012,7 @@ class PackFileCacheStrategy {
 									(err, valid) => {
 										if (err) {
 											logger.log(
-												`Restored pack from ${cacheLocation}.pack, but resolving of build dependencies errored: ${err}.`
+												`Restored pack from ${cacheLocation}${this._extension}, but resolving of build dependencies errored: ${err}.`
 											);
 											logger.debug(err.stack);
 											return resolve(false);
@@ -1012,7 +1023,7 @@ class PackFileCacheStrategy {
 											return resolve(true);
 										}
 										logger.log(
-											`Restored pack from ${cacheLocation}.pack, but build dependencies resolve to different locations.`
+											`Restored pack from ${cacheLocation}${this._extension}, but build dependencies resolve to different locations.`
 										);
 										return resolve(false);
 									}
@@ -1052,7 +1063,7 @@ class PackFileCacheStrategy {
 			})
 			.catch(err => {
 				this.logger.warn(
-					`Restoring pack from ${cacheLocation}.pack failed: ${err}`
+					`Restoring pack from ${cacheLocation}${this._extension} failed: ${err}`
 				);
 				this.logger.debug(err.stack);
 				return new Pack(logger, this.maxAge);
@@ -1236,8 +1247,8 @@ class PackFileCacheStrategy {
 					);
 					return this.fileSerializer
 						.serialize(content, {
-							filename: `${this.cacheLocation}/index.pack`,
-							extension: ".pack",
+							filename: `${this.cacheLocation}/index${this._extension}`,
+							extension: `${this._extension}`,
 							logger: this.logger,
 							profile: this.profile
 						})

package/lib/config/defaults.js

@@ -298,6 +298,7 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			);
 			D(cache, "hashAlgorithm", "md4");
 			D(cache, "store", "pack");
+			D(cache, "compression", development ? false : "gzip");
 			D(cache, "profile", false);
 			D(cache, "idleTimeout", 60000);
 			D(cache, "idleTimeoutForInitialStore", 5000);

package/lib/config/normalization.js

@@ -137,6 +137,7 @@ const getNormalizedWebpackOptions = config => {
 						cacheDirectory: cache.cacheDirectory,
 						cacheLocation: cache.cacheLocation,
 						hashAlgorithm: cache.hashAlgorithm,
+						compression: cache.compression,
 						idleTimeout: cache.idleTimeout,
 						idleTimeoutForInitialStore: cache.idleTimeoutForInitialStore,
 						idleTimeoutAfterLargeChanges: cache.idleTimeoutAfterLargeChanges,

package/lib/serialization/BinaryMiddleware.js

@@ -548,12 +548,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 		const isInCurrentBuffer = n => {
 			return currentIsBuffer && n + currentPosition <= currentBuffer.length;
 		};
-		/**
-		 * Reads n bytes
-		 * @param {number} n amount of bytes to read
-		 * @returns {Buffer} buffer with bytes
-		 */
-		const read = n => {
+		const ensureBuffer = () => {
 			if (!currentIsBuffer) {
 				throw new Error(
 					currentBuffer === null
@@ -561,9 +556,31 @@ class BinaryMiddleware extends SerializerMiddleware {
 						: "Unexpected lazy element in stream"
 				);
 			}
+		};
+		/**
+		 * Reads n bytes
+		 * @param {number} n amount of bytes to read
+		 * @returns {Buffer} buffer with bytes
+		 */
+		const read = n => {
+			ensureBuffer();
 			const rem = currentBuffer.length - currentPosition;
 			if (rem < n) {
-				return Buffer.concat([read(rem), read(n - rem)]);
+				const buffers = [read(rem)];
+				n -= rem;
+				ensureBuffer();
+				while (currentBuffer.length < n) {
+					const b = /** @type {Buffer} */ (currentBuffer);
+					buffers.push(b);
+					n -= b.length;
+					currentDataItem++;
+					currentBuffer =
+						currentDataItem < data.length ? data[currentDataItem] : null;
+					currentIsBuffer = Buffer.isBuffer(currentBuffer);
+					ensureBuffer();
+				}
+				buffers.push(read(n));
+				return Buffer.concat(buffers);
 			}
 			const b = /** @type {Buffer} */ (currentBuffer);
 			const res = Buffer.from(b.buffer, b.byteOffset + currentPosition, n);
@@ -577,13 +594,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 		 * @returns {Buffer} buffer with bytes
 		 */
 		const readUpTo = n => {
-			if (!currentIsBuffer) {
-				throw new Error(
-					currentBuffer === null
-						? "Unexpected end of stream"
-						: "Unexpected lazy element in stream"
-				);
-			}
+			ensureBuffer();
 			const rem = currentBuffer.length - currentPosition;
 			if (rem < n) {
 				n = rem;
@@ -595,13 +606,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 			return res;
 		};
 		const readU8 = () => {
-			if (!currentIsBuffer) {
-				throw new Error(
-					currentBuffer === null
-						? "Unexpected end of stream"
-						: "Unexpected lazy element in stream"
-				);
-			}
+			ensureBuffer();
 			/**
 			 * There is no need to check remaining buffer size here
 			 * since {@link checkOverflow} guarantees at least one byte remaining

package/lib/serialization/FileMiddleware.js

@@ -5,6 +5,14 @@
 "use strict";
 
 const { constants } = require("buffer");
+const { pipeline } = require("stream");
+const {
+	createBrotliCompress,
+	createBrotliDecompress,
+	createGzip,
+	createGunzip,
+	constants: zConstants
+} = require("zlib");
 const createHash = require("../util/createHash");
 const { dirname, join, mkdirp } = require("../util/fs");
 const memoize = require("../util/memoize");
@@ -37,6 +45,9 @@ const hashForName = buffers => {
 	return /** @type {string} */ (hash.digest("hex"));
 };
 
+const COMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
+const DECOMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
+
 const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
 	? (buf, value, offset) => {
 			buf.writeBigUInt64LE(BigInt(value), offset);
@@ -69,7 +80,7 @@ const readUInt64LE = Buffer.prototype.readBigUInt64LE
  * @param {FileMiddleware} middleware this
  * @param {BufferSerializableType[] | Promise<BufferSerializableType[]>} data data to be serialized
  * @param {string | boolean} name file base name
- * @param {function(string | false, Buffer[]): Promise} writeFile writes a file
+ * @param {function(string | false, Buffer[]): Promise<void>} writeFile writes a file
  * @returns {Promise<SerializeResult>} resulting file pointer and promise
  */
 const serialize = async (middleware, data, name, writeFile) => {
@@ -399,11 +410,37 @@ class FileMiddleware extends SerializerMiddleware {
 						? join(this.fs, filename, `../${name}${extension}`)
 						: filename;
 					await new Promise((resolve, reject) => {
-						const stream = this.fs.createWriteStream(file + "_");
+						let stream = this.fs.createWriteStream(file + "_");
+						let compression;
+						if (file.endsWith(".gz")) {
+							compression = createGzip({
+								chunkSize: COMPRESSION_CHUNK_SIZE,
+								level: zConstants.Z_BEST_SPEED
+							});
+						} else if (file.endsWith(".br")) {
+							compression = createBrotliCompress({
+								chunkSize: COMPRESSION_CHUNK_SIZE,
+								params: {
+									[zConstants.BROTLI_PARAM_MODE]: zConstants.BROTLI_MODE_TEXT,
+									[zConstants.BROTLI_PARAM_QUALITY]: 2,
+									[zConstants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: true,
+									[zConstants.BROTLI_PARAM_SIZE_HINT]: content.reduce(
+										(size, b) => size + b.length,
+										0
+									)
+								}
+							});
+						}
+						if (compression) {
+							pipeline(compression, stream, reject);
+							stream = compression;
+							stream.on("finish", () => resolve());
+						} else {
+							stream.on("error", err => reject(err));
+							stream.on("finish", () => resolve());
+						}
 						for (const b of content) stream.write(b);
 						stream.end();
-						stream.on("error", err => reject(err));
-						stream.on("finish", () => resolve());
 					});
 					if (name) allWrittenFiles.add(file);
 				};
@@ -470,6 +507,34 @@ class FileMiddleware extends SerializerMiddleware {
 					let currentBuffer;
 					let currentBufferUsed;
 					const buf = [];
+					let decompression;
+					if (file.endsWith(".gz")) {
+						decompression = createGunzip({
+							chunkSize: DECOMPRESSION_CHUNK_SIZE
+						});
+					} else if (file.endsWith(".br")) {
+						decompression = createBrotliDecompress({
+							chunkSize: DECOMPRESSION_CHUNK_SIZE
+						});
+					}
+					if (decompression) {
+						let newResolve, newReject;
+						resolve(
+							Promise.all([
+								new Promise((rs, rj) => {
+									newResolve = rs;
+									newReject = rj;
+								}),
+								new Promise((resolve, reject) => {
+									decompression.on("data", chunk => buf.push(chunk));
+									decompression.on("end", () => resolve());
+									decompression.on("error", err => reject(err));
+								})
+							]).then(() => buf)
+						);
+						resolve = newResolve;
+						reject = newReject;
+					}
 					this.fs.open(file, "r", (err, fd) => {
 						if (err) {
 							reject(err);
@@ -478,7 +543,11 @@ class FileMiddleware extends SerializerMiddleware {
 						const read = () => {
 							if (currentBuffer === undefined) {
 								currentBuffer = Buffer.allocUnsafeSlow(
-									Math.min(constants.MAX_LENGTH, remaining)
+									Math.min(
+										constants.MAX_LENGTH,
+										remaining,
+										decompression ? DECOMPRESSION_CHUNK_SIZE : Infinity
+									)
 								);
 								currentBufferUsed = 0;
 							}
@@ -509,9 +578,16 @@ class FileMiddleware extends SerializerMiddleware {
 									currentBufferUsed += bytesRead;
 									remaining -= bytesRead;
 									if (currentBufferUsed === currentBuffer.length) {
-										buf.push(currentBuffer);
+										if (decompression) {
+											decompression.write(currentBuffer);
+										} else {
+											buf.push(currentBuffer);
+										}
 										currentBuffer = undefined;
 										if (remaining === 0) {
+											if (decompression) {
+												decompression.end();
+											}
 											this.fs.close(fd, err => {
 												if (err) {
 													reject(err);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpack",
-  "version": "5.41.1",
+  "version": "5.42.0",
   "author": "Tobias Koppers @sokra",
   "description": "Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.",
   "license": "MIT",
@@ -10,7 +10,7 @@
     "@webassemblyjs/ast": "1.11.0",
     "@webassemblyjs/wasm-edit": "1.11.0",
     "@webassemblyjs/wasm-parser": "1.11.0",
-    "acorn": "^8.2.1",
+    "acorn": "^8.4.1",
     "browserslist": "^4.14.5",
     "chrome-trace-event": "^1.0.2",
     "enhanced-resolve": "^5.8.0",