webpack 5.51.2 → 5.54.0

package/lib/cache/PackFileCacheStrategy.js

@@ -76,7 +76,9 @@ makeSerializable(
 
 const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
 const CONTENT_COUNT_TO_MERGE = 10;
+const MIN_ITEMS_IN_FRESH_PACK = 100;
 const MAX_ITEMS_IN_FRESH_PACK = 50000;
+const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min
 
 class PackItemInfo {
 	/**
@@ -99,6 +101,7 @@ class Pack {
 		this.itemInfo = new Map();
 		/** @type {string[]} */
 		this.requests = [];
+		this.requestsTimeout = undefined;
 		/** @type {Map<string, PackItemInfo>} */
 		this.freshContent = new Map();
 		/** @type {(undefined | PackContent)[]} */
@@ -108,6 +111,24 @@ class Pack {
 		this.maxAge = maxAge;
 	}
 
+	_addRequest(identifier) {
+		this.requests.push(identifier);
+		if (this.requestsTimeout === undefined) {
+			this.requestsTimeout = setTimeout(() => {
+				this.requests.push(undefined);
+				this.requestsTimeout = undefined;
+			}, MAX_TIME_IN_FRESH_PACK);
+			if (this.requestsTimeout.unref) this.requestsTimeout.unref();
+		}
+	}
+
+	stopCapturingRequests() {
+		if (this.requestsTimeout !== undefined) {
+			clearTimeout(this.requestsTimeout);
+			this.requestsTimeout = undefined;
+		}
+	}
+
 	/**
 	 * @param {string} identifier unique name for the resource
 	 * @param {string | null} etag etag of the resource
@@ -115,7 +136,7 @@ class Pack {
 	 */
 	get(identifier, etag) {
 		const info = this.itemInfo.get(identifier);
-		this.requests.push(identifier);
+		this._addRequest(identifier);
 		if (info === undefined) {
 			return undefined;
 		}
@@ -147,12 +168,12 @@ class Pack {
 		if (info === undefined) {
 			const newInfo = new PackItemInfo(identifier, etag, data);
 			this.itemInfo.set(identifier, newInfo);
-			this.requests.push(identifier);
+			this._addRequest(identifier);
 			this.freshContent.set(identifier, newInfo);
 		} else {
 			const loc = info.location;
 			if (loc >= 0) {
-				this.requests.push(identifier);
+				this._addRequest(identifier);
 				this.freshContent.set(identifier, info);
 				const content = this.content[loc];
 				content.delete(identifier);
@@ -220,27 +241,39 @@ class Pack {
 	}
 
 	_persistFreshContent() {
-		if (this.freshContent.size > 0) {
-			const packCount = Math.ceil(
-				this.freshContent.size / MAX_ITEMS_IN_FRESH_PACK
-			);
-			const itemsPerPack = Math.ceil(this.freshContent.size / packCount);
-			this.logger.log(`${this.freshContent.size} fresh items in cache`);
-			const packs = Array.from({ length: packCount }, () => {
+		const itemsCount = this.freshContent.size;
+		if (itemsCount > 0) {
+			const packCount = Math.ceil(itemsCount / MAX_ITEMS_IN_FRESH_PACK);
+			const itemsPerPack = Math.ceil(itemsCount / packCount);
+			const packs = [];
+			let i = 0;
+			let ignoreNextTimeTick = false;
+			const createNextPack = () => {
 				const loc = this._findLocation();
 				this.content[loc] = null; // reserve
-				return {
+				const pack = {
 					/** @type {Set<string>} */
 					items: new Set(),
 					/** @type {Map<string, any>} */
 					map: new Map(),
 					loc
 				};
-			});
-			let i = 0;
-			let pack = packs[0];
-			let packIndex = 0;
+				packs.push(pack);
+				return pack;
+			};
+			let pack = createNextPack();
+			if (this.requestsTimeout !== undefined)
+				clearTimeout(this.requestsTimeout);
 			for (const identifier of this.requests) {
+				if (identifier === undefined) {
+					if (ignoreNextTimeTick) {
+						ignoreNextTimeTick = false;
+					} else if (pack.items.size >= MIN_ITEMS_IN_FRESH_PACK) {
+						i = 0;
+						pack = createNextPack();
+					}
+					continue;
+				}
 				const info = this.freshContent.get(identifier);
 				if (info === undefined) continue;
 				pack.items.add(identifier);
@@ -250,9 +283,11 @@ class Pack {
 				this.freshContent.delete(identifier);
 				if (++i > itemsPerPack) {
 					i = 0;
-					pack = packs[++packIndex];
+					pack = createNextPack();
+					ignoreNextTimeTick = true;
 				}
 			}
+			this.requests.length = 0;
 			for (const pack of packs) {
 				this.content[pack.loc] = new PackContent(
 					pack.items,
@@ -260,6 +295,15 @@ class Pack {
 					new PackContentItems(pack.map)
 				);
 			}
+			this.logger.log(
+				`${itemsCount} fresh items in cache put into pack ${
+					packs.length > 1
+						? packs
+								.map(pack => `${pack.loc} (${pack.items.size} items)`)
+								.join(", ")
+						: packs[0].loc
+				}`
+			);
 		}
 	}
 
@@ -332,7 +376,9 @@ class Pack {
 			addToMergedMap.push(async map => {
 				// unpack existing content
 				// after that values are accessible in .content
-				await content.unpack();
+				await content.unpack(
+					"it should be merged with other small pack contents"
+				);
 				for (const [identifier, value] of content.content) {
 					map.set(identifier, value);
 				}
@@ -392,7 +438,9 @@ class Pack {
 						usedItems,
 						new Set(usedItems),
 						async () => {
-							await content.unpack();
+							await content.unpack(
+								"it should be splitted into used and unused items"
+							);
 							const map = new Map();
 							for (const identifier of usedItems) {
 								map.set(identifier, content.content.get(identifier));
@@ -417,7 +465,9 @@ class Pack {
 						unusedItems,
 						usedOfUnusedItems,
 						async () => {
-							await content.unpack();
+							await content.unpack(
+								"it should be splitted into used and unused items"
+							);
 							const map = new Map();
 							for (const identifier of unusedItems) {
 								map.set(identifier, content.content.get(identifier));
@@ -466,7 +516,9 @@ class Pack {
 			this.content[loc] =
 				items.size > 0
 					? new PackContent(items, usedItems, async () => {
-							await content.unpack();
+							await content.unpack(
+								"it contains old items that should be garbage collected"
+							);
 							const map = new Map();
 							for (const identifier of items) {
 								map.set(identifier, content.content.get(identifier));
@@ -496,7 +548,7 @@ class Pack {
 			const content = this.content[i];
 			if (content !== undefined) {
 				write(content.items);
-				writeSeparate(content.getLazyContentItems(), { name: `${i}` });
+				content.writeLazy(lazy => writeSeparate(lazy, { name: `${i}` }));
 			} else {
 				write(undefined); // undefined marks an empty content slot
 			}
@@ -669,6 +721,25 @@ makeSerializable(
 );
 
 class PackContent {
+	/*
+		This class can be in these states:
+		   |   this.lazy    | this.content | this.outdated | state
+		A1 |   undefined    |     Map      |     false     | fresh content
+		A2 |   undefined    |     Map      |     true      | (will not happen)
+		B1 | lazy () => {}  |  undefined   |     false     | not deserialized
+		B2 | lazy () => {}  |  undefined   |     true      | not deserialized, but some items has been removed
+		C1 | lazy* () => {} |     Map      |     false     | deserialized
+		C2 | lazy* () => {} |     Map      |     true      | deserialized, and some items has been removed
+
+		this.used is a subset of this.items.
+		this.items is a subset of this.content.keys() resp. this.lazy().map.keys()
+		When this.outdated === false, this.items === this.content.keys() resp. this.lazy().map.keys()
+		When this.outdated === true, this.items should be used to recreated this.lazy/this.content.
+		When this.lazy and this.content is set, they contain the same data.
+		this.get must only be called with a valid item from this.items.
+		In state C this.lazy is unMemoized
+	*/
+
 	/**
 	 * @param {Set<string>} items keys
 	 * @param {Set<string>} usedItems used keys
@@ -678,7 +749,7 @@ class PackContent {
 	 */
 	constructor(items, usedItems, dataOrFn, logger, lazyName) {
 		this.items = items;
-		/** @type {function(): Promise<PackContentItems> | PackContentItems } */
+		/** @type {function(): Promise<PackContentItems> | PackContentItems} */
 		this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined;
 		/** @type {Map<string, any>} */
 		this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map;
@@ -693,6 +764,8 @@ class PackContent {
 		if (this.content) {
 			return this.content.get(identifier);
 		}
+
+		// We are in state B
 		const { lazyName } = this;
 		let timeMessage;
 		if (lazyName) {
@@ -715,6 +788,7 @@ class PackContent {
 				if (timeMessage) {
 					this.logger.timeEnd(timeMessage);
 				}
+				// Move to state C
 				this.content = map;
 				this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 				return map.get(identifier);
@@ -724,6 +798,7 @@ class PackContent {
 			if (timeMessage) {
 				this.logger.timeEnd(timeMessage);
 			}
+			// Move to state C
 			this.content = map;
 			this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 			return map.get(identifier);
@@ -731,10 +806,13 @@ class PackContent {
 	}
 
 	/**
+	 * @param {string} reason explanation why unpack is necessary
 	 * @returns {void | Promise} maybe a promise if lazy
 	 */
-	unpack() {
+	unpack(reason) {
 		if (this.content) return;
+
+		// Move from state B to C
 		if (this.lazy) {
 			const { lazyName } = this;
 			let timeMessage;
@@ -744,6 +822,11 @@ class PackContent {
 				timeMessage = `unpack cache content ${lazyName} (${formatSize(
 					this.getSize()
 				)})`;
+				this.logger.log(
+					`starting to unpack cache content ${lazyName} (${formatSize(
+						this.getSize()
+					)}) because ${reason}`
+				);
 				this.logger.time(timeMessage);
 			}
 			const value = this.lazy();
@@ -782,48 +865,93 @@ class PackContent {
 	}
 
 	/**
-	 * @returns {function(): PackContentItems | Promise<PackContentItems>} lazy content items
+	 * @template T
+	 * @param {function(any): function(): Promise<PackContentItems> | PackContentItems} write write function
+	 * @returns {void}
 	 */
-	getLazyContentItems() {
-		if (!this.outdated && this.lazy) return this.lazy;
+	writeLazy(write) {
+		if (!this.outdated && this.lazy) {
+			// State B1 or C1
+			// this.lazy is still the valid deserialized version
+			write(this.lazy);
+			return;
+		}
 		if (!this.outdated && this.content) {
+			// State A1
 			const map = new Map(this.content);
-			return (this.lazy = memoize(() => new PackContentItems(map)));
+			// Move to state C1
+			this.lazy = SerializerMiddleware.unMemoizeLazy(
+				write(() => new PackContentItems(map))
+			);
+			return;
 		}
-		this.outdated = false;
 		if (this.content) {
-			return (this.lazy = memoize(() => {
-				/** @type {Map<string, any>} */
-				const map = new Map();
-				for (const item of this.items) {
-					map.set(item, this.content.get(item));
-				}
-				return new PackContentItems(map);
-			}));
+			// State A2 or C2
+			/** @type {Map<string, any>} */
+			const map = new Map();
+			for (const item of this.items) {
+				map.set(item, this.content.get(item));
+			}
+			// Move to state C1
+			this.outdated = false;
+			this.content = map;
+			this.lazy = SerializerMiddleware.unMemoizeLazy(
+				write(() => new PackContentItems(map))
+			);
+			return;
 		}
-		const lazy = this.lazy;
-		return (this.lazy = () => {
-			const value = lazy();
-			if (value instanceof Promise) {
-				return value.then(data => {
+		// State B2
+		const { lazyName } = this;
+		let timeMessage;
+		if (lazyName) {
+			// only log once
+			this.lazyName = undefined;
+			timeMessage = `unpack cache content ${lazyName} (${formatSize(
+				this.getSize()
+			)})`;
+			this.logger.log(
+				`starting to unpack cache content ${lazyName} (${formatSize(
+					this.getSize()
+				)}) because it's outdated and need to be serialized`
+			);
+			this.logger.time(timeMessage);
+		}
+		const value = this.lazy();
+		this.outdated = false;
+		if (value instanceof Promise) {
+			// Move to state B1
+			this.lazy = write(() =>
+				value.then(data => {
+					if (timeMessage) {
+						this.logger.timeEnd(timeMessage);
+					}
 					const oldMap = data.map;
 					/** @type {Map<string, any>} */
 					const map = new Map();
 					for (const item of this.items) {
 						map.set(item, oldMap.get(item));
 					}
+					// Move to state C1 (or maybe C2)
+					this.content = map;
+					this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
+
 					return new PackContentItems(map);
-				});
-			} else {
-				const oldMap = value.map;
-				/** @type {Map<string, any>} */
-				const map = new Map();
-				for (const item of this.items) {
-					map.set(item, oldMap.get(item));
-				}
-				return new PackContentItems(map);
+				})
+			);
+		} else {
+			// Move to state C1
+			if (timeMessage) {
+				this.logger.timeEnd(timeMessage);
 			}
-		});
+			const oldMap = value.map;
+			/** @type {Map<string, any>} */
+			const map = new Map();
+			for (const item of this.items) {
+				map.set(item, oldMap.get(item));
+			}
+			this.content = map;
+			this.lazy = write(() => new PackContentItems(map));
+		}
 	}
 }
 
@@ -867,7 +995,8 @@ class PackFileCacheStrategy {
 		this.fileSystemInfo = new FileSystemInfo(fs, {
 			managedPaths: snapshot.managedPaths,
 			immutablePaths: snapshot.immutablePaths,
-			logger: logger.getChildLogger("webpack.FileSystemInfo")
+			logger: logger.getChildLogger("webpack.FileSystemInfo"),
+			hashFunction: compiler.options.output.hashFunction
 		});
 		this.compiler = compiler;
 		this.context = context;
@@ -1110,10 +1239,11 @@ class PackFileCacheStrategy {
 		const packPromise = this.packPromise;
 		if (packPromise === undefined) return Promise.resolve();
 		const reportProgress = ProgressPlugin.getReporter(this.compiler);
-		this.packPromise = undefined;
 		return (this.storePromise = packPromise
 			.then(pack => {
+				pack.stopCapturingRequests();
 				if (!pack.invalid) return;
+				this.packPromise = undefined;
 				this.logger.log(`Storing pack...`);
 				let promise;
 				const newBuildDependencies = new Set();

package/lib/cache/getLazyHashedEtag.js

@@ -8,6 +8,7 @@
 const createHash = require("../util/createHash");
 
 /** @typedef {import("../util/Hash")} Hash */
+/** @typedef {typeof import("../util/Hash")} HashConstructor */
 
 /**
  * @typedef {Object} HashableObject
@@ -17,10 +18,12 @@ const createHash = require("../util/createHash");
 class LazyHashedEtag {
 	/**
 	 * @param {HashableObject} obj object with updateHash method
+	 * @param {string | HashConstructor} hashFunction the hash function to use
 	 */
-	constructor(obj) {
+	constructor(obj, hashFunction = "md4") {
 		this._obj = obj;
 		this._hash = undefined;
+		this._hashFunction = hashFunction;
 	}
 
 	/**
@@ -28,7 +31,7 @@ class LazyHashedEtag {
 	 */
 	toString() {
 		if (this._hash === undefined) {
-			const hash = createHash("md4");
+			const hash = createHash(this._hashFunction);
 			this._obj.updateHash(hash);
 			this._hash = /** @type {string} */ (hash.digest("base64"));
 		}
@@ -36,18 +39,42 @@ class LazyHashedEtag {
 	}
 }
 
-/** @type {WeakMap<HashableObject, LazyHashedEtag>} */
-const map = new WeakMap();
+/** @type {Map<string | HashConstructor, WeakMap<HashableObject, LazyHashedEtag>>} */
+const mapStrings = new Map();
+
+/** @type {WeakMap<HashConstructor, WeakMap<HashableObject, LazyHashedEtag>>} */
+const mapObjects = new WeakMap();
 
 /**
  * @param {HashableObject} obj object with updateHash method
+ * @param {string | HashConstructor} hashFunction the hash function to use
  * @returns {LazyHashedEtag} etag
  */
-const getter = obj => {
-	const hash = map.get(obj);
+const getter = (obj, hashFunction = "md4") => {
+	let innerMap;
+	if (typeof hashFunction === "string") {
+		innerMap = mapStrings.get(hashFunction);
+		if (innerMap === undefined) {
+			const newHash = new LazyHashedEtag(obj, hashFunction);
+			innerMap = new WeakMap();
+			innerMap.set(obj, newHash);
+			mapStrings.set(hashFunction, innerMap);
+			return newHash;
+		}
+	} else {
+		innerMap = mapObjects.get(hashFunction);
+		if (innerMap === undefined) {
+			const newHash = new LazyHashedEtag(obj, hashFunction);
+			innerMap = new WeakMap();
+			innerMap.set(obj, newHash);
+			mapObjects.set(hashFunction, innerMap);
+			return newHash;
+		}
+	}
+	const hash = innerMap.get(obj);
 	if (hash !== undefined) return hash;
-	const newHash = new LazyHashedEtag(obj);
-	map.set(obj, newHash);
+	const newHash = new LazyHashedEtag(obj, hashFunction);
+	innerMap.set(obj, newHash);
 	return newHash;
 };
 

package/lib/config/defaults.js

@@ -159,20 +159,21 @@ const applyWebpackOptionsDefaults = options => {
 	D(options, "recordsInputPath", false);
 	D(options, "recordsOutputPath", false);
 
+	applyExperimentsDefaults(options.experiments, { production, development });
+
 	F(options, "cache", () =>
 		development ? { type: /** @type {"memory"} */ ("memory") } : false
 	);
 	applyCacheDefaults(options.cache, {
 		name: name || "default",
 		mode: mode || "production",
-		development
+		development,
+		cacheUnaffected: options.experiments.cacheUnaffected
 	});
 	const cache = !!options.cache;
 
 	applySnapshotDefaults(options.snapshot, { production });
 
-	applyExperimentsDefaults(options.experiments, { production, development });
-
 	applyModuleDefaults(options.module, {
 		cache,
 		syncWebAssembly: options.experiments.syncWebAssembly,
@@ -190,7 +191,8 @@ const applyWebpackOptionsDefaults = options => {
 		outputModule: options.experiments.outputModule,
 		development,
 		entry: options.entry,
-		module: options.module
+		module: options.module,
+		futureDefaults: options.experiments.futureDefaults
 	});
 
 	applyExternalsPresetsDefaults(options.externalsPresets, {
@@ -211,7 +213,10 @@ const applyWebpackOptionsDefaults = options => {
 			: "var";
 	});
 
-	applyNodeDefaults(options.node, { targetProperties });
+	applyNodeDefaults(options.node, {
+		futureDefaults: options.experiments.futureDefaults,
+		targetProperties
+	});
 
 	F(options, "performance", () =>
 		production &&
@@ -259,10 +264,11 @@ const applyExperimentsDefaults = (experiments, { production, development }) => {
 	D(experiments, "asyncWebAssembly", false);
 	D(experiments, "outputModule", false);
 	D(experiments, "asset", false);
-	D(experiments, "executeModule", false);
 	D(experiments, "layers", false);
 	D(experiments, "lazyCompilation", false);
 	D(experiments, "buildHttp", false);
+	D(experiments, "futureDefaults", false);
+	D(experiments, "cacheUnaffected", experiments.futureDefaults);
 
 	if (typeof experiments.buildHttp === "object") {
 		D(experiments.buildHttp, "frozen", production);
@@ -276,9 +282,13 @@ const applyExperimentsDefaults = (experiments, { production, development }) => {
  * @param {string} options.name name
  * @param {string} options.mode mode
  * @param {boolean} options.development is development mode
+ * @param {boolean} options.cacheUnaffected the cacheUnaffected experiment is enabled
  * @returns {void}
  */
-const applyCacheDefaults = (cache, { name, mode, development }) => {
+const applyCacheDefaults = (
+	cache,
+	{ name, mode, development, cacheUnaffected }
+) => {
 	if (cache === false) return;
 	switch (cache.type) {
 		case "filesystem":
@@ -322,12 +332,14 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			D(cache, "maxMemoryGenerations", development ? 5 : Infinity);
 			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
 			D(cache, "allowCollectingMemory", development);
+			D(cache, "memoryCacheUnaffected", development && cacheUnaffected);
 			D(cache.buildDependencies, "defaultWebpack", [
 				path.resolve(__dirname, "..") + path.sep
 			]);
 			break;
 		case "memory":
 			D(cache, "maxGenerations", Infinity);
+			D(cache, "cacheUnaffected", development && cacheUnaffected);
 			break;
 	}
 };
@@ -577,6 +589,7 @@ const applyModuleDefaults = (
  * @param {boolean} options.development is development mode
  * @param {Entry} options.entry entry option
  * @param {ModuleOptions} options.module module option
+ * @param {boolean} options.futureDefaults is future defaults enabled
  * @returns {void}
  */
 const applyOutputDefaults = (
@@ -588,7 +601,8 @@ const applyOutputDefaults = (
 		outputModule,
 		development,
 		entry,
-		module
+		module,
+		futureDefaults
 	}
 ) => {
 	/**
@@ -785,7 +799,7 @@ const applyOutputDefaults = (
 			: ""
 	);
 	D(output, "chunkLoadTimeout", 120000);
-	D(output, "hashFunction", "md4");
+	D(output, "hashFunction", futureDefaults ? "xxhash64" : "md4");
 	D(output, "hashDigest", "hex");
 	D(output, "hashDigestLength", 20);
 	D(output, "strictModuleExceptionHandling", false);
@@ -944,21 +958,26 @@ const applyLoaderDefaults = (loader, { targetProperties }) => {
  * @param {WebpackNode} node options
  * @param {Object} options options
  * @param {TargetProperties | false} options.targetProperties target properties
+ * @param {boolean} options.futureDefaults is future defaults enabled
  * @returns {void}
  */
-const applyNodeDefaults = (node, { targetProperties }) => {
+const applyNodeDefaults = (node, { futureDefaults, targetProperties }) => {
 	if (node === false) return;
+
 	F(node, "global", () => {
 		if (targetProperties && targetProperties.global) return false;
-		return true;
+		// TODO webpack 6 should always default to false
+		return futureDefaults ? "warn" : true;
 	});
 	F(node, "__filename", () => {
 		if (targetProperties && targetProperties.node) return "eval-only";
-		return "mock";
+		// TODO webpack 6 should always default to false
+		return futureDefaults ? "warn-mock" : "mock";
 	});
 	F(node, "__dirname", () => {
 		if (targetProperties && targetProperties.node) return "eval-only";
-		return "mock";
+		// TODO webpack 6 should always default to false
+		return futureDefaults ? "warn-mock" : "mock";
 	});
 };
 

package/lib/dependencies/CachedConstDependency.js

@@ -27,6 +27,7 @@ class CachedConstDependency extends NullDependency {
 		this.expression = expression;
 		this.range = range;
 		this.identifier = identifier;
+		this._hashUpdate = undefined;
 	}
 
 	/**
@@ -36,9 +37,9 @@ class CachedConstDependency extends NullDependency {
 	 * @returns {void}
 	 */
 	updateHash(hash, context) {
-		hash.update(this.identifier + "");
-		hash.update(this.range + "");
-		hash.update(this.expression + "");
+		if (this._hashUpdate === undefined)
+			this._hashUpdate = "" + this.identifier + this.range + this.expression;
+		hash.update(this._hashUpdate);
 	}
 
 	serialize(context) {