webpack 5.51.1 → 5.53.0

package/lib/cache/PackFileCacheStrategy.js

@@ -76,7 +76,9 @@ makeSerializable(
 
 const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
 const CONTENT_COUNT_TO_MERGE = 10;
+const MIN_ITEMS_IN_FRESH_PACK = 100;
 const MAX_ITEMS_IN_FRESH_PACK = 50000;
+const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min
 
 class PackItemInfo {
 	/**
@@ -99,6 +101,7 @@ class Pack {
 		this.itemInfo = new Map();
 		/** @type {string[]} */
 		this.requests = [];
+		this.requestsTimeout = undefined;
 		/** @type {Map<string, PackItemInfo>} */
 		this.freshContent = new Map();
 		/** @type {(undefined | PackContent)[]} */
@@ -108,6 +111,24 @@ class Pack {
 		this.maxAge = maxAge;
 	}
 
+	_addRequest(identifier) {
+		this.requests.push(identifier);
+		if (this.requestsTimeout === undefined) {
+			this.requestsTimeout = setTimeout(() => {
+				this.requests.push(undefined);
+				this.requestsTimeout = undefined;
+			}, MAX_TIME_IN_FRESH_PACK);
+			if (this.requestsTimeout.unref) this.requestsTimeout.unref();
+		}
+	}
+
+	stopCapturingRequests() {
+		if (this.requestsTimeout !== undefined) {
+			clearTimeout(this.requestsTimeout);
+			this.requestsTimeout = undefined;
+		}
+	}
+
 	/**
 	 * @param {string} identifier unique name for the resource
 	 * @param {string | null} etag etag of the resource
@@ -115,7 +136,7 @@ class Pack {
 	 */
 	get(identifier, etag) {
 		const info = this.itemInfo.get(identifier);
-		this.requests.push(identifier);
+		this._addRequest(identifier);
 		if (info === undefined) {
 			return undefined;
 		}
@@ -147,12 +168,12 @@ class Pack {
 		if (info === undefined) {
 			const newInfo = new PackItemInfo(identifier, etag, data);
 			this.itemInfo.set(identifier, newInfo);
-			this.requests.push(identifier);
+			this._addRequest(identifier);
 			this.freshContent.set(identifier, newInfo);
 		} else {
 			const loc = info.location;
 			if (loc >= 0) {
-				this.requests.push(identifier);
+				this._addRequest(identifier);
 				this.freshContent.set(identifier, info);
 				const content = this.content[loc];
 				content.delete(identifier);
@@ -220,27 +241,39 @@ class Pack {
 	}
 
 	_persistFreshContent() {
-		if (this.freshContent.size > 0) {
-			const packCount = Math.ceil(
-				this.freshContent.size / MAX_ITEMS_IN_FRESH_PACK
-			);
-			const itemsPerPack = Math.ceil(this.freshContent.size / packCount);
-			this.logger.log(`${this.freshContent.size} fresh items in cache`);
-			const packs = Array.from({ length: packCount }, () => {
+		const itemsCount = this.freshContent.size;
+		if (itemsCount > 0) {
+			const packCount = Math.ceil(itemsCount / MAX_ITEMS_IN_FRESH_PACK);
+			const itemsPerPack = Math.ceil(itemsCount / packCount);
+			const packs = [];
+			let i = 0;
+			let ignoreNextTimeTick = false;
+			const createNextPack = () => {
 				const loc = this._findLocation();
 				this.content[loc] = null; // reserve
-				return {
+				const pack = {
 					/** @type {Set<string>} */
 					items: new Set(),
 					/** @type {Map<string, any>} */
 					map: new Map(),
 					loc
 				};
-			});
-			let i = 0;
-			let pack = packs[0];
-			let packIndex = 0;
+				packs.push(pack);
+				return pack;
+			};
+			let pack = createNextPack();
+			if (this.requestsTimeout !== undefined)
+				clearTimeout(this.requestsTimeout);
 			for (const identifier of this.requests) {
+				if (identifier === undefined) {
+					if (ignoreNextTimeTick) {
+						ignoreNextTimeTick = false;
+					} else if (pack.items.size >= MIN_ITEMS_IN_FRESH_PACK) {
+						i = 0;
+						pack = createNextPack();
+					}
+					continue;
+				}
 				const info = this.freshContent.get(identifier);
 				if (info === undefined) continue;
 				pack.items.add(identifier);
@@ -250,9 +283,11 @@ class Pack {
 				this.freshContent.delete(identifier);
 				if (++i > itemsPerPack) {
 					i = 0;
-					pack = packs[++packIndex];
+					pack = createNextPack();
+					ignoreNextTimeTick = true;
 				}
 			}
+			this.requests.length = 0;
 			for (const pack of packs) {
 				this.content[pack.loc] = new PackContent(
 					pack.items,
@@ -260,6 +295,15 @@ class Pack {
 					new PackContentItems(pack.map)
 				);
 			}
+			this.logger.log(
+				`${itemsCount} fresh items in cache put into pack ${
+					packs.length > 1
+						? packs
+								.map(pack => `${pack.loc} (${pack.items.size} items)`)
+								.join(", ")
+						: packs[0].loc
+				}`
+			);
 		}
 	}
 
@@ -332,7 +376,9 @@ class Pack {
 			addToMergedMap.push(async map => {
 				// unpack existing content
 				// after that values are accessible in .content
-				await content.unpack();
+				await content.unpack(
+					"it should be merged with other small pack contents"
+				);
 				for (const [identifier, value] of content.content) {
 					map.set(identifier, value);
 				}
@@ -392,7 +438,9 @@ class Pack {
 						usedItems,
 						new Set(usedItems),
 						async () => {
-							await content.unpack();
+							await content.unpack(
+								"it should be splitted into used and unused items"
+							);
 							const map = new Map();
 							for (const identifier of usedItems) {
 								map.set(identifier, content.content.get(identifier));
@@ -417,7 +465,9 @@ class Pack {
 						unusedItems,
 						usedOfUnusedItems,
 						async () => {
-							await content.unpack();
+							await content.unpack(
+								"it should be splitted into used and unused items"
+							);
 							const map = new Map();
 							for (const identifier of unusedItems) {
 								map.set(identifier, content.content.get(identifier));
@@ -466,7 +516,9 @@ class Pack {
 			this.content[loc] =
 				items.size > 0
 					? new PackContent(items, usedItems, async () => {
-							await content.unpack();
+							await content.unpack(
+								"it contains old items that should be garbage collected"
+							);
 							const map = new Map();
 							for (const identifier of items) {
 								map.set(identifier, content.content.get(identifier));
@@ -496,7 +548,7 @@ class Pack {
 			const content = this.content[i];
 			if (content !== undefined) {
 				write(content.items);
-				writeSeparate(content.getLazyContentItems(), { name: `${i}` });
+				content.writeLazy(lazy => writeSeparate(lazy, { name: `${i}` }));
 			} else {
 				write(undefined); // undefined marks an empty content slot
 			}
@@ -669,6 +721,25 @@ makeSerializable(
 );
 
 class PackContent {
+	/*
+		This class can be in these states:
+		   |   this.lazy    | this.content | this.outdated | state
+		A1 |   undefined    |     Map      |     false     | fresh content
+		A2 |   undefined    |     Map      |     true      | (will not happen)
+		B1 | lazy () => {}  |  undefined   |     false     | not deserialized
+		B2 | lazy () => {}  |  undefined   |     true      | not deserialized, but some items has been removed
+		C1 | lazy* () => {} |     Map      |     false     | deserialized
+		C2 | lazy* () => {} |     Map      |     true      | deserialized, and some items has been removed
+
+		this.used is a subset of this.items.
+		this.items is a subset of this.content.keys() resp. this.lazy().map.keys()
+		When this.outdated === false, this.items === this.content.keys() resp. this.lazy().map.keys()
+		When this.outdated === true, this.items should be used to recreated this.lazy/this.content.
+		When this.lazy and this.content is set, they contain the same data.
+		this.get must only be called with a valid item from this.items.
+		In state C this.lazy is unMemoized
+	*/
+
 	/**
 	 * @param {Set<string>} items keys
 	 * @param {Set<string>} usedItems used keys
@@ -678,7 +749,7 @@ class PackContent {
 	 */
 	constructor(items, usedItems, dataOrFn, logger, lazyName) {
 		this.items = items;
-		/** @type {function(): Promise<PackContentItems> | PackContentItems } */
+		/** @type {function(): Promise<PackContentItems> | PackContentItems} */
 		this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined;
 		/** @type {Map<string, any>} */
 		this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map;
@@ -693,6 +764,8 @@ class PackContent {
 		if (this.content) {
 			return this.content.get(identifier);
 		}
+
+		// We are in state B
 		const { lazyName } = this;
 		let timeMessage;
 		if (lazyName) {
@@ -715,6 +788,7 @@ class PackContent {
 				if (timeMessage) {
 					this.logger.timeEnd(timeMessage);
 				}
+				// Move to state C
 				this.content = map;
 				this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 				return map.get(identifier);
@@ -724,6 +798,7 @@ class PackContent {
 			if (timeMessage) {
 				this.logger.timeEnd(timeMessage);
 			}
+			// Move to state C
 			this.content = map;
 			this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 			return map.get(identifier);
@@ -731,10 +806,13 @@ class PackContent {
 	}
 
 	/**
+	 * @param {string} reason explanation why unpack is necessary
 	 * @returns {void | Promise} maybe a promise if lazy
 	 */
-	unpack() {
+	unpack(reason) {
 		if (this.content) return;
+
+		// Move from state B to C
 		if (this.lazy) {
 			const { lazyName } = this;
 			let timeMessage;
@@ -744,6 +822,11 @@ class PackContent {
 				timeMessage = `unpack cache content ${lazyName} (${formatSize(
 					this.getSize()
 				)})`;
+				this.logger.log(
+					`starting to unpack cache content ${lazyName} (${formatSize(
+						this.getSize()
+					)}) because ${reason}`
+				);
 				this.logger.time(timeMessage);
 			}
 			const value = this.lazy();
@@ -782,48 +865,93 @@ class PackContent {
 	}
 
 	/**
-	 * @returns {function(): PackContentItems | Promise<PackContentItems>} lazy content items
+	 * @template T
+	 * @param {function(any): function(): Promise<PackContentItems> | PackContentItems} write write function
+	 * @returns {void}
 	 */
-	getLazyContentItems() {
-		if (!this.outdated && this.lazy) return this.lazy;
+	writeLazy(write) {
+		if (!this.outdated && this.lazy) {
+			// State B1 or C1
+			// this.lazy is still the valid deserialized version
+			write(this.lazy);
+			return;
+		}
 		if (!this.outdated && this.content) {
+			// State A1
 			const map = new Map(this.content);
-			return (this.lazy = memoize(() => new PackContentItems(map)));
+			// Move to state C1
+			this.lazy = SerializerMiddleware.unMemoizeLazy(
+				write(() => new PackContentItems(map))
+			);
+			return;
 		}
-		this.outdated = false;
 		if (this.content) {
-			return (this.lazy = memoize(() => {
-				/** @type {Map<string, any>} */
-				const map = new Map();
-				for (const item of this.items) {
-					map.set(item, this.content.get(item));
-				}
-				return new PackContentItems(map);
-			}));
+			// State A2 or C2
+			/** @type {Map<string, any>} */
+			const map = new Map();
+			for (const item of this.items) {
+				map.set(item, this.content.get(item));
+			}
+			// Move to state C1
+			this.outdated = false;
+			this.content = map;
+			this.lazy = SerializerMiddleware.unMemoizeLazy(
+				write(() => new PackContentItems(map))
+			);
+			return;
 		}
-		const lazy = this.lazy;
-		return (this.lazy = () => {
-			const value = lazy();
-			if (value instanceof Promise) {
-				return value.then(data => {
+		// State B2
+		const { lazyName } = this;
+		let timeMessage;
+		if (lazyName) {
+			// only log once
+			this.lazyName = undefined;
+			timeMessage = `unpack cache content ${lazyName} (${formatSize(
+				this.getSize()
+			)})`;
+			this.logger.log(
+				`starting to unpack cache content ${lazyName} (${formatSize(
+					this.getSize()
+				)}) because it's outdated and need to be serialized`
+			);
+			this.logger.time(timeMessage);
+		}
+		const value = this.lazy();
+		this.outdated = false;
+		if (value instanceof Promise) {
+			// Move to state B1
+			this.lazy = write(() =>
+				value.then(data => {
+					if (timeMessage) {
+						this.logger.timeEnd(timeMessage);
+					}
 					const oldMap = data.map;
 					/** @type {Map<string, any>} */
 					const map = new Map();
 					for (const item of this.items) {
 						map.set(item, oldMap.get(item));
 					}
+					// Move to state C1 (or maybe C2)
+					this.content = map;
+					this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
+
 					return new PackContentItems(map);
-				});
-			} else {
-				const oldMap = value.map;
-				/** @type {Map<string, any>} */
-				const map = new Map();
-				for (const item of this.items) {
-					map.set(item, oldMap.get(item));
-				}
-				return new PackContentItems(map);
+				})
+			);
+		} else {
+			// Move to state C1
+			if (timeMessage) {
+				this.logger.timeEnd(timeMessage);
 			}
-		});
+			const oldMap = value.map;
+			/** @type {Map<string, any>} */
+			const map = new Map();
+			for (const item of this.items) {
+				map.set(item, oldMap.get(item));
+			}
+			this.content = map;
+			this.lazy = write(() => new PackContentItems(map));
+		}
 	}
 }
 
@@ -1110,10 +1238,11 @@ class PackFileCacheStrategy {
 		const packPromise = this.packPromise;
 		if (packPromise === undefined) return Promise.resolve();
 		const reportProgress = ProgressPlugin.getReporter(this.compiler);
-		this.packPromise = undefined;
 		return (this.storePromise = packPromise
 			.then(pack => {
+				pack.stopCapturingRequests();
 				if (!pack.invalid) return;
+				this.packPromise = undefined;
 				this.logger.log(`Storing pack...`);
 				let promise;
 				const newBuildDependencies = new Set();

package/lib/config/defaults.js

@@ -211,7 +211,10 @@ const applyWebpackOptionsDefaults = options => {
 			: "var";
 	});
 
-	applyNodeDefaults(options.node, { targetProperties });
+	applyNodeDefaults(options.node, {
+		futureDefaults: options.experiments.futureDefaults,
+		targetProperties
+	});
 
 	F(options, "performance", () =>
 		production &&
@@ -259,10 +262,10 @@ const applyExperimentsDefaults = (experiments, { production, development }) => {
 	D(experiments, "asyncWebAssembly", false);
 	D(experiments, "outputModule", false);
 	D(experiments, "asset", false);
-	D(experiments, "executeModule", false);
 	D(experiments, "layers", false);
 	D(experiments, "lazyCompilation", false);
 	D(experiments, "buildHttp", false);
+	D(experiments, "futureDefaults", false);
 
 	if (typeof experiments.buildHttp === "object") {
 		D(experiments.buildHttp, "frozen", production);
@@ -944,21 +947,26 @@ const applyLoaderDefaults = (loader, { targetProperties }) => {
  * @param {WebpackNode} node options
  * @param {Object} options options
  * @param {TargetProperties | false} options.targetProperties target properties
+ * @param {boolean} options.futureDefaults is future defaults enabled
  * @returns {void}
  */
-const applyNodeDefaults = (node, { targetProperties }) => {
+const applyNodeDefaults = (node, { futureDefaults, targetProperties }) => {
 	if (node === false) return;
+
 	F(node, "global", () => {
 		if (targetProperties && targetProperties.global) return false;
-		return true;
+		// TODO webpack 6 should always default to false
+		return futureDefaults ? "warn" : true;
 	});
 	F(node, "__filename", () => {
 		if (targetProperties && targetProperties.node) return "eval-only";
-		return "mock";
+		// TODO webpack 6 should always default to false
+		return futureDefaults ? "warn-mock" : "mock";
 	});
 	F(node, "__dirname", () => {
 		if (targetProperties && targetProperties.node) return "eval-only";
-		return "mock";
+		// TODO webpack 6 should always default to false
+		return futureDefaults ? "warn-mock" : "mock";
 	});
 };
 

package/lib/dependencies/HarmonyExportImportedSpecifierDependency.js

@@ -1039,7 +1039,9 @@ HarmonyExportImportedSpecifierDependency.Template = class HarmonyExportImportedS
 										ids,
 										runtimeRequirements
 									),
-								InitFragment.STAGE_HARMONY_IMPORTS,
+								moduleGraph.isAsync(importedModule)
+									? InitFragment.STAGE_ASYNC_HARMONY_IMPORTS
+									: InitFragment.STAGE_HARMONY_IMPORTS,
 								dep.sourceOrder
 							)
 						);
@@ -1100,7 +1102,9 @@ HarmonyExportImportedSpecifierDependency.Template = class HarmonyExportImportedS
 				initFragments.push(
 					new InitFragment(
 						`${content}\n/* harmony reexport (unknown) */ ${RuntimeGlobals.definePropertyGetters}(${exportsName}, __WEBPACK_REEXPORT_OBJECT__);\n`,
-						InitFragment.STAGE_HARMONY_IMPORTS,
+						moduleGraph.isAsync(importedModule)
+							? InitFragment.STAGE_ASYNC_HARMONY_IMPORTS
+							: InitFragment.STAGE_HARMONY_IMPORTS,
 						dep.sourceOrder
 					)
 				);