webpack 5.51.0 → 5.52.1

package/lib/cache/PackFileCacheStrategy.js

@@ -76,7 +76,9 @@ makeSerializable(
 
 const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
 const CONTENT_COUNT_TO_MERGE = 10;
+const MIN_ITEMS_IN_FRESH_PACK = 100;
 const MAX_ITEMS_IN_FRESH_PACK = 50000;
+const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min
 
 class PackItemInfo {
 	/**
@@ -99,6 +101,7 @@ class Pack {
 		this.itemInfo = new Map();
 		/** @type {string[]} */
 		this.requests = [];
+		this.requestsTimeout = undefined;
 		/** @type {Map<string, PackItemInfo>} */
 		this.freshContent = new Map();
 		/** @type {(undefined | PackContent)[]} */
@@ -108,6 +111,16 @@ class Pack {
 		this.maxAge = maxAge;
 	}
 
+	_addRequest(identifier) {
+		this.requests.push(identifier);
+		if (this.requestsTimeout === undefined) {
+			this.requestsTimeout = setTimeout(() => {
+				this.requests.push(undefined);
+				this.requestsTimeout = undefined;
+			}, MAX_TIME_IN_FRESH_PACK);
+		}
+	}
+
 	/**
 	 * @param {string} identifier unique name for the resource
 	 * @param {string | null} etag etag of the resource
@@ -115,7 +128,7 @@ class Pack {
 	 */
 	get(identifier, etag) {
 		const info = this.itemInfo.get(identifier);
-		this.requests.push(identifier);
+		this._addRequest(identifier);
 		if (info === undefined) {
 			return undefined;
 		}
@@ -147,12 +160,12 @@ class Pack {
 		if (info === undefined) {
 			const newInfo = new PackItemInfo(identifier, etag, data);
 			this.itemInfo.set(identifier, newInfo);
-			this.requests.push(identifier);
+			this._addRequest(identifier);
 			this.freshContent.set(identifier, newInfo);
 		} else {
 			const loc = info.location;
 			if (loc >= 0) {
-				this.requests.push(identifier);
+				this._addRequest(identifier);
 				this.freshContent.set(identifier, info);
 				const content = this.content[loc];
 				content.delete(identifier);
@@ -220,27 +233,39 @@ class Pack {
 	}
 
 	_persistFreshContent() {
-		if (this.freshContent.size > 0) {
-			const packCount = Math.ceil(
-				this.freshContent.size / MAX_ITEMS_IN_FRESH_PACK
-			);
-			const itemsPerPack = Math.ceil(this.freshContent.size / packCount);
-			this.logger.log(`${this.freshContent.size} fresh items in cache`);
-			const packs = Array.from({ length: packCount }, () => {
+		const itemsCount = this.freshContent.size;
+		if (itemsCount > 0) {
+			const packCount = Math.ceil(itemsCount / MAX_ITEMS_IN_FRESH_PACK);
+			const itemsPerPack = Math.ceil(itemsCount / packCount);
+			const packs = [];
+			let i = 0;
+			let ignoreNextTimeTick = false;
+			const createNextPack = () => {
 				const loc = this._findLocation();
 				this.content[loc] = null; // reserve
-				return {
+				const pack = {
 					/** @type {Set<string>} */
 					items: new Set(),
 					/** @type {Map<string, any>} */
 					map: new Map(),
 					loc
 				};
-			});
-			let i = 0;
-			let pack = packs[0];
-			let packIndex = 0;
+				packs.push(pack);
+				return pack;
+			};
+			let pack = createNextPack();
+			if (this.requestsTimeout !== undefined)
+				clearTimeout(this.requestsTimeout);
 			for (const identifier of this.requests) {
+				if (identifier === undefined) {
+					if (ignoreNextTimeTick) {
+						ignoreNextTimeTick = false;
+					} else if (pack.items.size >= MIN_ITEMS_IN_FRESH_PACK) {
+						i = 0;
+						pack = createNextPack();
+					}
+					continue;
+				}
 				const info = this.freshContent.get(identifier);
 				if (info === undefined) continue;
 				pack.items.add(identifier);
@@ -250,9 +275,11 @@ class Pack {
 				this.freshContent.delete(identifier);
 				if (++i > itemsPerPack) {
 					i = 0;
-					pack = packs[++packIndex];
+					pack = createNextPack();
+					ignoreNextTimeTick = true;
 				}
 			}
+			this.requests.length = 0;
 			for (const pack of packs) {
 				this.content[pack.loc] = new PackContent(
 					pack.items,
@@ -260,6 +287,15 @@ class Pack {
 					new PackContentItems(pack.map)
 				);
 			}
+			this.logger.log(
+				`${itemsCount} fresh items in cache put into pack ${
+					packs.length > 1
+						? packs
+								.map(pack => `${pack.loc} (${pack.items.size} items)`)
+								.join(", ")
+						: packs[0].loc
+				}`
+			);
 		}
 	}
 
@@ -332,7 +368,9 @@ class Pack {
 			addToMergedMap.push(async map => {
 				// unpack existing content
 				// after that values are accessible in .content
-				await content.unpack();
+				await content.unpack(
+					"it should be merged with other small pack contents"
+				);
 				for (const [identifier, value] of content.content) {
 					map.set(identifier, value);
 				}
@@ -392,7 +430,9 @@ class Pack {
 						usedItems,
 						new Set(usedItems),
 						async () => {
-							await content.unpack();
+							await content.unpack(
+								"it should be splitted into used and unused items"
+							);
 							const map = new Map();
 							for (const identifier of usedItems) {
 								map.set(identifier, content.content.get(identifier));
@@ -417,7 +457,9 @@ class Pack {
 						unusedItems,
 						usedOfUnusedItems,
 						async () => {
-							await content.unpack();
+							await content.unpack(
+								"it should be splitted into used and unused items"
+							);
 							const map = new Map();
 							for (const identifier of unusedItems) {
 								map.set(identifier, content.content.get(identifier));
@@ -466,7 +508,9 @@ class Pack {
 			this.content[loc] =
 				items.size > 0
 					? new PackContent(items, usedItems, async () => {
-							await content.unpack();
+							await content.unpack(
+								"it contains old items that should be garbage collected"
+							);
 							const map = new Map();
 							for (const identifier of items) {
 								map.set(identifier, content.content.get(identifier));
@@ -496,7 +540,7 @@ class Pack {
 			const content = this.content[i];
 			if (content !== undefined) {
 				write(content.items);
-				writeSeparate(content.getLazyContentItems(), { name: `${i}` });
+				content.writeLazy(lazy => writeSeparate(lazy, { name: `${i}` }));
 			} else {
 				write(undefined); // undefined marks an empty content slot
 			}
@@ -669,6 +713,25 @@ makeSerializable(
 );
 
 class PackContent {
+	/*
+		This class can be in these states:
+		   |   this.lazy    | this.content | this.outdated | state
+		A1 |   undefined    |     Map      |     false     | fresh content
+		A2 |   undefined    |     Map      |     true      | (will not happen)
+		B1 | lazy () => {}  |  undefined   |     false     | not deserialized
+		B2 | lazy () => {}  |  undefined   |     true      | not deserialized, but some items has been removed
+		C1 | lazy* () => {} |     Map      |     false     | deserialized
+		C2 | lazy* () => {} |     Map      |     true      | deserialized, and some items has been removed
+
+		this.used is a subset of this.items.
+		this.items is a subset of this.content.keys() resp. this.lazy().map.keys()
+		When this.outdated === false, this.items === this.content.keys() resp. this.lazy().map.keys()
+		When this.outdated === true, this.items should be used to recreated this.lazy/this.content.
+		When this.lazy and this.content is set, they contain the same data.
+		this.get must only be called with a valid item from this.items.
+		In state C this.lazy is unMemoized
+	*/
+
 	/**
 	 * @param {Set<string>} items keys
 	 * @param {Set<string>} usedItems used keys
@@ -678,7 +741,7 @@ class PackContent {
 	 */
 	constructor(items, usedItems, dataOrFn, logger, lazyName) {
 		this.items = items;
-		/** @type {function(): Promise<PackContentItems> | PackContentItems } */
+		/** @type {function(): Promise<PackContentItems> | PackContentItems} */
 		this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined;
 		/** @type {Map<string, any>} */
 		this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map;
@@ -693,6 +756,8 @@ class PackContent {
 		if (this.content) {
 			return this.content.get(identifier);
 		}
+
+		// We are in state B
 		const { lazyName } = this;
 		let timeMessage;
 		if (lazyName) {
@@ -715,6 +780,7 @@ class PackContent {
 				if (timeMessage) {
 					this.logger.timeEnd(timeMessage);
 				}
+				// Move to state C
 				this.content = map;
 				this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 				return map.get(identifier);
@@ -724,6 +790,7 @@ class PackContent {
 			if (timeMessage) {
 				this.logger.timeEnd(timeMessage);
 			}
+			// Move to state C
 			this.content = map;
 			this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
 			return map.get(identifier);
@@ -731,10 +798,13 @@ class PackContent {
 	}
 
 	/**
+	 * @param {string} reason explanation why unpack is necessary
 	 * @returns {void | Promise} maybe a promise if lazy
 	 */
-	unpack() {
+	unpack(reason) {
 		if (this.content) return;
+
+		// Move from state B to C
 		if (this.lazy) {
 			const { lazyName } = this;
 			let timeMessage;
@@ -744,6 +814,11 @@ class PackContent {
 				timeMessage = `unpack cache content ${lazyName} (${formatSize(
 					this.getSize()
 				)})`;
+				this.logger.log(
+					`starting to unpack cache content ${lazyName} (${formatSize(
+						this.getSize()
+					)}) because ${reason}`
+				);
 				this.logger.time(timeMessage);
 			}
 			const value = this.lazy();
@@ -782,48 +857,93 @@ class PackContent {
 	}
 
 	/**
-	 * @returns {function(): PackContentItems | Promise<PackContentItems>} lazy content items
+	 * @template T
+	 * @param {function(any): function(): Promise<PackContentItems> | PackContentItems} write write function
+	 * @returns {void}
 	 */
-	getLazyContentItems() {
-		if (!this.outdated && this.lazy) return this.lazy;
+	writeLazy(write) {
+		if (!this.outdated && this.lazy) {
+			// State B1 or C1
+			// this.lazy is still the valid deserialized version
+			write(this.lazy);
+			return;
+		}
 		if (!this.outdated && this.content) {
+			// State A1
 			const map = new Map(this.content);
-			return (this.lazy = memoize(() => new PackContentItems(map)));
+			// Move to state C1
+			this.lazy = SerializerMiddleware.unMemoizeLazy(
+				write(() => new PackContentItems(map))
+			);
+			return;
 		}
-		this.outdated = false;
 		if (this.content) {
-			return (this.lazy = memoize(() => {
-				/** @type {Map<string, any>} */
-				const map = new Map();
-				for (const item of this.items) {
-					map.set(item, this.content.get(item));
-				}
-				return new PackContentItems(map);
-			}));
+			// State A2 or C2
+			/** @type {Map<string, any>} */
+			const map = new Map();
+			for (const item of this.items) {
+				map.set(item, this.content.get(item));
+			}
+			// Move to state C1
+			this.outdated = false;
+			this.content = map;
+			this.lazy = SerializerMiddleware.unMemoizeLazy(
+				write(() => new PackContentItems(map))
+			);
+			return;
 		}
-		const lazy = this.lazy;
-		return (this.lazy = () => {
-			const value = lazy();
-			if (value instanceof Promise) {
-				return value.then(data => {
+		// State B2
+		const { lazyName } = this;
+		let timeMessage;
+		if (lazyName) {
+			// only log once
+			this.lazyName = undefined;
+			timeMessage = `unpack cache content ${lazyName} (${formatSize(
+				this.getSize()
+			)})`;
+			this.logger.log(
+				`starting to unpack cache content ${lazyName} (${formatSize(
+					this.getSize()
+				)}) because it's outdated and need to be serialized`
+			);
+			this.logger.time(timeMessage);
+		}
+		const value = this.lazy();
+		this.outdated = false;
+		if (value instanceof Promise) {
+			// Move to state B1
+			this.lazy = write(() =>
+				value.then(data => {
+					if (timeMessage) {
+						this.logger.timeEnd(timeMessage);
+					}
 					const oldMap = data.map;
 					/** @type {Map<string, any>} */
 					const map = new Map();
 					for (const item of this.items) {
 						map.set(item, oldMap.get(item));
 					}
+					// Move to state C1 (or maybe C2)
+					this.content = map;
+					this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
+
 					return new PackContentItems(map);
-				});
-			} else {
-				const oldMap = value.map;
-				/** @type {Map<string, any>} */
-				const map = new Map();
-				for (const item of this.items) {
-					map.set(item, oldMap.get(item));
-				}
-				return new PackContentItems(map);
+				})
+			);
+		} else {
+			// Move to state C1
+			if (timeMessage) {
+				this.logger.timeEnd(timeMessage);
 			}
-		});
+			const oldMap = value.map;
+			/** @type {Map<string, any>} */
+			const map = new Map();
+			for (const item of this.items) {
+				map.set(item, oldMap.get(item));
+			}
+			this.content = map;
+			this.lazy = write(() => new PackContentItems(map));
+		}
 	}
 }
 
@@ -1110,10 +1230,10 @@ class PackFileCacheStrategy {
 		const packPromise = this.packPromise;
 		if (packPromise === undefined) return Promise.resolve();
 		const reportProgress = ProgressPlugin.getReporter(this.compiler);
-		this.packPromise = undefined;
 		return (this.storePromise = packPromise
 			.then(pack => {
 				if (!pack.invalid) return;
+				this.packPromise = undefined;
 				this.logger.log(`Storing pack...`);
 				let promise;
 				const newBuildDependencies = new Set();

package/lib/config/defaults.js

@@ -259,7 +259,6 @@ const applyExperimentsDefaults = (experiments, { production, development }) => {
 	D(experiments, "asyncWebAssembly", false);
 	D(experiments, "outputModule", false);
 	D(experiments, "asset", false);
-	D(experiments, "executeModule", false);
 	D(experiments, "layers", false);
 	D(experiments, "lazyCompilation", false);
 	D(experiments, "buildHttp", false);

package/lib/dependencies/HarmonyExportImportedSpecifierDependency.js

@@ -1039,7 +1039,9 @@ HarmonyExportImportedSpecifierDependency.Template = class HarmonyExportImportedS
 										ids,
 										runtimeRequirements
 									),
-								InitFragment.STAGE_HARMONY_IMPORTS,
+								moduleGraph.isAsync(importedModule)
+									? InitFragment.STAGE_ASYNC_HARMONY_IMPORTS
+									: InitFragment.STAGE_HARMONY_IMPORTS,
 								dep.sourceOrder
 							)
 						);
@@ -1100,7 +1102,9 @@ HarmonyExportImportedSpecifierDependency.Template = class HarmonyExportImportedS
 				initFragments.push(
 					new InitFragment(
 						`${content}\n/* harmony reexport (unknown) */ ${RuntimeGlobals.definePropertyGetters}(${exportsName}, __WEBPACK_REEXPORT_OBJECT__);\n`,
-						InitFragment.STAGE_HARMONY_IMPORTS,
+						moduleGraph.isAsync(importedModule)
+							? InitFragment.STAGE_ASYNC_HARMONY_IMPORTS
+							: InitFragment.STAGE_HARMONY_IMPORTS,
 						dep.sourceOrder
 					)
 				);

package/lib/dependencies/LoaderPlugin.js

@@ -37,11 +37,9 @@ const LoaderImportDependency = require("./LoaderImportDependency");
 class LoaderPlugin {
 	/**
 	 * @param {Object} options options
-	 * @param {boolean=} options.enableExecuteModule execute module enabled
 	 */
-	constructor(options = {}) {
-		this._enableExecuteModule = !!options.enableExecuteModule;
-	}
+	constructor(options = {}) {}
+
 	/**
 	 * Apply the plugin
 	 * @param {Compiler} compiler the compiler instance
@@ -155,106 +153,104 @@ class LoaderPlugin {
 						);
 					};
 
-					if (this._enableExecuteModule) {
-						/**
-						 * @param {string} request the request string to load the module from
-						 * @param {ImportModuleOptions=} options options
-						 * @param {ImportModuleCallback=} callback callback returning the exports
-						 * @returns {void}
-						 */
-						const importModule = (request, options, callback) => {
-							const dep = new LoaderImportDependency(request);
-							dep.loc = {
-								name: request
-							};
-							const factory = compilation.dependencyFactories.get(
-								/** @type {DepConstructor} */ (dep.constructor)
+					/**
+					 * @param {string} request the request string to load the module from
+					 * @param {ImportModuleOptions=} options options
+					 * @param {ImportModuleCallback=} callback callback returning the exports
+					 * @returns {void}
+					 */
+					const importModule = (request, options, callback) => {
+						const dep = new LoaderImportDependency(request);
+						dep.loc = {
+							name: request
+						};
+						const factory = compilation.dependencyFactories.get(
+							/** @type {DepConstructor} */ (dep.constructor)
+						);
+						if (factory === undefined) {
+							return callback(
+								new Error(
+									`No module factory available for dependency type: ${dep.constructor.name}`
+								)
 							);
-							if (factory === undefined) {
-								return callback(
-									new Error(
-										`No module factory available for dependency type: ${dep.constructor.name}`
-									)
-								);
-							}
-							compilation.buildQueue.increaseParallelism();
-							compilation.handleModuleCreation(
-								{
-									factory,
-									dependencies: [dep],
-									originModule: loaderContext._module,
-									contextInfo: {
-										issuerLayer: options.layer
-									},
-									context: loaderContext.context,
-									connectOrigin: false
+						}
+						compilation.buildQueue.increaseParallelism();
+						compilation.handleModuleCreation(
+							{
+								factory,
+								dependencies: [dep],
+								originModule: loaderContext._module,
+								contextInfo: {
+									issuerLayer: options.layer
 								},
-								err => {
-									compilation.buildQueue.decreaseParallelism();
-									if (err) {
-										return callback(err);
-									}
-									const referencedModule = moduleGraph.getModule(dep);
-									if (!referencedModule) {
-										return callback(new Error("Cannot load the module"));
-									}
-									compilation.executeModule(
-										referencedModule,
-										{
-											entryOptions: {
-												publicPath: options.publicPath
-											}
-										},
-										(err, result) => {
-											if (err) return callback(err);
-											for (const d of result.fileDependencies) {
-												loaderContext.addDependency(d);
-											}
-											for (const d of result.contextDependencies) {
-												loaderContext.addContextDependency(d);
-											}
-											for (const d of result.missingDependencies) {
-												loaderContext.addMissingDependency(d);
-											}
-											for (const d of result.buildDependencies) {
-												loaderContext.addBuildDependency(d);
-											}
-											if (result.cacheable === false)
-												loaderContext.cacheable(false);
-											for (const [name, { source, info }] of result.assets) {
-												const { buildInfo } = loaderContext._module;
-												if (!buildInfo.assets) {
-													buildInfo.assets = Object.create(null);
-													buildInfo.assetsInfo = new Map();
-												}
-												buildInfo.assets[name] = source;
-												buildInfo.assetsInfo.set(name, info);
+								context: loaderContext.context,
+								connectOrigin: false
+							},
+							err => {
+								compilation.buildQueue.decreaseParallelism();
+								if (err) {
+									return callback(err);
+								}
+								const referencedModule = moduleGraph.getModule(dep);
+								if (!referencedModule) {
+									return callback(new Error("Cannot load the module"));
+								}
+								compilation.executeModule(
+									referencedModule,
+									{
+										entryOptions: {
+											publicPath: options.publicPath
+										}
+									},
+									(err, result) => {
+										if (err) return callback(err);
+										for (const d of result.fileDependencies) {
+											loaderContext.addDependency(d);
+										}
+										for (const d of result.contextDependencies) {
+											loaderContext.addContextDependency(d);
+										}
+										for (const d of result.missingDependencies) {
+											loaderContext.addMissingDependency(d);
+										}
+										for (const d of result.buildDependencies) {
+											loaderContext.addBuildDependency(d);
+										}
+										if (result.cacheable === false)
+											loaderContext.cacheable(false);
+										for (const [name, { source, info }] of result.assets) {
+											const { buildInfo } = loaderContext._module;
+											if (!buildInfo.assets) {
+												buildInfo.assets = Object.create(null);
+												buildInfo.assetsInfo = new Map();
 											}
-											callback(null, result.exports);
+											buildInfo.assets[name] = source;
+											buildInfo.assetsInfo.set(name, info);
 										}
-									);
-								}
-							);
-						};
+										callback(null, result.exports);
+									}
+								);
+							}
+						);
+					};
 
-						/**
-						 * @param {string} request the request string to load the module from
-						 * @param {ImportModuleOptions} options options
-						 * @param {ImportModuleCallback=} callback callback returning the exports
-						 * @returns {Promise<any> | void} exports
-						 */
-						loaderContext.importModule = (request, options, callback) => {
-							if (!callback) {
-								return new Promise((resolve, reject) => {
-									importModule(request, options || {}, (err, result) => {
-										if (err) reject(err);
-										else resolve(result);
-									});
+					/**
+					 * @param {string} request the request string to load the module from
+					 * @param {ImportModuleOptions} options options
+					 * @param {ImportModuleCallback=} callback callback returning the exports
+					 * @returns {Promise<any> | void} exports
+					 */
+					loaderContext.importModule = (request, options, callback) => {
+						if (!callback) {
+							return new Promise((resolve, reject) => {
+								importModule(request, options || {}, (err, result) => {
+									if (err) reject(err);
+									else resolve(result);
 								});
-							}
-							return importModule(request, options || {}, callback);
-						};
-					}
+							});
+						}
+						return importModule(request, options || {}, callback);
+					};
 				}
 			);
 		});

package/lib/library/ModuleLibraryPlugin.js

@@ -78,6 +78,10 @@ class ModuleLibraryPlugin extends AbstractLibraryPlugin {
 		const result = new ConcatSource(source);
 		const exportsInfo = moduleGraph.getExportsInfo(module);
 		const exports = [];
+		const isAsync = moduleGraph.isAsync(module);
+		if (isAsync) {
+			result.add(`__webpack_exports__ = await __webpack_exports__;\n`);
+		}
 		for (const exportInfo of exportsInfo.orderedExports) {
 			if (!exportInfo.provided) continue;
 			const varName = `__webpack_exports__${Template.toIdentifier(