webpack 5.27.2 → 5.31.0

package/lib/asset/AssetGenerator.js

@@ -15,6 +15,7 @@ const { makePathsRelative } = require("../util/identifier");
 
 /** @typedef {import("webpack-sources").Source} Source */
 /** @typedef {import("../../declarations/WebpackOptions").AssetGeneratorOptions} AssetGeneratorOptions */
+/** @typedef {import("../../declarations/WebpackOptions").RawPublicPath} RawPublicPath */
 /** @typedef {import("../Compilation")} Compilation */
 /** @typedef {import("../Compiler")} Compiler */
 /** @typedef {import("../Generator").GenerateContext} GenerateContext */
@@ -24,6 +25,55 @@ const { makePathsRelative } = require("../util/identifier");
 /** @typedef {import("../RuntimeTemplate")} RuntimeTemplate */
 /** @typedef {import("../util/Hash")} Hash */
 
+const mergeMaybeArrays = (a, b) => {
+	const set = new Set();
+	if (Array.isArray(a)) for (const item of a) set.add(item);
+	else set.add(a);
+	if (Array.isArray(b)) for (const item of b) set.add(item);
+	else set.add(b);
+	return Array.from(set);
+};
+
+const mergeAssetInfo = (a, b) => {
+	const result = { ...a, ...b };
+	for (const key of Object.keys(a)) {
+		if (key in b) {
+			if (a[key] === b[key]) continue;
+			switch (key) {
+				case "fullhash":
+				case "chunkhash":
+				case "modulehash":
+				case "contenthash":
+					result[key] = mergeMaybeArrays(a[key], b[key]);
+					break;
+				case "immutable":
+				case "development":
+				case "hotModuleReplacement":
+				case "javascriptModule	":
+					result[key] = a[key] || b[key];
+					break;
+				case "related":
+					result[key] = mergeRelatedInfo(a[key], b[key]);
+					break;
+				default:
+					throw new Error(`Can't handle conflicting asset info for ${key}`);
+			}
+		}
+	}
+	return result;
+};
+
+const mergeRelatedInfo = (a, b) => {
+	const result = { ...a, ...b };
+	for (const key of Object.keys(a)) {
+		if (key in b) {
+			if (a[key] === b[key]) continue;
+			result[key] = mergeMaybeArrays(a[key], b[key]);
+		}
+	}
+	return result;
+};
+
 const JS_TYPES = new Set(["javascript"]);
 const JS_AND_ASSET_TYPES = new Set(["javascript", "asset"]);
 
@@ -31,12 +81,14 @@ class AssetGenerator extends Generator {
 	/**
 	 * @param {AssetGeneratorOptions["dataUrl"]=} dataUrlOptions the options for the data url
 	 * @param {string=} filename override for output.assetModuleFilename
+	 * @param {RawPublicPath=} publicPath override for output.assetModulePublicPath
 	 * @param {boolean=} emit generate output asset
 	 */
-	constructor(dataUrlOptions, filename, emit) {
+	constructor(dataUrlOptions, filename, publicPath, emit) {
 		super();
 		this.dataUrlOptions = dataUrlOptions;
 		this.filename = filename;
+		this.publicPath = publicPath;
 		this.emit = emit;
 	}
 
@@ -131,9 +183,9 @@ class AssetGenerator extends Generator {
 						module.matchResource || module.resource,
 						runtimeTemplate.compilation.compiler.root
 					).replace(/^\.\//, "");
-					const {
+					let {
 						path: filename,
-						info
+						info: assetInfo
 					} = runtimeTemplate.compilation.getAssetPathWithInfo(
 						assetModuleFilename,
 						{
@@ -144,11 +196,33 @@ class AssetGenerator extends Generator {
 							contentHash
 						}
 					);
-					module.buildInfo.filename = filename;
-					module.buildInfo.assetInfo = {
+					let publicPath;
+					if (this.publicPath) {
+						const {
+							path,
+							info
+						} = runtimeTemplate.compilation.getAssetPathWithInfo(
+							this.publicPath,
+							{
+								module,
+								runtime,
+								filename: sourceFilename,
+								chunkGraph,
+								contentHash
+							}
+						);
+						publicPath = JSON.stringify(path);
+						assetInfo = mergeAssetInfo(assetInfo, info);
+					} else {
+						publicPath = RuntimeGlobals.publicPath;
+						runtimeRequirements.add(RuntimeGlobals.publicPath); // add __webpack_require__.p
+					}
+					assetInfo = {
 						sourceFilename,
-						...info
+						...assetInfo
 					};
+					module.buildInfo.filename = filename;
+					module.buildInfo.assetInfo = assetInfo;
 					if (getData) {
 						// Due to code generation caching module.buildInfo.XXX can't used to store such information
 						// It need to be stored in the code generation results instead, where it's cached too
@@ -156,15 +230,13 @@ class AssetGenerator extends Generator {
 						const data = getData();
 						data.set("fullContentHash", fullHash);
 						data.set("filename", filename);
-						data.set("assetInfo", info);
+						data.set("assetInfo", assetInfo);
 					}
 
-					runtimeRequirements.add(RuntimeGlobals.publicPath); // add __webpack_require__.p
-
 					return new RawSource(
-						`${RuntimeGlobals.module}.exports = ${
-							RuntimeGlobals.publicPath
-						} + ${JSON.stringify(filename)};`
+						`${
+							RuntimeGlobals.module
+						}.exports = ${publicPath} + ${JSON.stringify(filename)};`
 					);
 				}
 			}

package/lib/asset/AssetModulesPlugin.js

@@ -118,8 +118,10 @@ class AssetModulesPlugin {
 							}
 
 							let filename = undefined;
+							let publicPath = undefined;
 							if (type !== "asset/inline") {
 								filename = generatorOptions.filename;
+								publicPath = generatorOptions.publicPath;
 							}
 
 							const AssetGenerator = getAssetGenerator();
@@ -127,6 +129,7 @@ class AssetModulesPlugin {
 							return new AssetGenerator(
 								dataUrl,
 								filename,
+								publicPath,
 								generatorOptions.emit !== false
 							);
 						});

package/lib/cache/IdleFileCachePlugin.js

@@ -30,7 +30,7 @@ class IdleFileCachePlugin {
 	 * @returns {void}
 	 */
 	apply(compiler) {
-		const strategy = this.strategy;
+		let strategy = this.strategy;
 		const idleTimeout = this.idleTimeout;
 		const idleTimeoutForInitialStore = Math.min(
 			idleTimeout,
@@ -54,20 +54,27 @@ class IdleFileCachePlugin {
 		compiler.cache.hooks.get.tapPromise(
 			{ name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
 			(identifier, etag, gotHandlers) => {
-				return strategy.restore(identifier, etag).then(cacheEntry => {
-					if (cacheEntry === undefined) {
-						gotHandlers.push((result, callback) => {
-							if (result !== undefined) {
-								pendingIdleTasks.set(identifier, () =>
-									strategy.store(identifier, etag, result)
-								);
-							}
-							callback();
-						});
-					} else {
-						return cacheEntry;
-					}
-				});
+				const restore = () =>
+					strategy.restore(identifier, etag).then(cacheEntry => {
+						if (cacheEntry === undefined) {
+							gotHandlers.push((result, callback) => {
+								if (result !== undefined) {
+									pendingIdleTasks.set(identifier, () =>
+										strategy.store(identifier, etag, result)
+									);
+								}
+								callback();
+							});
+						} else {
+							return cacheEntry;
+						}
+					});
+				const pendingTask = pendingIdleTasks.get(identifier);
+				if (pendingTask !== undefined) {
+					pendingIdleTasks.delete(identifier);
+					return pendingTask().then(restore);
+				}
+				return restore();
 			}
 		);
 
@@ -101,7 +108,10 @@ class IdleFileCachePlugin {
 						reportProgress(1, `stored`);
 					});
 				}
-				return currentIdlePromise;
+				return currentIdlePromise.then(() => {
+					// Reset strategy
+					if (strategy.clear) strategy.clear();
+				});
 			}
 		);
 

package/lib/cache/MemoryCachePlugin.js

@@ -46,6 +46,12 @@ class MemoryCachePlugin {
 				});
 			}
 		);
+		compiler.cache.hooks.shutdown.tap(
+			{ name: "MemoryCachePlugin", stage: Cache.STAGE_MEMORY },
+			() => {
+				cache.clear();
+			}
+		);
 	}
 }
 module.exports = MemoryCachePlugin;

package/lib/cache/MemoryWithGcCachePlugin.js

@@ -0,0 +1,129 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+
+"use strict";
+
+const Cache = require("../Cache");
+
+/** @typedef {import("webpack-sources").Source} Source */
+/** @typedef {import("../Cache").Etag} Etag */
+/** @typedef {import("../Compiler")} Compiler */
+/** @typedef {import("../Module")} Module */
+
+class MemoryWithGcCachePlugin {
+	constructor({ maxGenerations }) {
+		this._maxGenerations = maxGenerations;
+	}
+	/**
+	 * Apply the plugin
+	 * @param {Compiler} compiler the compiler instance
+	 * @returns {void}
+	 */
+	apply(compiler) {
+		const maxGenerations = this._maxGenerations;
+		/** @type {Map<string, { etag: Etag | null, data: any }>} */
+		const cache = new Map();
+		/** @type {Map<string, { entry: { etag: Etag | null, data: any }, until: number }>} */
+		const oldCache = new Map();
+		let generation = 0;
+		let cachePosition = 0;
+		const logger = compiler.getInfrastructureLogger("MemoryWithGcCachePlugin");
+		compiler.hooks.afterDone.tap("MemoryWithGcCachePlugin", () => {
+			generation++;
+			let clearedEntries = 0;
+			let lastClearedIdentifier;
+			for (const [identifier, entry] of oldCache) {
+				if (entry.until > generation) break;
+
+				oldCache.delete(identifier);
+				if (cache.get(identifier) === undefined) {
+					cache.delete(identifier);
+					clearedEntries++;
+					lastClearedIdentifier = identifier;
+				}
+			}
+			if (clearedEntries > 0 || oldCache.size > 0) {
+				logger.log(
+					`${cache.size - oldCache.size} active entries, ${
+						oldCache.size
+					} recently unused cached entries${
+						clearedEntries > 0
+							? `, ${clearedEntries} old unused cache entries removed e. g. ${lastClearedIdentifier}`
+							: ""
+					}`
+				);
+			}
+			let i = (cache.size / maxGenerations) | 0;
+			let j = cachePosition >= cache.size ? 0 : cachePosition;
+			cachePosition = j + i;
+			for (const [identifier, entry] of cache) {
+				if (j !== 0) {
+					j--;
+					continue;
+				}
+				if (entry !== undefined) {
+					// We don't delete the cache entry, but set it to undefined instead
+					// This reserves the location in the data table and avoids rehashing
+					// when constantly adding and removing entries.
+					// It will be deleted when removed from oldCache.
+					cache.set(identifier, undefined);
+					oldCache.delete(identifier);
+					oldCache.set(identifier, {
+						entry,
+						until: generation + maxGenerations
+					});
+					if (i-- === 0) break;
+				}
+			}
+		});
+		compiler.cache.hooks.store.tap(
+			{ name: "MemoryWithGcCachePlugin", stage: Cache.STAGE_MEMORY },
+			(identifier, etag, data) => {
+				cache.set(identifier, { etag, data });
+			}
+		);
+		compiler.cache.hooks.get.tap(
+			{ name: "MemoryWithGcCachePlugin", stage: Cache.STAGE_MEMORY },
+			(identifier, etag, gotHandlers) => {
+				const cacheEntry = cache.get(identifier);
+				if (cacheEntry === null) {
+					return null;
+				} else if (cacheEntry !== undefined) {
+					return cacheEntry.etag === etag ? cacheEntry.data : null;
+				}
+				const oldCacheEntry = oldCache.get(identifier);
+				if (oldCacheEntry !== undefined) {
+					const cacheEntry = oldCacheEntry.entry;
+					if (cacheEntry === null) {
+						oldCache.delete(identifier);
+						cache.set(identifier, cacheEntry);
+						return null;
+					} else {
+						if (cacheEntry.etag !== etag) return null;
+						oldCache.delete(identifier);
+						cache.set(identifier, cacheEntry);
+						return cacheEntry.data;
+					}
+				}
+				gotHandlers.push((result, callback) => {
+					if (result === undefined) {
+						cache.set(identifier, null);
+					} else {
+						cache.set(identifier, { etag, data: result });
+					}
+					return callback();
+				});
+			}
+		);
+		compiler.cache.hooks.shutdown.tap(
+			{ name: "MemoryWithGcCachePlugin", stage: Cache.STAGE_MEMORY },
+			() => {
+				cache.clear();
+				oldCache.clear();
+			}
+		);
+	}
+}
+module.exports = MemoryWithGcCachePlugin;

package/lib/cache/PackFileCacheStrategy.js

@@ -29,7 +29,7 @@ class PackContainer {
 	 * @param {string} version version identifier
 	 * @param {Snapshot} buildSnapshot snapshot of all build dependencies
 	 * @param {Set<string>} buildDependencies list of all unresolved build dependencies captured
-	 * @param {Map<string, string>} resolveResults result of the resolved build dependencies
+	 * @param {Map<string, string | false>} resolveResults result of the resolved build dependencies
 	 * @param {Snapshot} resolveBuildDependenciesSnapshot snapshot of the dependencies of the build dependencies resolving
 	 */
 	constructor(
@@ -75,7 +75,6 @@ makeSerializable(
 
 const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
 const CONTENT_COUNT_TO_MERGE = 10;
-const MAX_AGE = 1000 * 60 * 60 * 24 * 60; // 1 month
 const MAX_ITEMS_IN_FRESH_PACK = 50000;
 
 class PackItemInfo {
@@ -94,7 +93,7 @@ class PackItemInfo {
 }
 
 class Pack {
-	constructor(logger) {
+	constructor(logger, maxAge) {
 		/** @type {Map<string, PackItemInfo>} */
 		this.itemInfo = new Map();
 		/** @type {string[]} */
@@ -105,6 +104,7 @@ class Pack {
 		this.content = [];
 		this.invalid = false;
 		this.logger = logger;
+		this.maxAge = maxAge;
 	}
 
 	/**
@@ -167,6 +167,21 @@ class Pack {
 		}
 	}
 
+	getContentStats() {
+		let count = 0;
+		let size = 0;
+		for (const content of this.content) {
+			if (content !== undefined) {
+				count++;
+				const s = content.getSize();
+				if (s > 0) {
+					size += s;
+				}
+			}
+		}
+		return { count, size };
+	}
+
 	/**
 	 * @returns {number} new location of data entries
 	 */
@@ -182,7 +197,7 @@ class Pack {
 		const now = Date.now();
 		for (const identifier of items) {
 			const info = this.itemInfo.get(identifier);
-			if (now - info.lastAccess > MAX_AGE) {
+			if (now - info.lastAccess > this.maxAge) {
 				this.itemInfo.delete(identifier);
 				items.delete(identifier);
 				usedItems.delete(identifier);
@@ -194,9 +209,10 @@ class Pack {
 		}
 		if (count > 0) {
 			this.logger.log(
-				"Garbage Collected %d old items at pack %d e. g. %s",
+				"Garbage Collected %d old items at pack %d (%d items remaining) e. g. %s",
 				count,
 				newLoc,
+				items.size,
 				lastGC
 			);
 		}
@@ -426,10 +442,45 @@ class Pack {
 		}
 	}
 
+	/**
+	 * Find the content with the oldest item and run GC on that.
+	 * Only runs for one content to avoid large invalidation.
+	 */
+	_gcOldestContent() {
+		/** @type {PackItemInfo} */
+		let oldest = undefined;
+		for (const info of this.itemInfo.values()) {
+			if (oldest === undefined || info.lastAccess < oldest.lastAccess) {
+				oldest = info;
+			}
+		}
+		if (Date.now() - oldest.lastAccess > this.maxAge) {
+			const loc = oldest.location;
+			if (loc < 0) return;
+			const content = this.content[loc];
+			const items = new Set(content.items);
+			const usedItems = new Set(content.used);
+			this._gcAndUpdateLocation(items, usedItems, loc);
+
+			this.content[loc] =
+				items.size > 0
+					? new PackContent(items, usedItems, async () => {
+							await content.unpack();
+							const map = new Map();
+							for (const identifier of items) {
+								map.set(identifier, content.content.get(identifier));
+							}
+							return new PackContentItems(map);
+					  })
+					: undefined;
+		}
+	}
+
 	serialize({ write, writeSeparate }) {
 		this._persistFreshContent();
 		this._optimizeSmallContent();
 		this._optimizeUnusedContent();
+		this._gcOldestContent();
 		for (const identifier of this.itemInfo.keys()) {
 			write(identifier);
 		}
@@ -727,6 +778,7 @@ class PackFileCacheStrategy {
 	 * @param {string} options.version version identifier
 	 * @param {Logger} options.logger a logger
 	 * @param {SnapshotOptions} options.snapshot options regarding snapshotting
+	 * @param {number} options.maxAge max age of cache items
 	 */
 	constructor({
 		compiler,
@@ -735,7 +787,8 @@ class PackFileCacheStrategy {
 		cacheLocation,
 		version,
 		logger,
-		snapshot
+		snapshot,
+		maxAge
 	}) {
 		this.fileSerializer = createFileSerializer(fs);
 		this.fileSystemInfo = new FileSystemInfo(fs, {
@@ -748,6 +801,7 @@ class PackFileCacheStrategy {
 		this.cacheLocation = cacheLocation;
 		this.version = version;
 		this.logger = logger;
+		this.maxAge = maxAge;
 		this.snapshot = snapshot;
 		/** @type {Set<string>} */
 		this.buildDependencies = new Set();
@@ -755,12 +809,20 @@ class PackFileCacheStrategy {
 		this.newBuildDependencies = new LazySet();
 		/** @type {Snapshot} */
 		this.resolveBuildDependenciesSnapshot = undefined;
-		/** @type {Map<string, string>} */
+		/** @type {Map<string, string | false>} */
 		this.resolveResults = undefined;
 		/** @type {Snapshot} */
 		this.buildSnapshot = undefined;
 		/** @type {Promise<Pack>} */
 		this.packPromise = this._openPack();
+		this.storePromise = Promise.resolve();
+	}
+
+	_getPack() {
+		if (this.packPromise === undefined) {
+			this.packPromise = this.storePromise.then(() => this._openPack());
+		}
+		return this.packPromise;
 	}
 
 	/**
@@ -776,7 +838,7 @@ class PackFileCacheStrategy {
 		let newBuildDependencies;
 		/** @type {Snapshot} */
 		let resolveBuildDependenciesSnapshot;
-		/** @type {Map<string, string>} */
+		/** @type {Map<string, string | false>} */
 		let resolveResults;
 		logger.time("restore cache container");
 		return this.fileSerializer
@@ -899,6 +961,7 @@ class PackFileCacheStrategy {
 			})
 			.then(pack => {
 				if (pack) {
+					pack.maxAge = this.maxAge;
 					this.buildSnapshot = buildSnapshot;
 					if (buildDependencies) this.buildDependencies = buildDependencies;
 					if (newBuildDependencies)
@@ -907,14 +970,14 @@ class PackFileCacheStrategy {
 					this.resolveBuildDependenciesSnapshot = resolveBuildDependenciesSnapshot;
 					return pack;
 				}
-				return new Pack(logger);
+				return new Pack(logger, this.maxAge);
 			})
 			.catch(err => {
 				this.logger.warn(
 					`Restoring pack from ${cacheLocation}.pack failed: ${err}`
 				);
 				this.logger.debug(err.stack);
-				return new Pack(logger);
+				return new Pack(logger, this.maxAge);
 			});
 	}
 
@@ -925,7 +988,7 @@ class PackFileCacheStrategy {
 	 * @returns {Promise<void>} promise
 	 */
 	store(identifier, etag, data) {
-		return this.packPromise.then(pack => {
+		return this._getPack().then(pack => {
 			pack.set(identifier, etag === null ? null : etag.toString(), data);
 		});
 	}
@@ -936,7 +999,7 @@ class PackFileCacheStrategy {
 	 * @returns {Promise<any>} promise to the cached content
 	 */
 	restore(identifier, etag) {
-		return this.packPromise
+		return this._getPack()
 			.then(pack =>
 				pack.get(identifier, etag === null ? null : etag.toString())
 			)
@@ -955,8 +1018,11 @@ class PackFileCacheStrategy {
 	}
 
 	afterAllStored() {
+		const packPromise = this.packPromise;
+		if (packPromise === undefined) return Promise.resolve();
 		const reportProgress = ProgressPlugin.getReporter(this.compiler);
-		return this.packPromise
+		this.packPromise = undefined;
+		return (this.storePromise = packPromise
 			.then(pack => {
 				if (!pack.invalid) return;
 				this.logger.log(`Storing pack...`);
@@ -1100,7 +1166,13 @@ class PackFileCacheStrategy {
 							}
 							this.newBuildDependencies.clear();
 							this.logger.timeEnd(`store pack`);
-							this.logger.log(`Stored pack`);
+							const stats = pack.getContentStats();
+							this.logger.log(
+								"Stored pack (%d items, %d files, %d MiB)",
+								pack.itemInfo.size,
+								stats.count,
+								Math.round(stats.size / 1024 / 1024)
+							);
 						})
 						.catch(err => {
 							this.logger.timeEnd(`store pack`);
@@ -1112,7 +1184,17 @@ class PackFileCacheStrategy {
 			.catch(err => {
 				this.logger.warn(`Caching failed for pack: ${err}`);
 				this.logger.debug(err.stack);
-			});
+			}));
+	}
+
+	clear() {
+		this.fileSystemInfo.clear();
+		this.buildDependencies.clear();
+		this.newBuildDependencies.clear();
+		this.resolveBuildDependenciesSnapshot = undefined;
+		this.resolveResults = undefined;
+		this.buildSnapshot = undefined;
+		this.packPromise = undefined;
 	}
 }
 

package/lib/config/defaults.js

@@ -117,6 +117,7 @@ const A = (obj, prop, factory) => {
  */
 const applyWebpackOptionsBaseDefaults = options => {
 	F(options, "context", () => process.cwd());
+	applyInfrastructureLoggingDefaults(options.infrastructureLogging);
 };
 
 /**
@@ -163,7 +164,8 @@ const applyWebpackOptionsDefaults = options => {
 	);
 	applyCacheDefaults(options.cache, {
 		name: name || "default",
-		mode: mode || "production"
+		mode: mode || "production",
+		development
 	});
 	const cache = !!options.cache;
 
@@ -234,8 +236,6 @@ const applyWebpackOptionsDefaults = options => {
 		getResolveLoaderDefaults({ cache }),
 		options.resolveLoader
 	);
-
-	applyInfrastructureLoggingDefaults(options.infrastructureLogging);
 };
 
 /**
@@ -254,9 +254,10 @@ const applyExperimentsDefaults = experiments => {
  * @param {Object} options options
  * @param {string} options.name name
  * @param {string} options.mode mode
+ * @param {boolean} options.development is development mode
  * @returns {void}
  */
-const applyCacheDefaults = (cache, { name, mode }) => {
+const applyCacheDefaults = (cache, { name, mode, development }) => {
 	if (cache === false) return;
 	switch (cache.type) {
 		case "filesystem":
@@ -294,10 +295,15 @@ const applyCacheDefaults = (cache, { name, mode }) => {
 			D(cache, "store", "pack");
 			D(cache, "idleTimeout", 60000);
 			D(cache, "idleTimeoutForInitialStore", 0);
+			D(cache, "maxMemoryGenerations", development ? 10 : Infinity);
+			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
 			D(cache.buildDependencies, "defaultWebpack", [
 				path.resolve(__dirname, "..") + path.sep
 			]);
 			break;
+		case "memory":
+			D(cache, "maxGenerations", Infinity);
+			break;
 	}
 };
 
@@ -1070,8 +1076,14 @@ const getResolveLoaderDefaults = ({ cache }) => {
  * @returns {void}
  */
 const applyInfrastructureLoggingDefaults = infrastructureLogging => {
+	F(infrastructureLogging, "stream", () => process.stderr);
+	const tty =
+		/** @type {any} */ (infrastructureLogging.stream).isTTY &&
+		process.env.TERM !== "dumb";
 	D(infrastructureLogging, "level", "info");
 	D(infrastructureLogging, "debug", false);
+	D(infrastructureLogging, "colors", tty);
+	D(infrastructureLogging, "appendOnly", !tty);
 };
 
 exports.applyWebpackOptionsBaseDefaults = applyWebpackOptionsBaseDefaults;