webpack 5.12.3 → 5.16.0

package/lib/FileSystemInfo.js

@@ -11,31 +11,27 @@ const AsyncQueue = require("./util/AsyncQueue");
 const createHash = require("./util/createHash");
 const { join, dirname, relative } = require("./util/fs");
 const makeSerializable = require("./util/makeSerializable");
+const processAsyncTree = require("./util/processAsyncTree");
 
 /** @typedef {import("./WebpackError")} WebpackError */
 /** @typedef {import("./logging/Logger").Logger} Logger */
 /** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
 
-const resolveContext = createResolver({
-	resolveToContext: true,
-	exportsFields: []
-});
-const resolve = createResolver({
-	extensions: [".js", ".json", ".node"],
-	conditionNames: ["require"]
-});
+const supportsEsm = +process.versions.modules >= 83;
 
 let FS_ACCURACY = 2000;
 
 const EMPTY_SET = new Set();
 
-const RBDT_RESOLVE = 0;
-const RBDT_RESOLVE_DIRECTORY = 1;
-const RBDT_RESOLVE_FILE = 2;
-const RBDT_DIRECTORY = 3;
-const RBDT_FILE = 4;
-const RBDT_DIRECTORY_DEPENDENCIES = 5;
-const RBDT_FILE_DEPENDENCIES = 6;
+const RBDT_RESOLVE_CJS = 0;
+const RBDT_RESOLVE_ESM = 1;
+const RBDT_RESOLVE_DIRECTORY = 2;
+const RBDT_RESOLVE_CJS_FILE = 3;
+const RBDT_RESOLVE_ESM_FILE = 4;
+const RBDT_DIRECTORY = 5;
+const RBDT_FILE = 6;
+const RBDT_DIRECTORY_DEPENDENCIES = 7;
+const RBDT_FILE_DEPENDENCIES = 8;
 
 const INVALID = Symbol("invalid");
 
@@ -880,6 +876,8 @@ class FileSystemInfo {
 		this._cachedDeprecatedFileTimestamps = undefined;
 		this._cachedDeprecatedContextTimestamps = undefined;
 
+		this._warnAboutExperimentalEsmTracking = false;
+
 		this._statCreatedSnapshots = 0;
 		this._statTestedSnapshotsCached = 0;
 		this._statTestedSnapshotsNotCached = 0;
@@ -1052,6 +1050,26 @@ class FileSystemInfo {
 		this.contextHashQueue.add(path, callback);
 	}
 
+	_createBuildDependenciesResolvers() {
+		const resolveContext = createResolver({
+			resolveToContext: true,
+			exportsFields: [],
+			fileSystem: this.fs
+		});
+		const resolveCjs = createResolver({
+			extensions: [".js", ".json", ".node"],
+			conditionNames: ["require", "node"],
+			fileSystem: this.fs
+		});
+		const resolveEsm = createResolver({
+			extensions: [".js", ".json", ".node"],
+			fullySpecified: true,
+			conditionNames: ["import", "node"],
+			fileSystem: this.fs
+		});
+		return { resolveContext, resolveEsm, resolveCjs };
+	}
+
 	/**
 	 * @param {string} context context directory
 	 * @param {Iterable<string>} deps dependencies
@@ -1059,11 +1077,21 @@ class FileSystemInfo {
 	 * @returns {void}
 	 */
 	resolveBuildDependencies(context, deps, callback) {
+		const {
+			resolveContext,
+			resolveEsm,
+			resolveCjs
+		} = this._createBuildDependenciesResolvers();
+
 		/** @type {Set<string>} */
 		const files = new Set();
 		/** @type {Set<string>} */
+		const fileSymlinks = new Set();
+		/** @type {Set<string>} */
 		const directories = new Set();
 		/** @type {Set<string>} */
+		const directorySymlinks = new Set();
+		/** @type {Set<string>} */
 		const missing = new Set();
 		/** @type {Set<string>} */
 		const resolveFiles = new Set();
@@ -1073,83 +1101,99 @@ class FileSystemInfo {
 		const resolveMissing = new Set();
 		/** @type {Map<string, string>} */
 		const resolveResults = new Map();
-		/** @type {asyncLib.QueueObject<{type: number, path: string, context?: string, expected?: string }, Error>} */
-		const queue = asyncLib.queue(
-			({ type, context, path, expected }, callback) => {
+		const invalidResolveResults = new Set();
+		const resolverContext = {
+			fileDependencies: resolveFiles,
+			contextDependencies: resolveDirectories,
+			missingDependencies: resolveMissing
+		};
+		processAsyncTree(
+			Array.from(deps, dep => ({
+				type: RBDT_RESOLVE_CJS,
+				context,
+				path: dep,
+				expected: undefined
+			})),
+			20,
+			({ type, context, path, expected }, push, callback) => {
 				const resolveDirectory = path => {
 					const key = `d\n${context}\n${path}`;
 					if (resolveResults.has(key)) {
 						return callback();
 					}
-					resolveContext(
-						context,
-						path,
-						{
-							fileDependencies: resolveFiles,
-							contextDependencies: resolveDirectories,
-							missingDependencies: resolveMissing
-						},
-						(err, result) => {
+					resolveResults.set(key, undefined);
+					resolveContext(context, path, resolverContext, (err, result) => {
+						if (err) {
+							invalidResolveResults.add(key);
+							if (
+								err.code === "ENOENT" ||
+								err.code === "UNDECLARED_DEPENDENCY"
+							) {
+								return callback();
+							}
+							err.message += `\nwhile resolving '${path}' in ${context} to a directory`;
+							return callback(err);
+						}
+						resolveResults.set(key, result);
+						push({
+							type: RBDT_DIRECTORY,
+							context: undefined,
+							path: result,
+							expected: undefined
+						});
+						callback();
+					});
+				};
+				const resolveFile = (path, symbol, resolve) => {
+					const key = `${symbol}\n${context}\n${path}`;
+					if (resolveResults.has(key)) {
+						return callback();
+					}
+					resolveResults.set(key, undefined);
+					resolve(context, path, resolverContext, (err, result) => {
+						if (expected) {
+							if (result === expected) {
+								resolveResults.set(key, result);
+							} else {
+								invalidResolveResults.add(key);
+								this.logger.debug(
+									`Resolving '${path}' in ${context} for build dependencies doesn't lead to expected result '${expected}', but to '${result}' instead. Resolving dependencies are ignored for this path.`
+								);
+							}
+						} else {
 							if (err) {
+								invalidResolveResults.add(key);
 								if (
 									err.code === "ENOENT" ||
 									err.code === "UNDECLARED_DEPENDENCY"
 								) {
 									return callback();
 								}
-								err.message += `\nwhile resolving '${path}' in ${context} to a directory`;
+								err.message += `\nwhile resolving '${path}' in ${context} as file`;
 								return callback(err);
 							}
 							resolveResults.set(key, result);
-							queue.push({
-								type: RBDT_DIRECTORY,
-								path: result
+							push({
+								type: RBDT_FILE,
+								context: undefined,
+								path: result,
+								expected: undefined
 							});
-							callback();
-						}
-					);
-				};
-				const resolveFile = path => {
-					const key = `f\n${context}\n${path}`;
-					if (resolveResults.has(key)) {
-						return callback();
-					}
-					resolve(
-						context,
-						path,
-						{
-							fileDependencies: resolveFiles,
-							contextDependencies: resolveDirectories,
-							missingDependencies: resolveMissing
-						},
-						(err, result) => {
-							if (expected) {
-								if (result === expected) {
-									resolveResults.set(key, result);
-								}
-							} else {
-								if (err) {
-									if (
-										err.code === "ENOENT" ||
-										err.code === "UNDECLARED_DEPENDENCY"
-									) {
-										return callback();
-									}
-									err.message += `\nwhile resolving '${path}' in ${context} as file`;
-									return callback(err);
-								}
-								resolveResults.set(key, result);
-								queue.push({
-									type: RBDT_FILE,
-									path: result
-								});
-							}
-							callback();
 						}
-					);
+						callback();
+					});
 				};
 				switch (type) {
-					case RBDT_RESOLVE: {
+					case RBDT_RESOLVE_CJS: {
+						const isDirectory = /[\\/]$/.test(path);
+						if (isDirectory) {
+							resolveDirectory(path.slice(0, path.length - 1));
+						} else {
+							resolveFile(path, "f", resolveCjs);
+						}
+						break;
+					}
+					case RBDT_RESOLVE_ESM: {
 						const isDirectory = /[\\/]$/.test(path);
 						if (isDirectory) {
 							resolveDirectory(path.slice(0, path.length - 1));
@@ -1162,8 +1206,12 @@ class FileSystemInfo {
 						resolveDirectory(path);
 						break;
 					}
-					case RBDT_RESOLVE_FILE: {
-						resolveFile(path);
+					case RBDT_RESOLVE_CJS_FILE: {
+						resolveFile(path, "f", resolveCjs);
+						break;
+					}
+					case RBDT_RESOLVE_ESM_FILE: {
+						resolveFile(path, "e", resolveEsm);
 						break;
 					}
 					case RBDT_FILE: {
@@ -1171,18 +1219,22 @@ class FileSystemInfo {
 							callback();
 							break;
 						}
-						this.fs.realpath(path, (err, realPath) => {
+						files.add(path);
+						this.fs.realpath(path, (err, _realPath) => {
 							if (err) return callback(err);
+							const realPath = /** @type {string} */ (_realPath);
 							if (realPath !== path) {
+								fileSymlinks.add(path);
 								resolveFiles.add(path);
-							}
-							if (!files.has(realPath)) {
+								if (files.has(realPath)) return callback();
 								files.add(realPath);
-								queue.push({
-									type: RBDT_FILE_DEPENDENCIES,
-									path: realPath
-								});
 							}
+							push({
+								type: RBDT_FILE_DEPENDENCIES,
+								context: undefined,
+								path: realPath,
+								expected: undefined
+							});
 							callback();
 						});
 						break;
@@ -1192,69 +1244,140 @@ class FileSystemInfo {
 							callback();
 							break;
 						}
-						this.fs.realpath(path, (err, realPath) => {
+						directories.add(path);
+						this.fs.realpath(path, (err, _realPath) => {
 							if (err) return callback(err);
+							const realPath = /** @type {string} */ (_realPath);
 							if (realPath !== path) {
+								directorySymlinks.add(path);
 								resolveFiles.add(path);
-							}
-							if (!directories.has(realPath)) {
+								if (directories.has(realPath)) return callback();
 								directories.add(realPath);
-								queue.push({
-									type: RBDT_DIRECTORY_DEPENDENCIES,
-									path: realPath
-								});
 							}
+							push({
+								type: RBDT_DIRECTORY_DEPENDENCIES,
+								context: undefined,
+								path: realPath,
+								expected: undefined
+							});
 							callback();
 						});
 						break;
 					}
 					case RBDT_FILE_DEPENDENCIES: {
-						// TODO this probably doesn't work correctly with ESM dependencies
+						// Check for known files without dependencies
+						if (/\.json5?$|\.yarn-integrity$|yarn\.lock$|\.ya?ml/.test(path)) {
+							process.nextTick(callback);
+							break;
+						}
+						// Check commonjs cache for the module
 						/** @type {NodeModule} */
 						const module = require.cache[path];
 						if (module && Array.isArray(module.children)) {
 							children: for (const child of module.children) {
 								let childPath = child.filename;
 								if (childPath) {
-									queue.push({
+									push({
 										type: RBDT_FILE,
-										path: childPath
+										context: undefined,
+										path: childPath,
+										expected: undefined
 									});
-									if (childPath.endsWith(".js"))
-										childPath = childPath.slice(0, -3);
 									const context = dirname(this.fs, path);
 									for (const modulePath of module.paths) {
 										if (childPath.startsWith(modulePath)) {
-											const request = childPath.slice(modulePath.length + 1);
-											queue.push({
-												type: RBDT_RESOLVE_FILE,
+											let request = childPath.slice(modulePath.length + 1);
+											if (request.endsWith(".js"))
+												request = request.slice(0, -3);
+											push({
+												type: RBDT_RESOLVE_CJS_FILE,
 												context,
 												path: request,
-												expected: childPath
+												expected: child.filename
 											});
 											continue children;
 										}
 									}
 									let request = relative(this.fs, context, childPath);
+									if (request.endsWith(".js")) request = request.slice(0, -3);
 									request = request.replace(/\\/g, "/");
 									if (!request.startsWith("../")) request = `./${request}`;
-									queue.push({
-										type: RBDT_RESOLVE_FILE,
+									push({
+										type: RBDT_RESOLVE_CJS_FILE,
 										context,
 										path: request,
 										expected: child.filename
 									});
 								}
 							}
+						} else if (supportsEsm && /\.m?js$/.test(path)) {
+							if (!this._warnAboutExperimentalEsmTracking) {
+								this.logger.info(
+									"Node.js doesn't offer a (nice) way to introspect the ESM dependency graph yet.\n" +
+										"Until a full solution is available webpack uses an experimental ESM tracking based on parsing.\n" +
+										"As best effort webpack parses the ESM files to guess dependencies. But this can lead to expensive and incorrect tracking."
+								);
+								this._warnAboutExperimentalEsmTracking = true;
+							}
+							const lexer = require("es-module-lexer");
+							lexer.init.then(() => {
+								this.fs.readFile(path, (err, content) => {
+									if (err) return callback(err);
+									try {
+										const context = dirname(this.fs, path);
+										const source = content.toString();
+										const [imports] = lexer.parse(source);
+										for (const imp of imports) {
+											try {
+												let dependency;
+												if (imp.d === -1) {
+													// import ... from "..."
+													dependency = JSON.parse(
+														source.substring(imp.s - 1, imp.e + 1)
+													);
+												} else if (imp.d > -1) {
+													// import()
+													let expr = source.substring(imp.s, imp.e).trim();
+													if (expr[0] === "'")
+														expr = `"${expr
+															.slice(1, -1)
+															.replace(/"/g, '\\"')}"`;
+													dependency = JSON.parse(expr);
+												} else {
+													// e.g. import.meta
+													continue;
+												}
+												push({
+													type: RBDT_RESOLVE_ESM_FILE,
+													context,
+													path: dependency,
+													expected: undefined
+												});
+											} catch (e) {
+												this.logger.warn(
+													`Parsing of ${path} for build dependencies failed at 'import(${source.substring(
+														imp.s,
+														imp.e
+													)})'.\n` +
+														"Build dependencies behind this expression are ignored and might cause incorrect cache invalidation."
+												);
+												this.logger.debug(e.stack);
+											}
+										}
+									} catch (e) {
+										this.logger.warn(
+											`Parsing of ${path} for build dependencies failed and all dependencies of this file are ignored, which might cause incorrect cache invalidation..`
+										);
+										this.logger.debug(e.stack);
+									}
+									process.nextTick(callback);
+								});
+							}, callback);
+							break;
 						} else {
-							// Unable to get dependencies from module system
-							// This may be because of an incomplete require.cache implementation like in jest
-							// Assume requires stay in directory and add the whole directory
-							const directory = dirname(this.fs, path);
-							queue.push({
-								type: RBDT_DIRECTORY,
-								path: directory
-							});
+							this.logger.log(
+								`Assuming ${path} has no dependencies as we were unable to assign it to any module system.`
+							);
 						}
 						process.nextTick(callback);
 						break;
@@ -1271,9 +1394,11 @@ class FileSystemInfo {
 									resolveMissing.add(packageJson);
 									const parent = dirname(this.fs, packagePath);
 									if (parent !== packagePath) {
-										queue.push({
+										push({
 											type: RBDT_DIRECTORY_DEPENDENCIES,
-											path: parent
+											context: undefined,
+											path: parent,
+											expected: undefined
 										});
 									}
 									callback();
@@ -1291,10 +1416,11 @@ class FileSystemInfo {
 							const depsObject = packageData.dependencies;
 							if (typeof depsObject === "object" && depsObject) {
 								for (const dep of Object.keys(depsObject)) {
-									queue.push({
+									push({
 										type: RBDT_RESOLVE_DIRECTORY,
 										context: packagePath,
-										path: dep
+										path: dep,
+										expected: undefined
 									});
 								}
 							}
@@ -1304,38 +1430,24 @@ class FileSystemInfo {
 					}
 				}
 			},
-			50
+			err => {
+				if (err) return callback(err);
+				for (const l of fileSymlinks) files.delete(l);
+				for (const l of directorySymlinks) directories.delete(l);
+				for (const k of invalidResolveResults) resolveResults.delete(k);
+				callback(null, {
+					files,
+					directories,
+					missing,
+					resolveResults,
+					resolveDependencies: {
+						files: resolveFiles,
+						directories: resolveDirectories,
+						missing: resolveMissing
+					}
+				});
+			}
 		);
-		queue.drain = () => {
-			callback(null, {
-				files,
-				directories,
-				missing,
-				resolveResults,
-				resolveDependencies: {
-					files: resolveFiles,
-					directories: resolveDirectories,
-					missing: resolveMissing
-				}
-			});
-		};
-		queue.error = err => {
-			callback(err);
-			callback = () => {};
-		};
-		let jobQueued = false;
-		for (const dep of deps) {
-			queue.push({
-				type: RBDT_RESOLVE,
-				context,
-				path: dep
-			});
-			jobQueued = true;
-		}
-		if (!jobQueued) {
-			// queue won't call drain when no jobs are queue
-			queue.drain();
-		}
 	}
 
 	/**
@@ -1344,6 +1456,11 @@ class FileSystemInfo {
 	 * @returns {void}
 	 */
 	checkResolveResultsValid(resolveResults, callback) {
+		const {
+			resolveCjs,
+			resolveEsm,
+			resolveContext
+		} = this._createBuildDependenciesResolvers();
 		asyncLib.eachLimit(
 			resolveResults,
 			20,
@@ -1358,7 +1475,14 @@ class FileSystemInfo {
 						});
 						break;
 					case "f":
-						resolve(context, path, {}, (err, result) => {
+						resolveCjs(context, path, {}, (err, result) => {
+							if (err) return callback(err);
+							if (result !== expectedResult) return callback(INVALID);
+							callback();
+						});
+						break;
+					case "e":
+						resolveEsm(context, path, {}, (err, result) => {
 							if (err) return callback(err);
 							if (result !== expectedResult) return callback(INVALID);
 							callback();
@@ -1604,7 +1728,7 @@ class FileSystemInfo {
 								if (err) {
 									if (this.logger) {
 										this.logger.debug(
-											`Error snapshotting file timestamp hash combination of ${path}: ${err}`
+											`Error snapshotting file timestamp hash combination of ${path}: ${err.stack}`
 										);
 									}
 									jobError();
@@ -1632,7 +1756,7 @@ class FileSystemInfo {
 								if (err) {
 									if (this.logger) {
 										this.logger.debug(
-											`Error snapshotting file hash of ${path}: ${err}`
+											`Error snapshotting file hash of ${path}: ${err.stack}`
 										);
 									}
 									jobError();
@@ -1662,7 +1786,7 @@ class FileSystemInfo {
 								if (err) {
 									if (this.logger) {
 										this.logger.debug(
-											`Error snapshotting file timestamp of ${path}: ${err}`
+											`Error snapshotting file timestamp of ${path}: ${err.stack}`
 										);
 									}
 									jobError();
@@ -1698,7 +1822,7 @@ class FileSystemInfo {
 								if (err) {
 									if (this.logger) {
 										this.logger.debug(
-											`Error snapshotting context timestamp hash combination of ${path}: ${err}`
+											`Error snapshotting context timestamp hash combination of ${path}: ${err.stack}`
 										);
 									}
 									jobError();
@@ -1726,7 +1850,7 @@ class FileSystemInfo {
 								if (err) {
 									if (this.logger) {
 										this.logger.debug(
-											`Error snapshotting context hash of ${path}: ${err}`
+											`Error snapshotting context hash of ${path}: ${err.stack}`
 										);
 									}
 									jobError();
@@ -1756,7 +1880,7 @@ class FileSystemInfo {
 								if (err) {
 									if (this.logger) {
 										this.logger.debug(
-											`Error snapshotting context timestamp of ${path}: ${err}`
+											`Error snapshotting context timestamp of ${path}: ${err.stack}`
 										);
 									}
 									jobError();
@@ -1789,7 +1913,7 @@ class FileSystemInfo {
 						if (err) {
 							if (this.logger) {
 								this.logger.debug(
-									`Error snapshotting missing timestamp of ${path}: ${err}`
+									`Error snapshotting missing timestamp of ${path}: ${err.stack}`
 								);
 							}
 							jobError();
@@ -1816,7 +1940,7 @@ class FileSystemInfo {
 					if (err) {
 						if (this.logger) {
 							this.logger.debug(
-								`Error snapshotting managed item ${path}: ${err}`
+								`Error snapshotting managed item ${path}: ${err.stack}`
 							);
 						}
 						jobError();
@@ -2346,6 +2470,11 @@ class FileSystemInfo {
 					this._fileHashes.set(path, null);
 					return callback(null, null);
 				}
+				if (err.code === "ERR_FS_FILE_TOO_LARGE") {
+					this.logger.warn(`Ignoring ${path} for hashing as it's very large`);
+					this._fileHashes.set(path, "too large");
+					return callback(null, "too large");
+				}
 				return callback(err);
 			}
 
@@ -2405,7 +2534,7 @@ class FileSystemInfo {
 	}
 
 	_readContextTimestamp(path, callback) {
-		this.fs.readdir(path, (err, files) => {
+		this.fs.readdir(path, (err, _files) => {
 			if (err) {
 				if (err.code === "ENOENT") {
 					this._contextTimestamps.set(path, null);
@@ -2414,7 +2543,7 @@ class FileSystemInfo {
 				}
 				return callback(err);
 			}
-			files = files
+			const files = /** @type {string[]} */ (_files)
 				.map(file => file.normalize("NFC"))
 				.filter(file => !/^\./.test(file))
 				.sort();
@@ -2501,7 +2630,7 @@ class FileSystemInfo {
 	}
 
 	_readContextHash(path, callback) {
-		this.fs.readdir(path, (err, files) => {
+		this.fs.readdir(path, (err, _files) => {
 			if (err) {
 				if (err.code === "ENOENT") {
 					this._contextHashes.set(path, null);
@@ -2509,7 +2638,7 @@ class FileSystemInfo {
 				}
 				return callback(err);
 			}
-			files = files
+			const files = /** @type {string[]} */ (_files)
 				.map(file => file.normalize("NFC"))
 				.filter(file => !/^\./.test(file))
 				.sort();
@@ -2624,7 +2753,9 @@ class FileSystemInfo {
 				return callback(err);
 			}
 			const set = new Set(
-				elements.map(element => join(this.fs, path, element))
+				/** @type {string[]} */ (elements).map(element =>
+					join(this.fs, path, element)
+				)
 			);
 			callback(null, set);
 		});