webpack 5.34.0 → 5.35.0

package/lib/FileSystemInfo.js

@@ -1249,7 +1249,7 @@ class FileSystemInfo {
 						return callback();
 					}
 					resolveResults.set(key, undefined);
-					resolveContext(context, path, resolverContext, (err, result) => {
+					resolveContext(context, path, resolverContext, (err, _, result) => {
 						if (err) {
 							if (expected === false) {
 								resolveResults.set(key, false);
@@ -1259,11 +1259,12 @@ class FileSystemInfo {
 							err.message += `\nwhile resolving '${path}' in ${context} to a directory`;
 							return callback(err);
 						}
-						resolveResults.set(key, result);
+						const resultPath = result.path;
+						resolveResults.set(key, resultPath);
 						push({
 							type: RBDT_DIRECTORY,
 							context: undefined,
-							path: result,
+							path: resultPath,
 							expected: undefined,
 							issuer: job
 						});
@@ -1276,14 +1277,16 @@ class FileSystemInfo {
 						return callback();
 					}
 					resolveResults.set(key, undefined);
-					resolve(context, path, resolverContext, (err, result) => {
+					resolve(context, path, resolverContext, (err, _, result) => {
 						if (typeof expected === "string") {
-							if (result === expected) {
-								resolveResults.set(key, result);
+							if (!err && result && result.path === expected) {
+								resolveResults.set(key, result.path);
 							} else {
 								invalidResolveResults.add(key);
 								this.logger.warn(
-									`Resolving '${path}' in ${context} for build dependencies doesn't lead to expected result '${expected}', but to '${result}' instead. Resolving dependencies are ignored for this path.\n${pathToString(
+									`Resolving '${path}' in ${context} for build dependencies doesn't lead to expected result '${expected}', but to '${
+										err || (result && result.path)
+									}' instead. Resolving dependencies are ignored for this path.\n${pathToString(
 										job
 									)}`
 								);
@@ -1300,11 +1303,12 @@ class FileSystemInfo {
 								)}`;
 								return callback(err);
 							}
-							resolveResults.set(key, result);
+							const resultPath = result.path;
+							resolveResults.set(key, resultPath);
 							push({
 								type: RBDT_FILE,
 								context: undefined,
-								path: result,
+								path: resultPath,
 								expected: undefined,
 								issuer: job
 							});
@@ -1646,38 +1650,42 @@ class FileSystemInfo {
 				const [type, context, path] = key.split("\n");
 				switch (type) {
 					case "d":
-						resolveContext(context, path, {}, (err, result) => {
+						resolveContext(context, path, {}, (err, _, result) => {
 							if (expectedResult === false)
 								return callback(err ? undefined : INVALID);
 							if (err) return callback(err);
-							if (result !== expectedResult) return callback(INVALID);
+							const resultPath = result.path;
+							if (resultPath !== expectedResult) return callback(INVALID);
 							callback();
 						});
 						break;
 					case "f":
-						resolveCjs(context, path, {}, (err, result) => {
+						resolveCjs(context, path, {}, (err, _, result) => {
 							if (expectedResult === false)
 								return callback(err ? undefined : INVALID);
 							if (err) return callback(err);
-							if (result !== expectedResult) return callback(INVALID);
+							const resultPath = result.path;
+							if (resultPath !== expectedResult) return callback(INVALID);
 							callback();
 						});
 						break;
 					case "c":
-						resolveCjsAsChild(context, path, {}, (err, result) => {
+						resolveCjsAsChild(context, path, {}, (err, _, result) => {
 							if (expectedResult === false)
 								return callback(err ? undefined : INVALID);
 							if (err) return callback(err);
-							if (result !== expectedResult) return callback(INVALID);
+							const resultPath = result.path;
+							if (resultPath !== expectedResult) return callback(INVALID);
 							callback();
 						});
 						break;
 					case "e":
-						resolveEsm(context, path, {}, (err, result) => {
+						resolveEsm(context, path, {}, (err, _, result) => {
 							if (expectedResult === false)
 								return callback(err ? undefined : INVALID);
 							if (err) return callback(err);
-							if (result !== expectedResult) return callback(INVALID);
+							const resultPath = result.path;
+							if (resultPath !== expectedResult) return callback(INVALID);
 							callback();
 						});
 						break;

package/lib/WebpackOptionsApply.js

@@ -568,7 +568,8 @@ class WebpackOptionsApply extends OptionsApply {
 										"webpack.cache.PackFileCacheStrategy"
 									),
 									snapshot: options.snapshot,
-									maxAge: cacheOptions.maxAge
+									maxAge: cacheOptions.maxAge,
+									allowCollectingMemory: cacheOptions.allowCollectingMemory
 								}),
 								cacheOptions.idleTimeout,
 								cacheOptions.idleTimeoutForInitialStore

package/lib/cache/PackFileCacheStrategy.js

@@ -768,6 +768,14 @@ class PackContent {
 	}
 }
 
+const allowCollectingMemory = buf => {
+	const wasted = buf.buffer.byteLength - buf.byteLength;
+	if (wasted > 8192 && (wasted > 1048576 || wasted > buf.byteLength)) {
+		return Buffer.from(buf);
+	}
+	return buf;
+};
+
 class PackFileCacheStrategy {
 	/**
 	 * @param {Object} options options
@@ -779,6 +787,7 @@ class PackFileCacheStrategy {
 	 * @param {Logger} options.logger a logger
 	 * @param {SnapshotOptions} options.snapshot options regarding snapshotting
 	 * @param {number} options.maxAge max age of cache items
+	 * @param {boolean} options.allowCollectingMemory allow to collect unused memory created during deserialization
 	 */
 	constructor({
 		compiler,
@@ -788,7 +797,8 @@ class PackFileCacheStrategy {
 		version,
 		logger,
 		snapshot,
-		maxAge
+		maxAge,
+		allowCollectingMemory
 	}) {
 		this.fileSerializer = createFileSerializer(fs);
 		this.fileSystemInfo = new FileSystemInfo(fs, {
@@ -802,6 +812,7 @@ class PackFileCacheStrategy {
 		this.version = version;
 		this.logger = logger;
 		this.maxAge = maxAge;
+		this.allowCollectingMemory = allowCollectingMemory;
 		this.snapshot = snapshot;
 		/** @type {Set<string>} */
 		this.buildDependencies = new Set();
@@ -845,7 +856,10 @@ class PackFileCacheStrategy {
 			.deserialize(null, {
 				filename: `${cacheLocation}/index.pack`,
 				extension: ".pack",
-				logger
+				logger,
+				retainedBuffer: this.allowCollectingMemory
+					? allowCollectingMemory
+					: undefined
 			})
 			.catch(err => {
 				if (err.code !== "ENOENT") {

package/lib/config/defaults.js

@@ -295,8 +295,9 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			D(cache, "store", "pack");
 			D(cache, "idleTimeout", 60000);
 			D(cache, "idleTimeoutForInitialStore", 0);
-			D(cache, "maxMemoryGenerations", development ? 10 : Infinity);
+			D(cache, "maxMemoryGenerations", development ? 5 : Infinity);
 			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
+			D(cache, "allowCollectingMemory", development);
 			D(cache.buildDependencies, "defaultWebpack", [
 				path.resolve(__dirname, "..") + path.sep
 			]);

package/lib/serialization/BinaryMiddleware.js

@@ -123,6 +123,12 @@ class BinaryMiddleware extends SerializerMiddleware {
 		return this._serialize(data, context);
 	}
 
+	_serializeLazy(fn, context) {
+		return SerializerMiddleware.serializeLazy(fn, data =>
+			this._serialize(data, context)
+		);
+	}
+
 	/**
 	 * @param {DeserializedType} data data
 	 * @param {Object} context context object
@@ -135,7 +141,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 		let leftOverBuffer = null;
 		let currentPosition = 0;
 		/** @type {BufferSerializableType[]} */
-		const buffers = [];
+		let buffers = [];
 		let buffersTotalLength = 0;
 		const allocate = bytesNeeded => {
 			if (currentBuffer !== null) {
@@ -204,10 +210,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 								SerializerMiddleware.setLazySerializedValue(thing, data);
 								serializedData = data;
 							} else {
-								serializedData = SerializerMiddleware.serializeLazy(
-									thing,
-									data => this._serialize(data, context)
-								);
+								serializedData = this._serializeLazy(thing, context);
 							}
 						}
 						if (typeof serializedData === "function") {
@@ -474,7 +477,13 @@ class BinaryMiddleware extends SerializerMiddleware {
 		};
 		serializeData(data);
 		flush();
-		return buffers;
+
+		// avoid leaking memory
+		currentBuffer = null;
+		leftOverBuffer = null;
+		const _buffers = buffers;
+		buffers = undefined;
+		return _buffers;
 	}
 
 	/**
@@ -486,6 +495,21 @@ class BinaryMiddleware extends SerializerMiddleware {
 		return this._deserialize(data, context);
 	}
 
+	_createLazyDeserialized(content, context) {
+		return SerializerMiddleware.createLazy(
+			memoize(() => this._deserialize(content, context)),
+			this,
+			undefined,
+			content
+		);
+	}
+
+	_deserializeLazy(fn, context) {
+		return SerializerMiddleware.deserializeLazy(fn, data =>
+			this._deserialize(data, context)
+		);
+	}
+
 	/**
 	 * @param {SerializedType} data data
 	 * @param {Object} context context object
@@ -497,6 +521,8 @@ class BinaryMiddleware extends SerializerMiddleware {
 		let currentIsBuffer = Buffer.isBuffer(currentBuffer);
 		let currentPosition = 0;
 
+		const retainedBuffer = context.retainedBuffer || (x => x);
+
 		const checkOverflow = () => {
 			if (currentPosition >= currentBuffer.length) {
 				currentPosition = 0;
@@ -610,23 +636,16 @@ class BinaryMiddleware extends SerializerMiddleware {
 								do {
 									const buf = readUpTo(l);
 									l -= buf.length;
-									content.push(buf);
+									content.push(retainedBuffer(buf));
 								} while (l > 0);
 							}
 						}
-						result.push(
-							SerializerMiddleware.createLazy(
-								memoize(() => this._deserialize(content, context)),
-								this,
-								undefined,
-								content
-							)
-						);
+						result.push(this._createLazyDeserialized(content, context));
 					};
 				case BUFFER_HEADER:
 					return () => {
 						const len = readU32();
-						result.push(read(len));
+						result.push(retainedBuffer(read(len)));
 					};
 				case TRUE_HEADER:
 					return () => result.push(true);
@@ -852,14 +871,10 @@ class BinaryMiddleware extends SerializerMiddleware {
 		});
 
 		/** @type {DeserializedType} */
-		const result = [];
+		let result = [];
 		while (currentBuffer !== null) {
 			if (typeof currentBuffer === "function") {
-				result.push(
-					SerializerMiddleware.deserializeLazy(currentBuffer, data =>
-						this._deserialize(data, context)
-					)
-				);
+				result.push(this._deserializeLazy(currentBuffer, context));
 				currentDataItem++;
 				currentBuffer =
 					currentDataItem < data.length ? data[currentDataItem] : null;
@@ -869,7 +884,11 @@ class BinaryMiddleware extends SerializerMiddleware {
 				dispatchTable[header]();
 			}
 		}
-		return result;
+
+		// avoid leaking memory in context
+		let _result = result;
+		result = undefined;
+		return _result;
 	}
 }
 

package/lib/util/comparators.js

@@ -34,9 +34,7 @@ const createCachedParameterizedComparator = fn => {
 		 * @param {T} b second item
 		 * @returns {-1|0|1} compare result
 		 */
-		const result = (a, b) => {
-			return fn(arg, a, b);
-		};
+		const result = fn.bind(null, arg);
 		map.set(arg, result);
 		return result;
 	};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "webpack",
-  "version": "5.34.0",
+  "version": "5.35.0",
   "author": "Tobias Koppers @sokra",
   "description": "Packs CommonJs/AMD modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jsx, es7, css, less, ... and your custom stuff.",
   "license": "MIT",