webpack 5.59.0 → 5.94.0

package/lib/serialization/FileMiddleware.js

@@ -19,6 +19,7 @@ const memoize = require("../util/memoize");
 const SerializerMiddleware = require("./SerializerMiddleware");
 
 /** @typedef {typeof import("../util/Hash")} Hash */
+/** @typedef {import("../util/fs").IStats} IStats */
 /** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
 /** @typedef {import("./types").BufferSerializableType} BufferSerializableType */
 
@@ -40,6 +41,8 @@ Section -> Buffer
 
 // "wpc" + 1 in little-endian
 const VERSION = 0x01637077;
+const WRITE_LIMIT_TOTAL = 0x7fff0000;
+const WRITE_LIMIT_CHUNK = 511 * 1024 * 1024;
 
 /**
  * @param {Buffer[]} buffers buffers
@@ -55,39 +58,39 @@ const hashForName = (buffers, hashFunction) => {
 const COMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
 const DECOMPRESSION_CHUNK_SIZE = 100 * 1024 * 1024;
 
+/** @type {function(Buffer, number, number): void} */
 const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
 	? (buf, value, offset) => {
 			buf.writeBigUInt64LE(BigInt(value), offset);
-	  }
+		}
 	: (buf, value, offset) => {
 			const low = value % 0x100000000;
 			const high = (value - low) / 0x100000000;
 			buf.writeUInt32LE(low, offset);
 			buf.writeUInt32LE(high, offset + 4);
-	  };
+		};
 
+/** @type {function(Buffer, number): void} */
 const readUInt64LE = Buffer.prototype.readBigUInt64LE
-	? (buf, offset) => {
-			return Number(buf.readBigUInt64LE(offset));
-	  }
+	? (buf, offset) => Number(buf.readBigUInt64LE(offset))
 	: (buf, offset) => {
 			const low = buf.readUInt32LE(offset);
 			const high = buf.readUInt32LE(offset + 4);
 			return high * 0x100000000 + low;
-	  };
+		};
 
 /**
- * @typedef {Object} SerializeResult
+ * @typedef {object} SerializeResult
  * @property {string | false} name
  * @property {number} size
- * @property {Promise=} backgroundJob
+ * @property {Promise<any>=} backgroundJob
  */
 
 /**
  * @param {FileMiddleware} middleware this
  * @param {BufferSerializableType[] | Promise<BufferSerializableType[]>} data data to be serialized
  * @param {string | boolean} name file base name
- * @param {function(string | false, Buffer[]): Promise<void>} writeFile writes a file
+ * @param {function(string | false, Buffer[], number): Promise<void>} writeFile writes a file
  * @param {string | Hash} hashFunction hash function to use
  * @returns {Promise<SerializeResult>} resulting file pointer and promise
  */
@@ -102,8 +105,8 @@ const serialize = async (
 	const processedData = [];
 	/** @type {WeakMap<SerializeResult, function(): any | Promise<any>>} */
 	const resultToLazy = new WeakMap();
-	/** @type {Buffer[]} */
-	let lastBuffers = undefined;
+	/** @type {Buffer[] | undefined} */
+	let lastBuffers;
 	for (const item of await data) {
 		if (typeof item === "function") {
 			if (!SerializerMiddleware.isLazy(item))
@@ -161,9 +164,8 @@ const serialize = async (
 	const backgroundJobs = [];
 	const resolvedData = (
 		await Promise.all(
-			/** @type {Promise<Buffer[] | Buffer | SerializeResult>[]} */ (
-				processedData
-			)
+			/** @type {Promise<Buffer[] | Buffer | SerializeResult>[]} */
+			(processedData)
 		)
 	).map(item => {
 		if (Array.isArray(item) || Buffer.isBuffer(item)) return item;
@@ -179,6 +181,7 @@ const serialize = async (
 		SerializerMiddleware.setLazySerializedValue(lazy, buf);
 		return buf;
 	});
+	/** @type {number[]} */
 	const lengths = [];
 	for (const item of resolvedData) {
 		if (Array.isArray(item)) {
@@ -192,7 +195,7 @@ const serialize = async (
 		} else if (item) {
 			lengths.push(-item.length);
 		} else {
-			throw new Error("Unexpected falsy value in resolved data " + item);
+			throw new Error(`Unexpected falsy value in resolved data ${item}`);
 		}
 	}
 	const header = Buffer.allocUnsafe(8 + lengths.length * 4);
@@ -201,6 +204,7 @@ const serialize = async (
 	for (let i = 0; i < lengths.length; i++) {
 		header.writeInt32LE(lengths[i], 8 + i * 4);
 	}
+	/** @type {Buffer[]} */
 	const buf = [header];
 	for (const item of resolvedData) {
 		if (Array.isArray(item)) {
@@ -212,9 +216,9 @@ const serialize = async (
 	if (name === true) {
 		name = hashForName(buf, hashFunction);
 	}
-	backgroundJobs.push(writeFile(name, buf));
 	let size = 0;
 	for (const b of buf) size += b.length;
+	backgroundJobs.push(writeFile(name, buf, size));
 	return {
 		size,
 		name,
@@ -233,18 +237,21 @@ const serialize = async (
  */
 const deserialize = async (middleware, name, readFile) => {
 	const contents = await readFile(name);
-	if (contents.length === 0) throw new Error("Empty file " + name);
+	if (contents.length === 0) throw new Error(`Empty file ${name}`);
 	let contentsIndex = 0;
 	let contentItem = contents[0];
 	let contentItemLength = contentItem.length;
 	let contentPosition = 0;
-	if (contentItemLength === 0) throw new Error("Empty file " + name);
+	if (contentItemLength === 0) throw new Error(`Empty file ${name}`);
 	const nextContent = () => {
 		contentsIndex++;
 		contentItem = contents[contentsIndex];
 		contentItemLength = contentItem.length;
 		contentPosition = 0;
 	};
+	/**
+	 * @param {number} n number of bytes to ensure
+	 */
 	const ensureData = n => {
 		if (contentPosition === contentItemLength) {
 			nextContent();
@@ -272,18 +279,28 @@ const deserialize = async (middleware, name, readFile) => {
 			contentPosition = 0;
 		}
 	};
+	/**
+	 * @returns {number} value value
+	 */
 	const readUInt32LE = () => {
 		ensureData(4);
 		const value = contentItem.readUInt32LE(contentPosition);
 		contentPosition += 4;
 		return value;
 	};
+	/**
+	 * @returns {number} value value
+	 */
 	const readInt32LE = () => {
 		ensureData(4);
 		const value = contentItem.readInt32LE(contentPosition);
 		contentPosition += 4;
 		return value;
 	};
+	/**
+	 * @param {number} l length
+	 * @returns {Buffer} buffer
+	 */
 	const readSlice = l => {
 		ensureData(l);
 		if (contentPosition === 0 && contentItemLength === l) {
@@ -361,18 +378,16 @@ const deserialize = async (middleware, name, readFile) => {
 					length -= l;
 					contentPosition = contentItemLength;
 				}
+			} else if (length >= contentItemLength) {
+				result.push(contentItem);
+				length -= contentItemLength;
+				contentPosition = contentItemLength;
 			} else {
-				if (length >= contentItemLength) {
-					result.push(contentItem);
-					length -= contentItemLength;
-					contentPosition = contentItemLength;
-				} else {
-					result.push(
-						Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
-					);
-					contentPosition += length;
-					length = 0;
-				}
+				result.push(
+					Buffer.from(contentItem.buffer, contentItem.byteOffset, length)
+				);
+				contentPosition += length;
+				length = 0;
 			}
 			while (length > 0) {
 				nextContent();
@@ -393,6 +408,8 @@ const deserialize = async (middleware, name, readFile) => {
 	return result;
 };
 
+/** @typedef {{ filename: string, extension?: string }} FileMiddlewareContext */
+
 /**
  * @typedef {BufferSerializableType[]} DeserializedType
  * @typedef {true} SerializedType
@@ -408,9 +425,10 @@ class FileMiddleware extends SerializerMiddleware {
 		this.fs = fs;
 		this._hashFunction = hashFunction;
 	}
+
 	/**
 	 * @param {DeserializedType} data data
-	 * @param {Object} context context object
+	 * @param {object} context context object
 	 * @returns {SerializedType|Promise<SerializedType>} serialized data
 	 */
 	serialize(data, context) {
@@ -422,43 +440,92 @@ class FileMiddleware extends SerializerMiddleware {
 				// It's important that we don't touch existing files during serialization
 				// because serialize may read existing files (when deserializing)
 				const allWrittenFiles = new Set();
-				const writeFile = async (name, content) => {
+				/**
+				 * @param {string | false} name name
+				 * @param {Buffer[]} content content
+				 * @param {number} size size
+				 * @returns {Promise<void>}
+				 */
+				const writeFile = async (name, content, size) => {
 					const file = name
 						? join(this.fs, filename, `../${name}${extension}`)
 						: filename;
-					await new Promise((resolve, reject) => {
-						let stream = this.fs.createWriteStream(file + "_");
-						let compression;
-						if (file.endsWith(".gz")) {
-							compression = createGzip({
-								chunkSize: COMPRESSION_CHUNK_SIZE,
-								level: zConstants.Z_BEST_SPEED
-							});
-						} else if (file.endsWith(".br")) {
-							compression = createBrotliCompress({
-								chunkSize: COMPRESSION_CHUNK_SIZE,
-								params: {
-									[zConstants.BROTLI_PARAM_MODE]: zConstants.BROTLI_MODE_TEXT,
-									[zConstants.BROTLI_PARAM_QUALITY]: 2,
-									[zConstants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: true,
-									[zConstants.BROTLI_PARAM_SIZE_HINT]: content.reduce(
-										(size, b) => size + b.length,
-										0
-									)
+					await new Promise(
+						/**
+						 * @param {(value?: undefined) => void} resolve resolve
+						 * @param {(reason?: Error | null) => void} reject reject
+						 */
+						(resolve, reject) => {
+							let stream = this.fs.createWriteStream(`${file}_`);
+							let compression;
+							if (file.endsWith(".gz")) {
+								compression = createGzip({
+									chunkSize: COMPRESSION_CHUNK_SIZE,
+									level: zConstants.Z_BEST_SPEED
+								});
+							} else if (file.endsWith(".br")) {
+								compression = createBrotliCompress({
+									chunkSize: COMPRESSION_CHUNK_SIZE,
+									params: {
+										[zConstants.BROTLI_PARAM_MODE]: zConstants.BROTLI_MODE_TEXT,
+										[zConstants.BROTLI_PARAM_QUALITY]: 2,
+										[zConstants.BROTLI_PARAM_DISABLE_LITERAL_CONTEXT_MODELING]: true,
+										[zConstants.BROTLI_PARAM_SIZE_HINT]: size
+									}
+								});
+							}
+							if (compression) {
+								pipeline(compression, stream, reject);
+								stream = compression;
+								stream.on("finish", () => resolve());
+							} else {
+								stream.on("error", err => reject(err));
+								stream.on("finish", () => resolve());
+							}
+							// split into chunks for WRITE_LIMIT_CHUNK size
+							/** @type {TODO[]} */
+							const chunks = [];
+							for (const b of content) {
+								if (b.length < WRITE_LIMIT_CHUNK) {
+									chunks.push(b);
+								} else {
+									for (let i = 0; i < b.length; i += WRITE_LIMIT_CHUNK) {
+										chunks.push(b.slice(i, i + WRITE_LIMIT_CHUNK));
+									}
 								}
-							});
-						}
-						if (compression) {
-							pipeline(compression, stream, reject);
-							stream = compression;
-							stream.on("finish", () => resolve());
-						} else {
-							stream.on("error", err => reject(err));
-							stream.on("finish", () => resolve());
+							}
+
+							const len = chunks.length;
+							let i = 0;
+							/**
+							 * @param {(Error | null)=} err err
+							 */
+							const batchWrite = err => {
+								// will be handled in "on" error handler
+								if (err) return;
+
+								if (i === len) {
+									stream.end();
+									return;
+								}
+
+								// queue up a batch of chunks up to the write limit
+								// end is exclusive
+								let end = i;
+								let sum = chunks[end++].length;
+								while (end < len) {
+									sum += chunks[end].length;
+									if (sum > WRITE_LIMIT_TOTAL) break;
+									end++;
+								}
+								while (i < end - 1) {
+									stream.write(chunks[i++]);
+								}
+								stream.write(chunks[i++], batchWrite);
+							};
+							batchWrite();
 						}
-						for (const b of content) stream.write(b);
-						stream.end();
-					});
+					);
 					if (name) allWrittenFiles.add(file);
 				};
 
@@ -468,10 +535,15 @@ class FileMiddleware extends SerializerMiddleware {
 							await backgroundJob;
 
 							// Rename the index file to disallow access during inconsistent file state
-							await new Promise(resolve =>
-								this.fs.rename(filename, filename + ".old", err => {
-									resolve();
-								})
+							await new Promise(
+								/**
+								 * @param {(value?: undefined) => void} resolve resolve
+								 */
+								resolve => {
+									this.fs.rename(filename, `${filename}.old`, err => {
+										resolve();
+									});
+								}
 							);
 
 							// update all written files
@@ -479,22 +551,35 @@ class FileMiddleware extends SerializerMiddleware {
 								Array.from(
 									allWrittenFiles,
 									file =>
-										new Promise((resolve, reject) => {
-											this.fs.rename(file + "_", file, err => {
-												if (err) return reject(err);
-												resolve();
-											});
-										})
+										new Promise(
+											/**
+											 * @param {(value?: undefined) => void} resolve resolve
+											 * @param {(reason?: Error | null) => void} reject reject
+											 * @returns {void}
+											 */
+											(resolve, reject) => {
+												this.fs.rename(`${file}_`, file, err => {
+													if (err) return reject(err);
+													resolve();
+												});
+											}
+										)
 								)
 							);
 
 							// As final step automatically update the index file to have a consistent pack again
-							await new Promise(resolve => {
-								this.fs.rename(filename + "_", filename, err => {
-									if (err) return reject(err);
-									resolve();
-								});
-							});
+							await new Promise(
+								/**
+								 * @param {(value?: undefined) => void} resolve resolve
+								 * @returns {void}
+								 */
+								resolve => {
+									this.fs.rename(`${filename}_`, filename, err => {
+										if (err) return reject(err);
+										resolve();
+									});
+								}
+							);
 							return /** @type {true} */ (true);
 						}
 					)
@@ -505,11 +590,15 @@ class FileMiddleware extends SerializerMiddleware {
 
 	/**
 	 * @param {SerializedType} data data
-	 * @param {Object} context context object
+	 * @param {object} context context object
 	 * @returns {DeserializedType|Promise<DeserializedType>} deserialized data
 	 */
 	deserialize(data, context) {
 		const { filename, extension = "" } = context;
+		/**
+		 * @param {string | boolean} name name
+		 * @returns {Promise<TODO>} result
+		 */
 		const readFile = name =>
 			new Promise((resolve, reject) => {
 				const file = name
@@ -520,10 +609,14 @@ class FileMiddleware extends SerializerMiddleware {
 						reject(err);
 						return;
 					}
-					let remaining = /** @type {number} */ (stats.size);
+					let remaining = /** @type {IStats} */ (stats).size;
+					/** @type {Buffer | undefined} */
 					let currentBuffer;
+					/** @type {number | undefined} */
 					let currentBufferUsed;
+					/** @type {any[]} */
 					const buf = [];
+					/** @type {import("zlib").Zlib & import("stream").Transform | undefined} */
 					let decompression;
 					if (file.endsWith(".gz")) {
 						decompression = createGunzip({
@@ -535,7 +628,8 @@ class FileMiddleware extends SerializerMiddleware {
 						});
 					}
 					if (decompression) {
-						let newResolve, newReject;
+						let newResolve;
+						let newReject;
 						resolve(
 							Promise.all([
 								new Promise((rs, rj) => {
@@ -552,11 +646,12 @@ class FileMiddleware extends SerializerMiddleware {
 						resolve = newResolve;
 						reject = newReject;
 					}
-					this.fs.open(file, "r", (err, fd) => {
+					this.fs.open(file, "r", (err, _fd) => {
 						if (err) {
 							reject(err);
 							return;
 						}
+						const fd = /** @type {number} */ (_fd);
 						const read = () => {
 							if (currentBuffer === undefined) {
 								currentBuffer = Buffer.allocUnsafeSlow(
@@ -569,8 +664,10 @@ class FileMiddleware extends SerializerMiddleware {
 								currentBufferUsed = 0;
 							}
 							let readBuffer = currentBuffer;
-							let readOffset = currentBufferUsed;
-							let readLength = currentBuffer.length - currentBufferUsed;
+							let readOffset = /** @type {number} */ (currentBufferUsed);
+							let readLength =
+								currentBuffer.length -
+								/** @type {number} */ (currentBufferUsed);
 							// values passed to fs.read must be valid int32 values
 							if (readOffset > 0x7fffffff) {
 								readBuffer = currentBuffer.slice(readOffset);
@@ -592,9 +689,13 @@ class FileMiddleware extends SerializerMiddleware {
 										});
 										return;
 									}
-									currentBufferUsed += bytesRead;
+									/** @type {number} */
+									(currentBufferUsed) += bytesRead;
 									remaining -= bytesRead;
-									if (currentBufferUsed === currentBuffer.length) {
+									if (
+										currentBufferUsed ===
+										/** @type {Buffer} */ (currentBuffer).length
+									) {
 										if (decompression) {
 											decompression.write(currentBuffer);
 										} else {

package/lib/serialization/MapObjectSerializer.js

@@ -4,25 +4,42 @@
 
 "use strict";
 
+/** @typedef {import("./ObjectMiddleware").ObjectDeserializerContext} ObjectDeserializerContext */
+/** @typedef {import("./ObjectMiddleware").ObjectSerializerContext} ObjectSerializerContext */
+
 class MapObjectSerializer {
-	serialize(obj, { write }) {
-		write(obj.size);
+	/**
+	 * @template K, V
+	 * @param {Map<K, V>} obj map
+	 * @param {ObjectSerializerContext} context context
+	 */
+	serialize(obj, context) {
+		context.write(obj.size);
 		for (const key of obj.keys()) {
-			write(key);
+			context.write(key);
 		}
 		for (const value of obj.values()) {
-			write(value);
+			context.write(value);
 		}
 	}
-	deserialize({ read }) {
-		let size = read();
+
+	/**
+	 * @template K, V
+	 * @param {ObjectDeserializerContext} context context
+	 * @returns {Map<K, V>} map
+	 */
+	deserialize(context) {
+		/** @type {number} */
+		const size = context.read();
+		/** @type {Map<K, V>} */
 		const map = new Map();
+		/** @type {K[]} */
 		const keys = [];
 		for (let i = 0; i < size; i++) {
-			keys.push(read());
+			keys.push(context.read());
 		}
 		for (let i = 0; i < size; i++) {
-			map.set(keys[i], read());
+			map.set(keys[i], context.read());
 		}
 		return map;
 	}

package/lib/serialization/NullPrototypeObjectSerializer.js

@@ -4,27 +4,45 @@
 
 "use strict";
 
+/** @typedef {import("./ObjectMiddleware").ObjectDeserializerContext} ObjectDeserializerContext */
+/** @typedef {import("./ObjectMiddleware").ObjectSerializerContext} ObjectSerializerContext */
+
 class NullPrototypeObjectSerializer {
-	serialize(obj, { write }) {
+	/**
+	 * @template {object} T
+	 * @param {T} obj null object
+	 * @param {ObjectSerializerContext} context context
+	 */
+	serialize(obj, context) {
+		/** @type {string[]} */
 		const keys = Object.keys(obj);
 		for (const key of keys) {
-			write(key);
+			context.write(key);
 		}
-		write(null);
+		context.write(null);
 		for (const key of keys) {
-			write(obj[key]);
+			context.write(obj[/** @type {keyof T} */ (key)]);
 		}
 	}
-	deserialize({ read }) {
+
+	/**
+	 * @template {object} T
+	 * @param {ObjectDeserializerContext} context context
+	 * @returns {T} null object
+	 */
+	deserialize(context) {
+		/** @type {T} */
 		const obj = Object.create(null);
+		/** @type {string[]} */
 		const keys = [];
-		let key = read();
+		/** @type {string | null} */
+		let key = context.read();
 		while (key !== null) {
 			keys.push(key);
-			key = read();
+			key = context.read();
 		}
 		for (const key of keys) {
-			obj[key] = read();
+			obj[/** @type {keyof T} */ (key)] = context.read();
 		}
 		return obj;
 	}