webpack 5.10.0 → 5.11.0

package/lib/serialization/FileMiddleware.js

@@ -4,6 +4,7 @@
 
 "use strict";
 
+const { constants } = require("buffer");
 const createHash = require("../util/createHash");
 const { dirname, join, mkdirp } = require("../util/fs");
 const memorize = require("../util/memorize");
@@ -28,14 +29,35 @@ Section -> Buffer
 
 */
 
-// "wpc" + 0 in little-endian
-const VERSION = 0x00637077;
+// "wpc" + 1 in little-endian
+const VERSION = 0x01637077;
 const hashForName = buffers => {
 	const hash = createHash("md4");
 	for (const buf of buffers) hash.update(buf);
 	return /** @type {string} */ (hash.digest("hex"));
 };
 
+const writeUInt64LE = Buffer.prototype.writeBigUInt64LE
+	? (buf, value, offset) => {
+			buf.writeBigUInt64LE(BigInt(value), offset);
+	  }
+	: (buf, value, offset) => {
+			const low = value % 0x100000000;
+			const high = (value - low) / 0x100000000;
+			buf.writeUInt32LE(low, offset);
+			buf.writeUInt32LE(high, offset + 4);
+	  };
+
+const readUInt64LE = Buffer.prototype.readBigUInt64LE
+	? (buf, offset) => {
+			return Number(buf.readBigUInt64LE(offset));
+	  }
+	: (buf, offset) => {
+			const low = buf.readUInt32LE(offset);
+			const high = buf.readUInt32LE(offset + 4);
+			return high * 0x100000000 + low;
+	  };
+
 /**
  * @typedef {Object} SerializeResult
  * @property {string | false} name
@@ -122,24 +144,29 @@ const serialize = async (middleware, data, name, writeFile) => {
 		// create pointer buffer from size and name
 		const name = /** @type {string} */ (item.name);
 		const nameBuffer = Buffer.from(name);
-		const buf = Buffer.allocUnsafe(4 + nameBuffer.length);
-		buf.writeUInt32LE(item.size, 0);
-		nameBuffer.copy(buf, 4, 0);
+		const buf = Buffer.allocUnsafe(8 + nameBuffer.length);
+		writeUInt64LE(buf, item.size, 0);
+		nameBuffer.copy(buf, 8, 0);
 		const lazy = resultToLazy.get(item);
 		SerializerMiddleware.setLazySerializedValue(lazy, buf);
 		return buf;
 	});
-	const lengths = resolvedData.map(item => {
+	const lengths = [];
+	for (const item of resolvedData) {
 		if (Array.isArray(item)) {
 			let l = 0;
 			for (const b of item) l += b.length;
-			return l;
+			while (l > 0x7fffffff) {
+				lengths.push(0x7fffffff);
+				l -= 0x7fffffff;
+			}
+			lengths.push(l);
 		} else if (item) {
-			return -item.length;
+			lengths.push(-item.length);
 		} else {
 			throw new Error("Unexpected falsy value in resolved data " + item);
 		}
-	});
+	}
 	const header = Buffer.allocUnsafe(8 + lengths.length * 4);
 	header.writeUInt32LE(VERSION, 0);
 	header.writeUInt32LE(lengths.length, 4);
@@ -173,45 +200,145 @@ const serialize = async (middleware, data, name, writeFile) => {
 /**
  * @param {FileMiddleware} middleware this
  * @param {string | false} name filename
- * @param {function(string | false): Promise<Buffer>} readFile read content of a file
+ * @param {function(string | false): Promise<Buffer[]>} readFile read content of a file
  * @returns {Promise<BufferSerializableType[]>} deserialized data
  */
 const deserialize = async (middleware, name, readFile) => {
-	const content = await readFile(name);
-	if (content.length === 0) throw new Error("Empty file " + name);
-	const version = content.readUInt32LE(0);
+	const contents = await readFile(name);
+	if (contents.length === 0) throw new Error("Empty file " + name);
+	let contentsIndex = 0;
+	let contentItem = contents[0];
+	let contentItemLength = contentItem.length;
+	let contentPosition = 0;
+	if (contentItemLength === 0) throw new Error("Empty file " + name);
+	const nextContent = () => {
+		contentsIndex++;
+		contentItem = contents[contentsIndex];
+		contentItemLength = contentItem.length;
+		contentPosition = 0;
+	};
+	const ensureData = n => {
+		if (contentPosition === contentItemLength) {
+			nextContent();
+		}
+		while (contentItemLength - contentPosition < n) {
+			const remaining = contentItem.slice(contentPosition);
+			let lengthFromNext = n - remaining.length;
+			const buffers = [remaining];
+			for (let i = contentsIndex + 1; i < contents.length; i++) {
+				const l = contents[i].length;
+				if (l > lengthFromNext) {
+					buffers.push(contents[i].slice(0, lengthFromNext));
+					contents[i] = contents[i].slice(lengthFromNext);
+					lengthFromNext = 0;
+					break;
+				} else {
+					buffers.push(contents[i]);
+					contentsIndex = i;
+					lengthFromNext -= l;
+				}
+			}
+			if (lengthFromNext > 0) throw new Error("Unexpected end of data");
+			contentItem = Buffer.concat(buffers, n);
+			contentItemLength = n;
+			contentPosition = 0;
+		}
+	};
+	const readUInt32LE = () => {
+		ensureData(4);
+		const value = contentItem.readUInt32LE(contentPosition);
+		contentPosition += 4;
+		return value;
+	};
+	const readInt32LE = () => {
+		ensureData(4);
+		const value = contentItem.readInt32LE(contentPosition);
+		contentPosition += 4;
+		return value;
+	};
+	const readSlice = l => {
+		ensureData(l);
+		if (contentPosition === 0 && contentItemLength === l) {
+			const result = contentItem;
+			if (contentsIndex + 1 < contents.length) {
+				nextContent();
+			} else {
+				contentPosition = l;
+			}
+			return result;
+		}
+		const result = contentItem.slice(contentPosition, contentPosition + l);
+		contentPosition += l;
+		// we clone the buffer here to allow the original content to be garbage collected
+		return l * 2 < contentItem.buffer.byteLength ? Buffer.from(result) : result;
+	};
+	const version = readUInt32LE();
 	if (version !== VERSION) {
 		throw new Error("Invalid file version");
 	}
-	const sectionCount = content.readUInt32LE(4);
+	const sectionCount = readUInt32LE();
 	const lengths = [];
 	for (let i = 0; i < sectionCount; i++) {
-		lengths.push(content.readInt32LE(8 + 4 * i));
+		lengths.push(readInt32LE());
 	}
-	let position = sectionCount * 4 + 8;
-	const result = lengths.map(length => {
-		const l = Math.abs(length);
-		const section = content.slice(position, position + l);
-		position += l;
+	const result = [];
+	for (let length of lengths) {
 		if (length < 0) {
-			// we clone the buffer here to allow the original content to be garbage collected
-			const clone = Buffer.from(section);
-			const size = section.readUInt32LE(0);
-			const nameBuffer = clone.slice(4);
+			const slice = readSlice(-length);
+			const size = Number(readUInt64LE(slice, 0));
+			const nameBuffer = slice.slice(8);
 			const name = nameBuffer.toString();
-			return SerializerMiddleware.createLazy(
-				memorize(() => deserialize(middleware, name, readFile)),
-				middleware,
-				{
-					name,
-					size
-				},
-				clone
+			result.push(
+				SerializerMiddleware.createLazy(
+					memorize(() => deserialize(middleware, name, readFile)),
+					middleware,
+					{
+						name,
+						size
+					},
+					slice
+				)
 			);
 		} else {
-			return section;
+			if (contentPosition === contentItemLength) {
+				nextContent();
+			} else if (contentPosition !== 0) {
+				if (length <= contentItemLength - contentPosition) {
+					result.push(
+						contentItem.slice(contentPosition, contentPosition + length)
+					);
+					contentPosition += length;
+					length = 0;
+				} else {
+					result.push(contentItem.slice(contentPosition));
+					length -= contentItemLength - contentPosition;
+					contentPosition = contentItemLength;
+				}
+			} else {
+				if (length >= contentItemLength) {
+					result.push(contentItem);
+					length -= contentItemLength;
+					contentPosition = contentItemLength;
+				} else {
+					result.push(contentItem.slice(0, length));
+					contentPosition += length;
+					length = 0;
+				}
+			}
+			while (length > 0) {
+				nextContent();
+				if (length >= contentItemLength) {
+					result.push(contentItem);
+					length -= contentItemLength;
+					contentPosition = contentItemLength;
+				} else {
+					result.push(contentItem.slice(0, length));
+					contentPosition += length;
+					length = 0;
+				}
+			}
 		}
-	});
+	}
 	return result;
 };
 
@@ -309,9 +436,72 @@ class FileMiddleware extends SerializerMiddleware {
 				const file = name
 					? join(this.fs, filename, `../${name}${extension}`)
 					: filename;
-				return this.fs.readFile(file, (err, content) => {
-					if (err) return reject(err);
-					resolve(content);
+				this.fs.stat(file, (err, stats) => {
+					if (err) {
+						reject(err);
+						return;
+					}
+					let remaining = stats.size;
+					let currentBuffer;
+					let currentBufferUsed;
+					const buf = [];
+					this.fs.open(file, "r", (err, fd) => {
+						if (err) {
+							reject(err);
+							return;
+						}
+						const read = () => {
+							if (currentBuffer === undefined) {
+								currentBuffer = Buffer.allocUnsafeSlow(
+									Math.min(constants.MAX_LENGTH, remaining)
+								);
+								currentBufferUsed = 0;
+							}
+							let readBuffer = currentBuffer;
+							let readOffset = currentBufferUsed;
+							let readLength = currentBuffer.length - currentBufferUsed;
+							if (readOffset > 0x7fffffff) {
+								readBuffer = currentBuffer.slice(readOffset);
+								readOffset = 0;
+							}
+							if (readLength > 0x7fffffff) {
+								readLength = 0x7fffffff;
+							}
+							this.fs.read(
+								fd,
+								readBuffer,
+								readOffset,
+								readLength,
+								null,
+								(err, bytesRead) => {
+									if (err) {
+										this.fs.close(fd, () => {
+											reject(err);
+										});
+										return;
+									}
+									currentBufferUsed += bytesRead;
+									remaining -= bytesRead;
+									if (currentBufferUsed === currentBuffer.length) {
+										buf.push(currentBuffer);
+										currentBuffer = undefined;
+										if (remaining === 0) {
+											this.fs.close(fd, err => {
+												if (err) {
+													reject(err);
+													return;
+												}
+												resolve(buf);
+											});
+											return;
+										}
+									}
+									read();
+								}
+							);
+						};
+						read();
+					});
 				});
 			});
 		return deserialize(this, false, readFile);

package/lib/serialization/ObjectMiddleware.js

@@ -4,6 +4,7 @@
 
 "use strict";
 
+const createHash = require("../util/createHash");
 const ArraySerializer = require("./ArraySerializer");
 const DateObjectSerializer = require("./DateObjectSerializer");
 const ErrorObjectSerializer = require("./ErrorObjectSerializer");
@@ -75,6 +76,12 @@ const setMapSize = (map, size) => {
 	}
 };
 
+const toHash = buffer => {
+	const hash = createHash("md4");
+	hash.update(buffer);
+	return /** @type {string} */ (hash.digest("latin1"));
+};
+
 const ESCAPE = null;
 const ESCAPE_ESCAPE_VALUE = null;
 const ESCAPE_END_OBJECT = true;
@@ -244,6 +251,68 @@ class ObjectMiddleware extends SerializerMiddleware {
 		const addReferenceable = item => {
 			referenceable.set(item, currentPos++);
 		};
+		const bufferDedupeMap = new Map();
+		const dedupeBuffer = buf => {
+			const len = buf.length;
+			const entry = bufferDedupeMap.get(len);
+			if (entry === undefined) {
+				bufferDedupeMap.set(len, buf);
+				return buf;
+			}
+			if (Buffer.isBuffer(entry)) {
+				if (len < 32) {
+					if (buf.equals(entry)) {
+						return entry;
+					}
+					bufferDedupeMap.set(len, [entry, buf]);
+					return buf;
+				} else {
+					const hash = toHash(entry);
+					const newMap = new Map();
+					newMap.set(hash, entry);
+					bufferDedupeMap.set(len, newMap);
+					const hashBuf = toHash(buf);
+					if (hash === hashBuf) {
+						return entry;
+					}
+					return buf;
+				}
+			} else if (Array.isArray(entry)) {
+				if (entry.length < 16) {
+					for (const item of entry) {
+						if (buf.equals(item)) {
+							return item;
+						}
+					}
+					entry.push(buf);
+					return buf;
+				} else {
+					const newMap = new Map();
+					const hash = toHash(buf);
+					let found;
+					for (const item of entry) {
+						const itemHash = toHash(item);
+						newMap.set(itemHash, item);
+						if (found === undefined && itemHash === hash) found = item;
+					}
+					bufferDedupeMap.set(len, newMap);
+					if (found === undefined) {
+						newMap.set(hash, buf);
+						return buf;
+					} else {
+						return found;
+					}
+				}
+			} else {
+				const hash = toHash(buf);
+				const item = entry.get(hash);
+				if (item !== undefined) {
+					return item;
+				}
+				entry.set(hash, buf);
+				return buf;
+			}
+		};
 		let currentPosTypeLookup = 0;
 		const objectTypeLookup = new Map();
 		const cycleStack = new Set();
@@ -339,6 +408,16 @@ class ObjectMiddleware extends SerializerMiddleware {
 			}
 
 			if (Buffer.isBuffer(item)) {
+				const alreadyUsedBuffer = dedupeBuffer(item);
+				if (alreadyUsedBuffer !== item) {
+					const ref = referenceable.get(alreadyUsedBuffer);
+					if (ref !== undefined) {
+						referenceable.set(item, ref);
+						result.push(ESCAPE, ref - currentPos);
+						return;
+					}
+					item = alreadyUsedBuffer;
+				}
 				addReferenceable(item);
 
 				result.push(item);
@@ -373,8 +452,8 @@ class ObjectMiddleware extends SerializerMiddleware {
 
 				addReferenceable(item);
 			} else if (typeof item === "string") {
-				if (item !== "") {
-					// empty strings are shorter when not emitting a reference (this saves 1 byte per empty string)
+				if (item.length > 1) {
+					// short strings are shorter when not emitting a reference (this saves 1 byte per empty string)
 					addReferenceable(item);
 				}
 
@@ -449,13 +528,13 @@ class ObjectMiddleware extends SerializerMiddleware {
 			throw new Error("Version mismatch, serializer changed");
 
 		let currentPos = 0;
-		const referenceable = [];
+		let referenceable = [];
 		const addReferenceable = item => {
 			referenceable.push(item);
 			currentPos++;
 		};
 		let currentPosTypeLookup = 0;
-		const objectTypeLookup = [];
+		let objectTypeLookup = [];
 		const result = [];
 		const ctx = {
 			read() {
@@ -558,7 +637,7 @@ class ObjectMiddleware extends SerializerMiddleware {
 					}
 				}
 			} else if (typeof item === "string") {
-				if (item !== "") {
+				if (item.length > 1) {
 					addReferenceable(item);
 				}
 
@@ -581,6 +660,11 @@ class ObjectMiddleware extends SerializerMiddleware {
 			result.push(decodeValue());
 		}
 
+		// Help the GC, as functions above might be cached in inline caches
+		referenceable = undefined;
+		objectTypeLookup = undefined;
+		data = undefined;
+
 		return result;
 	}
 }

package/lib/sharing/ConsumeSharedRuntimeModule.js

@@ -22,7 +22,7 @@ const {
 
 class ConsumeSharedRuntimeModule extends RuntimeModule {
 	constructor(runtimeRequirements) {
-		super("consumes", 10);
+		super("consumes", RuntimeModule.STAGE_ATTACH);
 		this._runtimeRequirements = runtimeRequirements;
 	}
 

package/lib/util/fs.js

@@ -16,6 +16,7 @@ const path = require("path");
 /** @typedef {function(NodeJS.ErrnoException=, Buffer|string=): void} BufferOrStringCallback */
 /** @typedef {function(NodeJS.ErrnoException=, string[]=): void} StringArrayCallback */
 /** @typedef {function(NodeJS.ErrnoException=, string=): void} StringCallback */
+/** @typedef {function(NodeJS.ErrnoException=, number=): void} NumberCallback */
 /** @typedef {function(NodeJS.ErrnoException=, NodeFsStats=): void} StatsCallback */
 /** @typedef {function((NodeJS.ErrnoException | Error)=, any=): void} ReadJsonCallback */
 
@@ -73,6 +74,9 @@ const path = require("path");
  * @typedef {Object} IntermediateFileSystemExtras
  * @property {function(string): void} mkdirSync
  * @property {function(string): import("fs").WriteStream} createWriteStream
+ * @property {function(string, string, NumberCallback): void} open
+ * @property {function(number, Buffer, number, number, number, NumberCallback): void} read
+ * @property {function(number, Callback): void} close
  * @property {function(string, string, Callback): void} rename
  */
 

package/lib/util/source.js

@@ -0,0 +1,61 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+
+"use strict";
+
+/** @typedef {import("webpack-sources").Source} Source */
+
+/** @type {WeakMap<Source, WeakMap<Source, boolean>>} */
+const equalityCache = new WeakMap();
+
+/**
+ * @param {Source} a a source
+ * @param {Source} b another source
+ * @returns {boolean} true, when both sources are equal
+ */
+const _isSourceEqual = (a, b) => {
+	// prefer .buffer(), it's called anyway during emit
+	/** @type {Buffer|string} */
+	let aSource = typeof a.buffer === "function" ? a.buffer() : a.source();
+	/** @type {Buffer|string} */
+	let bSource = typeof b.buffer === "function" ? b.buffer() : b.source();
+	if (aSource === bSource) return true;
+	if (typeof aSource === "string" && typeof bSource === "string") return false;
+	if (!Buffer.isBuffer(aSource)) aSource = Buffer.from(aSource, "utf-8");
+	if (!Buffer.isBuffer(bSource)) bSource = Buffer.from(bSource, "utf-8");
+	return aSource.equals(bSource);
+};
+
+/**
+ * @param {Source} a a source
+ * @param {Source} b another source
+ * @returns {boolean} true, when both sources are equal
+ */
+const isSourceEqual = (a, b) => {
+	if (a === b) return true;
+	const cache1 = equalityCache.get(a);
+	if (cache1 !== undefined) {
+		const result = cache1.get(b);
+		if (result !== undefined) return result;
+	}
+	const result = _isSourceEqual(a, b);
+	if (cache1 !== undefined) {
+		cache1.set(b, result);
+	} else {
+		const map = new WeakMap();
+		map.set(b, result);
+		equalityCache.set(a, map);
+	}
+	const cache2 = equalityCache.get(b);
+	if (cache2 !== undefined) {
+		cache2.set(a, result);
+	} else {
+		const map = new WeakMap();
+		map.set(a, result);
+		equalityCache.set(b, map);
+	}
+	return result;
+};
+exports.isSourceEqual = isSourceEqual;