webpack 5.10.0 → 5.11.0

package/lib/serialization/BinaryMiddleware.js

@@ -10,31 +10,36 @@ const SerializerMiddleware = require("./SerializerMiddleware");
 /** @typedef {import("./types").BufferSerializableType} BufferSerializableType */
 /** @typedef {import("./types").PrimitiveSerializableType} PrimitiveSerializableType */
 
+/* eslint-disable no-loop-func */
+
 /*
 Format:
 
 File -> Section*
 
 Section -> NullsSection |
+					 BooleansSection |
 					 F64NumbersSection |
 					 I32NumbersSection |
 					 I8NumbersSection |
 					 ShortStringSection |
 					 StringSection |
 					 BufferSection |
-					 BooleanSection |
 					 NopSection
 
 
 
-NullsSection -> NullsSectionHeaderByte
+NullsSection ->
+	NullHeaderByte | Null2HeaderByte | Null3HeaderByte |
+	Nulls8HeaderByte 0xnn (n:count - 4) |
+	Nulls32HeaderByte n:ui32 (n:count - 260) |
+BooleansSection -> TrueHeaderByte | FalseHeaderByte | BooleansSectionHeaderByte BooleansCountAndBitsByte
 F64NumbersSection -> F64NumbersSectionHeaderByte f64*
 I32NumbersSection -> I32NumbersSectionHeaderByte i32*
 I8NumbersSection -> I8NumbersSectionHeaderByte i8*
-ShortStringSection -> ShortStringSectionHeaderByte utf8-byte*
+ShortStringSection -> ShortStringSectionHeaderByte ascii-byte*
 StringSection -> StringSectionHeaderByte i32:length utf8-byte*
 BufferSection -> BufferSectionHeaderByte i32:length byte*
-BooleanSection -> TrueHeaderByte | FalseHeaderByte
 NopSection --> NopSectionHeaderByte
 
 ShortStringSectionHeaderByte -> 0b1nnn_nnnn (n:length)
@@ -44,6 +49,13 @@ I32NumbersSectionHeaderByte -> 0b010n_nnnn (n:count - 1)
 I8NumbersSectionHeaderByte -> 0b011n_nnnn (n:count - 1)
 
 NullsSectionHeaderByte -> 0b0001_nnnn (n:count - 1)
+BooleansCountAndBitsByte ->
+	0b0000_1xxx (count = 3) |
+	0b0001_xxxx (count = 4) |
+	0b001x_xxxx (count = 5) |
+	0b01xx_xxxx (count = 6) |
+	0b1nnn_nnnn (n:count - 7, 7 <= count <= 133)
+	0xff n:ui32 (n:count, 134 <= count < 2^32)
 
 StringSectionHeaderByte -> 0b0000_1110
 BufferSectionHeaderByte -> 0b0000_1111
@@ -58,20 +70,26 @@ RawNumber -> n (n <= 10)
 const LAZY_HEADER = 0x0b;
 const TRUE_HEADER = 0x0c;
 const FALSE_HEADER = 0x0d;
-const STRING_HEADER = 0x0e;
-const BUFFER_HEADER = 0x0f;
-const NULLS_HEADER = 0x10;
+const BOOLEANS_HEADER = 0x0e;
+const NULL_HEADER = 0x10;
+const NULL2_HEADER = 0x11;
+const NULL3_HEADER = 0x12;
+const NULLS8_HEADER = 0x13;
+const NULLS32_HEADER = 0x14;
+const NULL_AND_I8_HEADER = 0x15;
+const NULL_AND_I32_HEADER = 0x16;
+const NULL_AND_TRUE_HEADER = 0x17;
+const NULL_AND_FALSE_HEADER = 0x18;
+const STRING_HEADER = 0x1e;
+const BUFFER_HEADER = 0x1f;
 const I8_HEADER = 0x60;
 const I32_HEADER = 0x40;
 const F64_HEADER = 0x20;
 const SHORT_STRING_HEADER = 0x80;
 
-/** Uplift high-order bits */
-const NULLS_HEADER_MASK = 0xf0;
 /** Uplift high-order bits */
 const NUMBERS_HEADER_MASK = 0xe0;
 const NUMBERS_COUNT_MASK = 0x1f; // 0b0001_1111
-const NULLS_COUNT_MASK = 0x0f; // 0b0000_1111
 const SHORT_STRING_LENGTH_MASK = 0x7f; // 0b0111_1111
 
 const HEADER_SIZE = 1;
@@ -198,51 +216,73 @@ class BinaryMiddleware extends SerializerMiddleware {
 					case "function": {
 						if (!SerializerMiddleware.isLazy(thing))
 							throw new Error("Unexpected function " + thing);
-						/** @type {SerializedType[0]} */
-						const serializedData = SerializerMiddleware.getLazySerializedValue(
+						/** @type {SerializedType | (() => SerializedType)} */
+						let serializedData = SerializerMiddleware.getLazySerializedValue(
 							thing
 						);
-						if (serializedData !== undefined) {
-							if (typeof serializedData === "function") {
-								flush();
-								buffers.push(serializedData);
+						if (serializedData === undefined) {
+							if (SerializerMiddleware.isLazy(thing, this)) {
+								const data = this._serialize(thing(), context);
+								SerializerMiddleware.setLazySerializedValue(thing, data);
+								serializedData = data;
 							} else {
-								serializeData(serializedData);
-								allocate(5);
-								writeU8(LAZY_HEADER);
-								writeU32(serializedData.length);
+								serializedData = SerializerMiddleware.serializeLazy(
+									thing,
+									data => this._serialize(data, context)
+								);
 							}
-						} else if (SerializerMiddleware.isLazy(thing, this)) {
-							/** @type {SerializedType} */
-							const data = BinaryMiddleware.optimizeSerializedData(
-								this._serialize(thing(), context)
-							);
-							SerializerMiddleware.setLazySerializedValue(thing, data);
-							serializeData(data);
-							allocate(5);
-							writeU8(LAZY_HEADER);
-							writeU32(data.length);
-						} else {
+						}
+						if (typeof serializedData === "function") {
 							flush();
-							buffers.push(
-								SerializerMiddleware.serializeLazy(thing, data =>
-									this._serialize(data, context)
-								)
-							);
+							buffers.push(serializedData);
+						} else {
+							const lengths = [];
+							for (const item of serializedData) {
+								let last;
+								if (typeof item === "function") {
+									lengths.push(0);
+								} else if (item.length === 0) {
+									// ignore
+								} else if (
+									lengths.length > 0 &&
+									(last = lengths[lengths.length - 1]) !== 0
+								) {
+									const remaining = 0xffffffff - last;
+									if (remaining >= item.length) {
+										lengths[lengths.length - 1] += item.length;
+									} else {
+										lengths.push(item.length - remaining);
+										lengths[lengths.length - 2] = 0xffffffff;
+									}
+								} else {
+									lengths.push(item.length);
+								}
+							}
+							allocate(5 + lengths.length * 4);
+							writeU8(LAZY_HEADER);
+							writeU32(lengths.length);
+							for (const l of lengths) {
+								writeU32(l);
+							}
+							for (const item of serializedData) {
+								flush();
+								buffers.push(item);
+							}
 						}
 						break;
 					}
 					case "string": {
 						const len = Buffer.byteLength(thing);
-						if (len >= 128) {
+						if (len >= 128 || len !== thing.length) {
 							allocate(len + HEADER_SIZE + I32_SIZE);
 							writeU8(STRING_HEADER);
 							writeU32(len);
+							currentBuffer.write(thing, currentPosition);
 						} else {
 							allocate(len + HEADER_SIZE);
 							writeU8(SHORT_STRING_HEADER | len);
+							currentBuffer.write(thing, currentPosition, "latin1");
 						}
-						currentBuffer.write(thing, currentPosition);
 						currentPosition += len;
 						break;
 					}
@@ -309,20 +349,111 @@ class BinaryMiddleware extends SerializerMiddleware {
 						i--;
 						break;
 					}
-					case "boolean":
-						allocate(HEADER_SIZE);
-						writeU8(thing === true ? TRUE_HEADER : FALSE_HEADER);
+					case "boolean": {
+						let lastByte = thing === true ? 1 : 0;
+						const bytes = [];
+						let count = 1;
+						let n;
+						for (n = 1; n < 0xffffffff && i + n < data.length; n++) {
+							const item = data[i + n];
+							if (typeof item !== "boolean") break;
+							const pos = count & 0x7;
+							if (pos === 0) {
+								bytes.push(lastByte);
+								lastByte = item === true ? 1 : 0;
+							} else if (item === true) {
+								lastByte |= 1 << pos;
+							}
+							count++;
+						}
+						i += count - 1;
+						if (count === 1) {
+							allocate(HEADER_SIZE);
+							writeU8(lastByte === 1 ? TRUE_HEADER : FALSE_HEADER);
+						} else if (count === 2) {
+							allocate(HEADER_SIZE * 2);
+							writeU8(lastByte & 1 ? TRUE_HEADER : FALSE_HEADER);
+							writeU8(lastByte & 2 ? TRUE_HEADER : FALSE_HEADER);
+						} else if (count <= 6) {
+							allocate(HEADER_SIZE + I8_SIZE);
+							writeU8(BOOLEANS_HEADER);
+							writeU8((1 << count) | lastByte);
+						} else if (count <= 133) {
+							allocate(HEADER_SIZE + I8_SIZE * bytes.length + I8_SIZE);
+							writeU8(BOOLEANS_HEADER);
+							writeU8(0x80 | (count - 7));
+							for (const byte of bytes) writeU8(byte);
+							writeU8(lastByte);
+						} else {
+							allocate(HEADER_SIZE + I8_SIZE * bytes.length + I8_SIZE);
+							writeU8(BOOLEANS_HEADER);
+							writeU8(0xff);
+							writeU32(count);
+							for (const byte of bytes) writeU8(byte);
+							writeU8(lastByte);
+						}
 						break;
+					}
 					case "object": {
 						if (thing === null) {
 							let n;
-							for (n = 1; n < 16 && i + n < data.length; n++) {
+							for (n = 1; n < 0x100000104 && i + n < data.length; n++) {
 								const item = data[i + n];
 								if (item !== null) break;
 							}
-							allocate(HEADER_SIZE);
-							writeU8(NULLS_HEADER | (n - 1));
 							i += n - 1;
+							if (n === 1) {
+								if (i + 1 < data.length) {
+									const next = data[i + 1];
+									if (next === true) {
+										allocate(HEADER_SIZE);
+										writeU8(NULL_AND_TRUE_HEADER);
+										i++;
+									} else if (next === false) {
+										allocate(HEADER_SIZE);
+										writeU8(NULL_AND_FALSE_HEADER);
+										i++;
+									} else if (typeof next === "number") {
+										const type = identifyNumber(next);
+										if (type === 0) {
+											allocate(HEADER_SIZE + I8_SIZE);
+											writeU8(NULL_AND_I8_HEADER);
+											currentBuffer.writeInt8(next, currentPosition);
+											currentPosition += I8_SIZE;
+											i++;
+										} else if (type === 1) {
+											allocate(HEADER_SIZE + I32_SIZE);
+											writeU8(NULL_AND_I32_HEADER);
+											currentBuffer.writeInt32LE(next, currentPosition);
+											currentPosition += I32_SIZE;
+											i++;
+										} else {
+											allocate(HEADER_SIZE);
+											writeU8(NULL_HEADER);
+										}
+									} else {
+										allocate(HEADER_SIZE);
+										writeU8(NULL_HEADER);
+									}
+								} else {
+									allocate(HEADER_SIZE);
+									writeU8(NULL_HEADER);
+								}
+							} else if (n === 2) {
+								allocate(HEADER_SIZE);
+								writeU8(NULL2_HEADER);
+							} else if (n === 3) {
+								allocate(HEADER_SIZE);
+								writeU8(NULL3_HEADER);
+							} else if (n < 260) {
+								allocate(HEADER_SIZE + I8_SIZE);
+								writeU8(NULLS8_HEADER);
+								writeU8(n - 4);
+							} else {
+								allocate(HEADER_SIZE + I32_SIZE);
+								writeU8(NULLS32_HEADER);
+								writeU32(n - 260);
+							}
 						} else if (Buffer.isBuffer(thing)) {
 							allocate(HEADER_SIZE + I32_SIZE, true);
 							writeU8(BUFFER_HEADER);
@@ -409,6 +540,31 @@ class BinaryMiddleware extends SerializerMiddleware {
 			checkOverflow();
 			return res;
 		};
+		/**
+		 * Reads up to n bytes
+		 * @param {number} n amount of bytes to read
+		 * @returns {Buffer} buffer with bytes
+		 */
+		const readUpTo = n => {
+			if (!currentIsBuffer) {
+				throw new Error(
+					currentBuffer === null
+						? "Unexpected end of stream"
+						: "Unexpected lazy element in stream"
+				);
+			}
+			const rem = currentBuffer.length - currentPosition;
+			if (rem < n) {
+				n = rem;
+			}
+			const res = /** @type {Buffer} */ (currentBuffer).slice(
+				currentPosition,
+				currentPosition + n
+			);
+			currentPosition += n;
+			checkOverflow();
+			return res;
+		};
 		const readU8 = () => {
 			if (!currentIsBuffer) {
 				throw new Error(
@@ -431,117 +587,292 @@ class BinaryMiddleware extends SerializerMiddleware {
 		const readU32 = () => {
 			return read(I32_SIZE).readUInt32LE(0);
 		};
-
-		/** @type {DeserializedType} */
-		const result = [];
-		while (currentBuffer !== null) {
-			if (typeof currentBuffer === "function") {
-				result.push(
-					SerializerMiddleware.deserializeLazy(currentBuffer, data =>
-						this._deserialize(data, context)
-					)
-				);
-				currentDataItem++;
-				currentBuffer =
-					currentDataItem < data.length ? data[currentDataItem] : null;
-				currentIsBuffer = Buffer.isBuffer(currentBuffer);
-				continue;
+		const readBits = (data, n) => {
+			let mask = 1;
+			while (n !== 0) {
+				result.push((data & mask) !== 0);
+				mask = mask << 1;
+				n--;
 			}
-			const header = readU8();
+		};
+		const dispatchTable = Array.from({ length: 256 }).map((_, header) => {
 			switch (header) {
-				case LAZY_HEADER: {
-					const count = readU32();
-					const start = result.length - count;
-					const data = /** @type {SerializedType} */ (result.slice(start));
-					result.length = start;
-					result.push(
-						SerializerMiddleware.createLazy(
-							memorize(() => this._deserialize(data, context)),
-							this,
-							undefined,
-							data
-						)
-					);
-					break;
-				}
-				case BUFFER_HEADER: {
-					const len = readU32();
-					result.push(read(len));
-					break;
-				}
+				case LAZY_HEADER:
+					return () => {
+						const count = readU32();
+						const lengths = Array.from({ length: count }).map(() => readU32());
+						const content = [];
+						for (let l of lengths) {
+							if (l === 0) {
+								if (typeof currentBuffer !== "function") {
+									throw new Error("Unexpected non-lazy element in stream");
+								}
+								content.push(currentBuffer);
+								currentDataItem++;
+								currentBuffer =
+									currentDataItem < data.length ? data[currentDataItem] : null;
+								currentIsBuffer = Buffer.isBuffer(currentBuffer);
+							} else {
+								do {
+									const buf = readUpTo(l);
+									l -= buf.length;
+									content.push(buf);
+								} while (l > 0);
+							}
+						}
+						result.push(
+							SerializerMiddleware.createLazy(
+								memorize(() => this._deserialize(content, context)),
+								this,
+								undefined,
+								content
+							)
+						);
+					};
+				case BUFFER_HEADER:
+					return () => {
+						const len = readU32();
+						result.push(read(len));
+					};
 				case TRUE_HEADER:
-					result.push(true);
-					break;
+					return () => result.push(true);
 				case FALSE_HEADER:
-					result.push(false);
-					break;
-				case STRING_HEADER: {
-					const len = readU32();
-					const buf = read(len);
-					result.push(buf.toString());
-					break;
-				}
+					return () => result.push(false);
+				case NULL3_HEADER:
+					return () => result.push(null, null, null);
+				case NULL2_HEADER:
+					return () => result.push(null, null);
+				case NULL_HEADER:
+					return () => result.push(null);
+				case NULL_AND_TRUE_HEADER:
+					return () => result.push(null, true);
+				case NULL_AND_FALSE_HEADER:
+					return () => result.push(null, false);
+				case NULL_AND_I8_HEADER:
+					return () => {
+						if (currentIsBuffer) {
+							result.push(
+								null,
+								/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
+							);
+							currentPosition += I8_SIZE;
+							checkOverflow();
+						} else {
+							result.push(null, read(I8_SIZE).readInt8(0));
+						}
+					};
+				case NULL_AND_I32_HEADER:
+					return () => {
+						result.push(null);
+						if (isInCurrentBuffer(I32_SIZE)) {
+							result.push(
+								/** @type {Buffer} */ (currentBuffer).readInt32LE(
+									currentPosition
+								)
+							);
+							currentPosition += I32_SIZE;
+							checkOverflow();
+						} else {
+							result.push(read(I32_SIZE).readInt32LE(0));
+						}
+					};
+				case NULLS8_HEADER:
+					return () => {
+						const len = readU8() + 4;
+						for (let i = 0; i < len; i++) {
+							result.push(null);
+						}
+					};
+				case NULLS32_HEADER:
+					return () => {
+						const len = readU32() + 260;
+						for (let i = 0; i < len; i++) {
+							result.push(null);
+						}
+					};
+				case BOOLEANS_HEADER:
+					return () => {
+						const innerHeader = readU8();
+						if ((innerHeader & 0xf0) === 0) {
+							readBits(innerHeader, 3);
+						} else if ((innerHeader & 0xe0) === 0) {
+							readBits(innerHeader, 4);
+						} else if ((innerHeader & 0xc0) === 0) {
+							readBits(innerHeader, 5);
+						} else if ((innerHeader & 0x80) === 0) {
+							readBits(innerHeader, 6);
+						} else if (innerHeader !== 0xff) {
+							let count = (innerHeader & 0x7f) + 7;
+							while (count > 8) {
+								readBits(readU8(), 8);
+								count -= 8;
+							}
+							readBits(readU8(), count);
+						} else {
+							let count = readU32();
+							while (count > 8) {
+								readBits(readU8(), 8);
+								count -= 8;
+							}
+							readBits(readU8(), count);
+						}
+					};
+				case STRING_HEADER:
+					return () => {
+						const len = readU32();
+						if (isInCurrentBuffer(len)) {
+							result.push(
+								currentBuffer.toString(
+									undefined,
+									currentPosition,
+									currentPosition + len
+								)
+							);
+							currentPosition += len;
+							checkOverflow();
+						} else {
+							result.push(read(len).toString());
+						}
+					};
+				case SHORT_STRING_HEADER:
+					return () => result.push("");
+				case SHORT_STRING_HEADER | 1:
+					return () => {
+						if (currentIsBuffer) {
+							result.push(
+								currentBuffer.toString(
+									"latin1",
+									currentPosition,
+									currentPosition + 1
+								)
+							);
+							currentPosition++;
+							checkOverflow();
+						} else {
+							result.push(read(1).toString("latin1"));
+						}
+					};
+				case I8_HEADER:
+					return () => {
+						if (currentIsBuffer) {
+							result.push(
+								/** @type {Buffer} */ (currentBuffer).readInt8(currentPosition)
+							);
+							currentPosition++;
+							checkOverflow();
+						} else {
+							result.push(read(1).readInt8(0));
+						}
+					};
 				default:
 					if (header <= 10) {
-						result.push(header);
+						return () => result.push(header);
 					} else if ((header & SHORT_STRING_HEADER) === SHORT_STRING_HEADER) {
 						const len = header & SHORT_STRING_LENGTH_MASK;
-						const buf = read(len);
-						result.push(buf.toString());
+						return () => {
+							if (isInCurrentBuffer(len)) {
+								result.push(
+									currentBuffer.toString(
+										"latin1",
+										currentPosition,
+										currentPosition + len
+									)
+								);
+								currentPosition += len;
+								checkOverflow();
+							} else {
+								result.push(read(len).toString("latin1"));
+							}
+						};
 					} else if ((header & NUMBERS_HEADER_MASK) === F64_HEADER) {
 						const len = (header & NUMBERS_COUNT_MASK) + 1;
-						const need = F64_SIZE * len;
-						if (isInCurrentBuffer(need)) {
-							for (let i = 0; i < len; i++) {
-								result.push(currentBuffer.readDoubleLE(currentPosition));
-								currentPosition += F64_SIZE;
-							}
-							checkOverflow();
-						} else {
-							const buf = read(need);
-							for (let i = 0; i < len; i++) {
-								result.push(buf.readDoubleLE(i * F64_SIZE));
+						return () => {
+							const need = F64_SIZE * len;
+							if (isInCurrentBuffer(need)) {
+								for (let i = 0; i < len; i++) {
+									result.push(
+										/** @type {Buffer} */ (currentBuffer).readDoubleLE(
+											currentPosition
+										)
+									);
+									currentPosition += F64_SIZE;
+								}
+								checkOverflow();
+							} else {
+								const buf = read(need);
+								for (let i = 0; i < len; i++) {
+									result.push(buf.readDoubleLE(i * F64_SIZE));
+								}
 							}
-						}
+						};
 					} else if ((header & NUMBERS_HEADER_MASK) === I32_HEADER) {
 						const len = (header & NUMBERS_COUNT_MASK) + 1;
-						const need = I32_SIZE * len;
-						if (isInCurrentBuffer(need)) {
-							for (let i = 0; i < len; i++) {
-								result.push(currentBuffer.readInt32LE(currentPosition));
-								currentPosition += I32_SIZE;
-							}
-							checkOverflow();
-						} else {
-							const buf = read(need);
-							for (let i = 0; i < len; i++) {
-								result.push(buf.readInt32LE(i * I32_SIZE));
+						return () => {
+							const need = I32_SIZE * len;
+							if (isInCurrentBuffer(need)) {
+								for (let i = 0; i < len; i++) {
+									result.push(
+										/** @type {Buffer} */ (currentBuffer).readInt32LE(
+											currentPosition
+										)
+									);
+									currentPosition += I32_SIZE;
+								}
+								checkOverflow();
+							} else {
+								const buf = read(need);
+								for (let i = 0; i < len; i++) {
+									result.push(buf.readInt32LE(i * I32_SIZE));
+								}
 							}
-						}
+						};
 					} else if ((header & NUMBERS_HEADER_MASK) === I8_HEADER) {
 						const len = (header & NUMBERS_COUNT_MASK) + 1;
-						const need = I8_SIZE * len;
-						if (isInCurrentBuffer(need)) {
-							for (let i = 0; i < len; i++) {
-								result.push(currentBuffer.readInt8(currentPosition));
-								currentPosition += I8_SIZE;
-							}
-							checkOverflow();
-						} else {
-							const buf = read(need);
-							for (let i = 0; i < len; i++) {
-								result.push(buf.readInt8(i * I8_SIZE));
+						return () => {
+							const need = I8_SIZE * len;
+							if (isInCurrentBuffer(need)) {
+								for (let i = 0; i < len; i++) {
+									result.push(
+										/** @type {Buffer} */ (currentBuffer).readInt8(
+											currentPosition
+										)
+									);
+									currentPosition += I8_SIZE;
+								}
+								checkOverflow();
+							} else {
+								const buf = read(need);
+								for (let i = 0; i < len; i++) {
+									result.push(buf.readInt8(i * I8_SIZE));
+								}
 							}
-						}
-					} else if ((header & NULLS_HEADER_MASK) === NULLS_HEADER) {
-						const len = (header & NULLS_COUNT_MASK) + 1;
-						for (let i = 0; i < len; i++) {
-							result.push(null);
-						}
+						};
 					} else {
-						throw new Error(`Unexpected header byte 0x${header.toString(16)}`);
+						return () => {
+							throw new Error(
+								`Unexpected header byte 0x${header.toString(16)}`
+							);
+						};
 					}
-					break;
+			}
+		});
+
+		/** @type {DeserializedType} */
+		const result = [];
+		while (currentBuffer !== null) {
+			if (typeof currentBuffer === "function") {
+				result.push(
+					SerializerMiddleware.deserializeLazy(currentBuffer, data =>
+						this._deserialize(data, context)
+					)
+				);
+				currentDataItem++;
+				currentBuffer =
+					currentDataItem < data.length ? data[currentDataItem] : null;
+				currentIsBuffer = Buffer.isBuffer(currentBuffer);
+			} else {
+				const header = readU8();
+				dispatchTable[header]();
 			}
 		}
 		return result;