querysub 0.433.0 → 0.436.0

package/src/-h-path-value-serialize/stringSerializer.ts

@@ -1,176 +1,176 @@
-import debugbreak from "debugbreak";
-import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { asFloat64 } from "../buffers";
-
-export class StringSerialize {
-
-    // TIMING: About 1.6X SLOWER than JSON.stringify (HOWEVER, if you JSON.stringify+parse, stringify
-    //      only takes about 15% of the time, so... stringify time is mostly irrelevant).
-    //  - ALSO, uses less bytes than JSON.stringify, especially for small strings (JSON.stringify
-    //      has at least 3 bytes overhead per string, more if the strings contain `"`)
-    @measureFnc
-    public static serializeStrings(texts: string[]): Buffer {
-        let nullValues: {
-            index: number;
-            count: number;
-        }[] = [];
-        for (let i = 0; i < texts.length; i++) {
-            let text = texts[i];
-            if (text.includes("\0")) {
-                let nullCount = 0;
-                for (let i = 0; i < text.length; i++) {
-                    if (text[i] === "\0") {
-                        nullCount++;
-                    }
-                }
-                nullValues.push({
-                    index: i,
-                    count: nullCount,
-                });
-            }
-        }
-        function float64Buffer(values: number[]) {
-            return Buffer.from(new Float64Array(values).buffer);
-        }
-
-        return Buffer.concat([
-            float64Buffer([texts.length]),
-            float64Buffer([nullValues.length]),
-            float64Buffer(nullValues.map(v => v.index)),
-            float64Buffer(nullValues.map(v => v.count)),
-            Buffer.from(texts.join("\0")),
-        ]);
-    }
-
-    // TIMING: About 2X faster than JSON.parse
-    //  - UNLESS you HEAVILY use "\0", then 5x slower than JSON.parse
-    @measureFnc
-    public static deserializeStrings(buffer: Buffer): string[] {
-        let getObj = this.deserializeStringsLazy(buffer);
-        let strings: string[][] = [];
-        while (true) {
-            let nextStrings = getObj.getNextStrings();
-            if (!nextStrings) {
-                break;
-            }
-            strings.push(nextStrings);
-        }
-        // Unfortunate to have to .flat it, but... it's probably faster than parsing the strings?
-        let result = strings.flat();
-        return result;
-    }
-    public static deserializeStringsLazy(buffer: Buffer, readChunkSize = 1_000_000): {
-        getNextStrings(): string[] | undefined;
-    } {
-        let curReadPos = 0;
-        function readFloat64(count: number): number[] {
-            let results: number[] = [];
-            for (let i = 0; i < count; i++) {
-                results.push(buffer.readDoubleLE(curReadPos));
-                curReadPos += 8;
-            }
-            return results;
-        }
-        let auditCountRead = 0;
-        let count = readFloat64(1)[0];
-        let nullValueCount = readFloat64(1)[0];
-        let nullIndexes = readFloat64(nullValueCount);
-        let nullCounts = readFloat64(nullValueCount);
-        let indexToNullCount = new Map<number, number>();
-        for (let i = 0; i < nullValueCount; i++) {
-            indexToNullCount.set(nullIndexes[i], nullCounts[i]);
-        }
-        // "" splits into [""], so we need to check count to see if it is really [""], or actually [].
-        //      (otherwise "" is fine, as we "\0" join, it is just the zero count case that is tricky)
-        if (count === 0) {
-            return { getNextStrings() { return undefined; } };
-        }
-
-        let prevParts: string[] | undefined;
-
-        return { getNextStrings: measureWrap(getNextStrings) };
-        function getNextStrings() {
-            let currentEnd = curReadPos + readChunkSize;
-            if (currentEnd > buffer.length) currentEnd = buffer.length;
-            while (currentEnd < buffer.length && buffer[currentEnd] !== 0) {
-                currentEnd++;
-            }
-            if (curReadPos >= currentEnd && !prevParts) {
-                if (auditCountRead !== count) {
-                    throw new Error(`Expected to read ${count} strings, but only read ${auditCountRead}. Either the data is corrupted, or our code is bugged.`);
-                }
-                return undefined;
-            }
-            let text = buffer.toString("utf8", curReadPos, currentEnd);
-            curReadPos = currentEnd + 1;
-
-            let textParts = text.split("\0");
-            if (prevParts) {
-                textParts = prevParts.concat(textParts);
-                prevParts = undefined;
-            }
-
-            let outputParts: string[] = [];
-            let index = 0;
-            while (index < textParts.length) {
-                let partCount = (indexToNullCount.get(outputParts.length) ?? 0) + 1;
-                if (partCount === 1) {
-                    outputParts.push(textParts[index]);
-                } else {
-                    if (index + partCount > textParts.length) {
-                        prevParts = textParts.slice(index);
-                        break;
-                    } else {
-                        outputParts.push(textParts.slice(index, index + partCount).join("\0"));
-                    }
-                }
-                index += partCount;
-            }
-
-            auditCountRead += outputParts.length;
-            return outputParts;
-        }
-    }
-    public static deserializeStringsOld(buffer: Buffer): string[] {
-        let offset = 0;
-        function readFloat64(count: number): number[] {
-            let results: number[] = [];
-            for (let i = 0; i < count; i++) {
-                results.push(buffer.readDoubleLE(offset));
-                offset += 8;
-            }
-            return results;
-        }
-        let count = readFloat64(1)[0];
-        let nullValueCount = readFloat64(1)[0];
-        let nullIndexes = readFloat64(nullValueCount);
-        let nullCounts = readFloat64(nullValueCount);
-        let indexToNullCount = new Map<number, number>();
-        for (let i = 0; i < nullValueCount; i++) {
-            indexToNullCount.set(nullIndexes[i], nullCounts[i]);
-        }
-        // "" splits into [""], so we need to check count to see if it is really [""], or actually [].
-        //      (otherwise "" is fine, as we "\0" join, it is just the zero count case that is tricky)
-        if (count === 0) {
-            return [];
-        }
-
-        let text = buffer.toString("utf8", offset);
-
-        let textParts = text.split("\0");
-
-        let outputParts: string[] = [];
-        let index = 0;
-        while (index < textParts.length) {
-            let partCount = (indexToNullCount.get(outputParts.length) ?? 0) + 1;
-            if (partCount === 1) {
-                outputParts.push(textParts[index]);
-            } else {
-                outputParts.push(textParts.slice(index, index + partCount).join("\0"));
-            }
-            index += partCount;
-        }
-
-        return outputParts;
-    }
+import debugbreak from "debugbreak";
+import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { asFloat64 } from "../buffers";
+
+export class StringSerialize {
+
+    // TIMING: About 1.6X SLOWER than JSON.stringify (HOWEVER, if you JSON.stringify+parse, stringify
+    //      only takes about 15% of the time, so... stringify time is mostly irrelevant).
+    //  - ALSO, uses less bytes than JSON.stringify, especially for small strings (JSON.stringify
+    //      has at least 3 bytes overhead per string, more if the strings contain `"`)
+    @measureFnc
+    public static serializeStrings(texts: string[]): Buffer {
+        let nullValues: {
+            index: number;
+            count: number;
+        }[] = [];
+        for (let i = 0; i < texts.length; i++) {
+            let text = texts[i];
+            if (text.includes("\0")) {
+                let nullCount = 0;
+                for (let i = 0; i < text.length; i++) {
+                    if (text[i] === "\0") {
+                        nullCount++;
+                    }
+                }
+                nullValues.push({
+                    index: i,
+                    count: nullCount,
+                });
+            }
+        }
+        function float64Buffer(values: number[]) {
+            return Buffer.from(new Float64Array(values).buffer);
+        }
+
+        return Buffer.concat([
+            float64Buffer([texts.length]),
+            float64Buffer([nullValues.length]),
+            float64Buffer(nullValues.map(v => v.index)),
+            float64Buffer(nullValues.map(v => v.count)),
+            Buffer.from(texts.join("\0")),
+        ]);
+    }
+
+    // TIMING: About 2X faster than JSON.parse
+    //  - UNLESS you HEAVILY use "\0", then 5x slower than JSON.parse
+    @measureFnc
+    public static deserializeStrings(buffer: Buffer): string[] {
+        let getObj = this.deserializeStringsLazy(buffer);
+        let strings: string[][] = [];
+        while (true) {
+            let nextStrings = getObj.getNextStrings();
+            if (!nextStrings) {
+                break;
+            }
+            strings.push(nextStrings);
+        }
+        // Unfortunate to have to .flat it, but... it's probably faster than parsing the strings?
+        let result = strings.flat();
+        return result;
+    }
+    public static deserializeStringsLazy(buffer: Buffer, readChunkSize = 1_000_000): {
+        getNextStrings(): string[] | undefined;
+    } {
+        let curReadPos = 0;
+        function readFloat64(count: number): number[] {
+            let results: number[] = [];
+            for (let i = 0; i < count; i++) {
+                results.push(buffer.readDoubleLE(curReadPos));
+                curReadPos += 8;
+            }
+            return results;
+        }
+        let auditCountRead = 0;
+        let count = readFloat64(1)[0];
+        let nullValueCount = readFloat64(1)[0];
+        let nullIndexes = readFloat64(nullValueCount);
+        let nullCounts = readFloat64(nullValueCount);
+        let indexToNullCount = new Map<number, number>();
+        for (let i = 0; i < nullValueCount; i++) {
+            indexToNullCount.set(nullIndexes[i], nullCounts[i]);
+        }
+        // "" splits into [""], so we need to check count to see if it is really [""], or actually [].
+        //      (otherwise "" is fine, as we "\0" join, it is just the zero count case that is tricky)
+        if (count === 0) {
+            return { getNextStrings() { return undefined; } };
+        }
+
+        let prevParts: string[] | undefined;
+
+        return { getNextStrings: measureWrap(getNextStrings) };
+        function getNextStrings() {
+            let currentEnd = curReadPos + readChunkSize;
+            if (currentEnd > buffer.length) currentEnd = buffer.length;
+            while (currentEnd < buffer.length && buffer[currentEnd] !== 0) {
+                currentEnd++;
+            }
+            if (curReadPos >= currentEnd && !prevParts) {
+                if (auditCountRead !== count) {
+                    throw new Error(`Expected to read ${count} strings, but only read ${auditCountRead}. Either the data is corrupted, or our code is bugged.`);
+                }
+                return undefined;
+            }
+            let text = buffer.toString("utf8", curReadPos, currentEnd);
+            curReadPos = currentEnd + 1;
+
+            let textParts = text.split("\0");
+            if (prevParts) {
+                textParts = prevParts.concat(textParts);
+                prevParts = undefined;
+            }
+
+            let outputParts: string[] = [];
+            let index = 0;
+            while (index < textParts.length) {
+                let partCount = (indexToNullCount.get(outputParts.length) ?? 0) + 1;
+                if (partCount === 1) {
+                    outputParts.push(textParts[index]);
+                } else {
+                    if (index + partCount > textParts.length) {
+                        prevParts = textParts.slice(index);
+                        break;
+                    } else {
+                        outputParts.push(textParts.slice(index, index + partCount).join("\0"));
+                    }
+                }
+                index += partCount;
+            }
+
+            auditCountRead += outputParts.length;
+            return outputParts;
+        }
+    }
+    public static deserializeStringsOld(buffer: Buffer): string[] {
+        let offset = 0;
+        function readFloat64(count: number): number[] {
+            let results: number[] = [];
+            for (let i = 0; i < count; i++) {
+                results.push(buffer.readDoubleLE(offset));
+                offset += 8;
+            }
+            return results;
+        }
+        let count = readFloat64(1)[0];
+        let nullValueCount = readFloat64(1)[0];
+        let nullIndexes = readFloat64(nullValueCount);
+        let nullCounts = readFloat64(nullValueCount);
+        let indexToNullCount = new Map<number, number>();
+        for (let i = 0; i < nullValueCount; i++) {
+            indexToNullCount.set(nullIndexes[i], nullCounts[i]);
+        }
+        // "" splits into [""], so we need to check count to see if it is really [""], or actually [].
+        //      (otherwise "" is fine, as we "\0" join, it is just the zero count case that is tricky)
+        if (count === 0) {
+            return [];
+        }
+
+        let text = buffer.toString("utf8", offset);
+
+        let textParts = text.split("\0");
+
+        let outputParts: string[] = [];
+        let index = 0;
+        while (index < textParts.length) {
+            let partCount = (indexToNullCount.get(outputParts.length) ?? 0) + 1;
+            if (partCount === 1) {
+                outputParts.push(textParts[index]);
+            } else {
+                outputParts.push(textParts.slice(index, index + partCount).join("\0"));
+            }
+            index += partCount;
+        }
+
+        return outputParts;
+    }
 }