querysub 0.2.0

package/src/-h-path-value-serialize/stringSerializer.ts

@@ -0,0 +1,176 @@
+import debugbreak from "debugbreak";
+import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
+import { asFloat64 } from "../buffers";
+
+export class StringSerialize {
+
+    // TIMING: About 1.6X SLOWER than JSON.stringify (HOWEVER, if you JSON.stringify+parse, stringify
+    //      only takes about 15% of the time, so... stringify time is mostly irrelevant).
+    //  - ALSO, uses less bytes than JSON.stringify, especially for small strings (JSON.stringify
+    //      has at least 3 bytes overhead per string, more if the strings contain `"`)
+    @measureFnc
+    public static serializeStrings(texts: string[]): Buffer {
+        let nullValues: {
+            index: number;
+            count: number;
+        }[] = [];
+        for (let i = 0; i < texts.length; i++) {
+            let text = texts[i];
+            if (text.includes("\0")) {
+                let nullCount = 0;
+                for (let i = 0; i < text.length; i++) {
+                    if (text[i] === "\0") {
+                        nullCount++;
+                    }
+                }
+                nullValues.push({
+                    index: i,
+                    count: nullCount,
+                });
+            }
+        }
+        function float64Buffer(values: number[]) {
+            return Buffer.from(new Float64Array(values).buffer);
+        }
+
+        return Buffer.concat([
+            float64Buffer([texts.length]),
+            float64Buffer([nullValues.length]),
+            float64Buffer(nullValues.map(v => v.index)),
+            float64Buffer(nullValues.map(v => v.count)),
+            Buffer.from(texts.join("\0")),
+        ]);
+    }
+
+    // TIMING: About 2X faster than JSON.parse
+    //  - UNLESS you HEAVILY use "\0", then 5x slower than JSON.parse
+    @measureFnc
+    public static deserializeStrings(buffer: Buffer): string[] {
+        let getObj = this.deserializeStringsLazy(buffer);
+        let strings: string[][] = [];
+        while (true) {
+            let nextStrings = getObj.getNextStrings();
+            if (!nextStrings) {
+                break;
+            }
+            strings.push(nextStrings);
+        }
+        // Unfortunate to have to .flat it, but... it's probably faster than parsing the strings?
+        let result = strings.flat();
+        return result;
+    }
+    public static deserializeStringsLazy(buffer: Buffer, readChunkSize = 1_000_000): {
+        getNextStrings(): string[] | undefined;
+    } {
+        let curReadPos = 0;
+        function readFloat64(count: number): number[] {
+            let results: number[] = [];
+            for (let i = 0; i < count; i++) {
+                results.push(buffer.readDoubleLE(curReadPos));
+                curReadPos += 8;
+            }
+            return results;
+        }
+        let auditCountRead = 0;
+        let count = readFloat64(1)[0];
+        let nullValueCount = readFloat64(1)[0];
+        let nullIndexes = readFloat64(nullValueCount);
+        let nullCounts = readFloat64(nullValueCount);
+        let indexToNullCount = new Map<number, number>();
+        for (let i = 0; i < nullValueCount; i++) {
+            indexToNullCount.set(nullIndexes[i], nullCounts[i]);
+        }
+        // "" splits into [""], so we need to check count to see if it is really [""], or actually [].
+        //      (otherwise "" is fine, as we "\0" join, it is just the zero count case that is tricky)
+        if (count === 0) {
+            return { getNextStrings() { return undefined; } };
+        }
+
+        let prevParts: string[] | undefined;
+
+        return { getNextStrings: measureWrap(getNextStrings) };
+        function getNextStrings() {
+            let currentEnd = curReadPos + readChunkSize;
+            if (currentEnd > buffer.length) currentEnd = buffer.length;
+            while (currentEnd < buffer.length && buffer[currentEnd] !== 0) {
+                currentEnd++;
+            }
+            if (curReadPos >= currentEnd && !prevParts) {
+                if (auditCountRead !== count) {
+                    throw new Error(`Expected to read ${count} strings, but only read ${auditCountRead}. Either the data is corrupted, or our code is bugged.`);
+                }
+                return undefined;
+            }
+            let text = buffer.toString("utf8", curReadPos, currentEnd);
+            curReadPos = currentEnd + 1;
+
+            let textParts = text.split("\0");
+            if (prevParts) {
+                textParts = prevParts.concat(textParts);
+                prevParts = undefined;
+            }
+
+            let outputParts: string[] = [];
+            let index = 0;
+            while (index < textParts.length) {
+                let partCount = (indexToNullCount.get(outputParts.length) ?? 0) + 1;
+                if (partCount === 1) {
+                    outputParts.push(textParts[index]);
+                } else {
+                    if (index + partCount > textParts.length) {
+                        prevParts = textParts.slice(index);
+                        break;
+                    } else {
+                        outputParts.push(textParts.slice(index, index + partCount).join("\0"));
+                    }
+                }
+                index += partCount;
+            }
+
+            auditCountRead += outputParts.length;
+            return outputParts;
+        }
+    }
+    public static deserializeStringsOld(buffer: Buffer): string[] {
+        let offset = 0;
+        function readFloat64(count: number): number[] {
+            let results: number[] = [];
+            for (let i = 0; i < count; i++) {
+                results.push(buffer.readDoubleLE(offset));
+                offset += 8;
+            }
+            return results;
+        }
+        let count = readFloat64(1)[0];
+        let nullValueCount = readFloat64(1)[0];
+        let nullIndexes = readFloat64(nullValueCount);
+        let nullCounts = readFloat64(nullValueCount);
+        let indexToNullCount = new Map<number, number>();
+        for (let i = 0; i < nullValueCount; i++) {
+            indexToNullCount.set(nullIndexes[i], nullCounts[i]);
+        }
+        // "" splits into [""], so we need to check count to see if it is really [""], or actually [].
+        //      (otherwise "" is fine, as we "\0" join, it is just the zero count case that is tricky)
+        if (count === 0) {
+            return [];
+        }
+
+        let text = buffer.toString("utf8", offset);
+
+        let textParts = text.split("\0");
+
+        let outputParts: string[] = [];
+        let index = 0;
+        while (index < textParts.length) {
+            let partCount = (indexToNullCount.get(outputParts.length) ?? 0) + 1;
+            if (partCount === 1) {
+                outputParts.push(textParts[index]);
+            } else {
+                outputParts.push(textParts.slice(index, index + partCount).join("\0"));
+            }
+            index += partCount;
+        }
+
+        return outputParts;
+    }
+}

package/src/0-path-value-core/LoggingClient.tsx

@@ -0,0 +1,24 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+
+class LoggingClientBase {
+    public async onMessage(config: {
+        type: "error" | "warn";
+        message: string;
+        context: string;
+    }) {
+        let { type, message, context } = config;
+        if (type === "error") {
+            console.error(message, context);
+        } else {
+            console.warn(message, context);
+        }
+    }
+}
+
+export const LoggingClient = SocketFunction.register(
+    "LoggingClient-8f0484e0-f755-49cb-bb52-0b7693eef5f4",
+    new LoggingClientBase(),
+    () => ({
+        onMessage: {}
+    })
+);