querysub 0.2.0

package/src/5-diagnostics/nodeMetadata.ts

@@ -0,0 +1,135 @@
+
+import { MaybePromise } from "socket-function/src/types";
+import { JSXFormatter } from "./GenericFormat";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { cache } from "socket-function/src/caching";
+import { TimeGrouper } from "./TimeGrouper";
+import { runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { timeInMinute } from "socket-function/src/misc";
+import { formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
+import os from "os";
+import { isNode } from "typesafecss";
+
+// Undefined means we infer the column
+// Null means the column is removed
+export type ColumnType = undefined | null | {
+    // Defaults to column name
+    title?: string;
+    formatter?: JSXFormatter;
+};
+export type RowType = {
+    [columnName: string]: unknown;
+};
+export type ColumnsType = { [columnName: string]: ColumnType };
+export type TableType<ColumnsT extends ColumnsType = ColumnsType> = {
+    columns: ColumnsT;
+    rows: { [columnName in keyof ColumnsT]: unknown }[];
+};
+
+export type ExtraMetadata = {
+    columnName: string;
+    column?: ColumnType;
+    getValue: () => MaybePromise<unknown>;
+};
+let extraMetadatas: ExtraMetadata[] = [];
+export function registerNodeMetadata(getter: ExtraMetadata) {
+    //console.log(`Registering metadata: ${getter.columnName}`);
+    extraMetadatas.push(getter);
+}
+
+let stateValues = cache((title: string, format: (value: number) => string): { (value: number): void } => {
+    let timeGrouper = new TimeGrouper();
+    function onValue(value: number) {
+        timeGrouper.onValueChanged(value);
+    }
+    registerNodeMetadata({
+        columnName: title,
+        column: {},
+        getValue() {
+            return timeGrouper.getStateSummary(format);
+        },
+    });
+    return onValue;
+});
+let eventValues = cache((title: string, format: (value: number) => string): { (value: number): void } => {
+    let timeGrouper = new TimeGrouper();
+    function onValue(value: number) {
+        timeGrouper.onValueChanged(value);
+    }
+    registerNodeMetadata({
+        columnName: title,
+        column: {},
+        getValue() {
+            return timeGrouper.getEventSummary(format);
+        },
+    });
+    return onValue;
+});
+
+export function logNodeStats(title: string, format: (value: number) => string): { (value: number): void } {
+    return eventValues(title, format);
+}
+export function logNodeStateStats(title: string, format: (value: number) => string): { (value: number): void } {
+    return stateValues(title, format);
+}
+
+if (isNode()) {
+    void runInfinitePollCallAtStart(timeInMinute, async () => {
+        // Get system memory usage
+
+        const free = os.freemem();
+        const total = os.totalmem();
+
+        let constrainedMemory = process.constrainedMemory();
+
+        logNodeStateStats("System|Memory Free", x => formatNumber(x) + "B")(free);
+        logNodeStateStats("System|Memory Free %", formatPercent)(free / total);
+        if (constrainedMemory !== undefined) {
+            logNodeStateStats("System|Constrained Memory", x => formatNumber(x) + "B")(constrainedMemory);
+        }
+
+        let usedCPU = process.cpuUsage();
+        logNodeStateStats("System|CPU User", formatTime)(usedCPU.user * 1000);
+        logNodeStateStats("System|CPU System", formatTime)(usedCPU.system * 1000);
+
+        let cpuCount = os.cpus().length;
+        logNodeStateStats("System|CPU Count", formatNumber)(cpuCount);
+
+        let load = os.loadavg();
+        logNodeStateStats("System|Average Free Cores", formatPercent)((1 - load[0]) * os.cpus().length);
+
+    });
+}
+
+registerNodeMetadata({
+    columnName: "System|Uptime",
+    column: { formatter: "timeSpan" },
+    getValue() {
+        return process.uptime() * 1000;
+    },
+});
+
+class NodeMetadataBase {
+    public async verifyAccess() {
+        return true;
+    }
+    public async getExtraMetadatas() {
+        return await Promise.all(extraMetadatas.map(async ({ columnName, column, getValue }) => {
+            return {
+                columnName,
+                column,
+                value: await getValue(),
+            };
+        }));
+    }
+}
+export const NodeMetadataController = SocketFunction.register(
+    "NodeMetadataController-8be51a16-27c2-449e-b044-dcef6c3efcaf",
+    new NodeMetadataBase(),
+    () => ({
+        getExtraMetadatas: {
+            hooks: [requiresNetworkTrustHook],
+        },
+    }),
+);

package/src/5-diagnostics/qreactDebug.tsx

@@ -0,0 +1,309 @@
+import { lazy } from "socket-function/src/caching";
+import { ExternalRenderClass, __INTERNAL__QRenderClass, getSchemaPrefix, getSourceVSCodeLink, qreact, triggerRerenderAll } from "../4-dom/qreact";
+import { blue, green, magenta, yellow } from "socket-function/src/formatting/logColors";
+import { getPathFromStr, getPathStr } from "../path";
+import { delay } from "socket-function/src/batching";
+import { logErrors } from "../errors";
+import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { Querysub } from "../4-querysub/QuerysubController";
+import { closeAllModals, showModal } from "./Modal";
+import { FullscreenModal } from "./FullscreenModal";
+import { css } from "typesafecss";
+import { Button } from "../library-components/Button";
+import { authorityStorage } from "../0-path-value-core/pathValueCore";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { PathValueProxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { InputLabel, InputLabelURL } from "../library-components/InputLabel";
+import { URLParam } from "../library-components/URLParam";
+import { hotReloadingGuard, isHotReloading, onHotReload } from "socket-function/hot/HotReloadController";
+
+// Map, so hot reloading doesn't break things
+let componentButtons = new Map<string, { title: string, callback: (component: ExternalRenderClass) => void }>();
+
+export function addComponentButton(config: {
+    title: string;
+    callback: (component: ExternalRenderClass) => void;
+}) {
+    if (!isHotReloading() && componentButtons.has(config.title)) {
+        throw new Error(`Component button with title ${config.title} already exists`);
+    }
+    componentButtons.set(config.title, config);
+}
+
+export const enableDebugComponents = lazy(function enableDebugComponents() {
+    console.log(magenta(`middleclick + shift on components to debug them`));
+    // mousedown is more reliable than click, because the click will be aborted if they middle click
+    //  in something with a scrollbar.
+    document.addEventListener("mousedown", function (e) {
+        // Middle click to debug
+        if (e.button === 1 && (e.altKey || e.shiftKey)) {
+            logErrors(triggerDebug(e.target as HTMLElement, e));
+        }
+    });
+
+    async function triggerDebug(target: HTMLElement, event: MouseEvent) {
+        const component = __INTERNAL__QRenderClass.getInstanceFromDOM(target);
+        if (!component) return;
+        event.preventDefault();
+        event.stopPropagation();
+
+        let prevInject = qreact.INJECT_LINE_NUMBERS;
+        try {
+            qreact.INJECT_LINE_NUMBERS = true;
+            triggerRerenderAll();
+            await clientWatcher.waitForTriggerFinished();
+        } finally {
+            qreact.INJECT_LINE_NUMBERS = prevInject;
+        }
+        // Get new target under the mouse
+        target = document.elementFromPoint(event.clientX, event.clientY) as HTMLElement;
+
+        let source = getSourceVSCodeLink(target);
+        if (event.shiftKey) {
+            let newComponent = __INTERNAL__QRenderClass.getInstanceFromDOM(target) || component;
+            showModal({ content: <WatchModal component={newComponent} /> });
+            return;
+        }
+
+        let ancestors = component.getDebugComponentStack().slice(1).reverse();
+        let schemaPrefix = getSchemaPrefix();
+        let schemaLength = getPathFromStr(schemaPrefix).length;
+        function encodePath(pathStr: string) {
+            let path = getPathFromStr(pathStr);
+            if (pathStr.startsWith(schemaPrefix)) {
+                let underPath = path.slice(schemaLength);
+                let componentId = Number(underPath[0]);
+                let component = __INTERNAL__QRenderClass.getInstanceAllowedUndefined(componentId);
+                if (component) {
+                    path = [component.debugName].concat(underPath.slice(1));
+                }
+            }
+            //  If paths are to other components, replace the prefix with the component debugName
+            return path.map(x => green(x)).join(".");
+        }
+
+        console.log(" ");
+        console.log(" ");
+        console.log(" ");
+        if (event.altKey) {
+            window.open(source);
+        }
+        // VSCode link
+        if (source) {
+            console.log(source);
+        }
+        console.group(`Component ${blue(component.debugName)}`);
+        console.groupCollapsed(`    Ancestors (${ancestors.length})`);
+        for (let ancestor of ancestors) {
+            console.log("        " + ancestor.debugName);
+            let source = ancestor.getVSCodeLink();
+            if (source) {
+                console.log(`            ${source}`);
+            }
+        }
+        console.groupEnd();
+        let lastWatches = component.renderWatcher!.lastWatches;
+
+        let watchPaths = Array.from(lastWatches.paths).filter(x => getPathFromStr(x).length > schemaLength);
+        console.groupCollapsed(`    Watches (${watchPaths.length}):`);
+        for (let key of watchPaths) {
+            console.log("        " + encodePath(key));
+        }
+        console.groupEnd();
+
+        let parentPaths = Array.from(lastWatches.parentPaths).filter(x => getPathFromStr(x).length > schemaLength);
+        if (parentPaths.length > 0) {
+            console.groupCollapsed(`    Parent Watches (${parentPaths.length}):`);
+            for (let key of parentPaths) {
+                if (getPathFromStr(key).length <= schemaLength) continue;
+                console.log("        " + encodePath(key));
+            }
+            console.groupEnd();
+        }
+
+        console.log(`    debugHolder = `, component);
+        (globalThis as any).debugHolder = component;
+        console.log(`    debugInstance`, component.instance);
+        (globalThis as any).debugInstance = component.instance;
+
+        console.groupEnd();
+    }
+});
+
+
+const pathFilter = new URLParam("pathfilter", "");
+class WatchModal extends qreact.Component<{
+    component: ExternalRenderClass;
+}> {
+    state = {
+        parentNavigate: 0,
+        pos: "fill" as "top" | "fill" | "bottom",
+    };
+    render() {
+        let component = this.props.component;
+        for (let i = 0; i < this.state.parentNavigate; i++) {
+            component = component.getParent() || component;
+        }
+        let lastWatches = component.renderWatcher.lastWatches;
+        let parent = component.getParent();
+        function nicerPath(path: string[]) {
+            if (path[1] === "PathFunctionRunner") {
+                path.splice(1, 1);
+            }
+            return path;
+        }
+        let pos = this.state.pos;
+        let filter = pathFilter.value.toLowerCase();
+        return (
+            <FullscreenModal
+                outerStyle={
+                    pos === "fill" && {}
+                    || pos === "top" && { height: "25vh", padding: 10 }
+                    || pos === "bottom" && { height: "25vh", padding: 10, top: undefined, bottom: 0 }
+                    || {}
+                }
+                style={
+                    pos === "fill" && {}
+                    || pos === "top" && { maxHeight: "100%" }
+                    || pos === "bottom" && { maxHeight: "100%" }
+                    || {}
+                }
+                onCancel={() => closeAllModals()}
+            >
+                <div class={css.hsl(0, 0, 100).vbox(6)}>
+                    <div class={css.hbox(10).fillWidth}>
+                        {component.debugName}
+                        {parent && <Button onClick={() => this.state.parentNavigate++}>
+                            Go to parent ({parent.debugName})
+                        </Button>}
+                        {this.state.parentNavigate > 0 && <Button onClick={() => this.state.parentNavigate--}>
+                            Child
+                        </Button>}
+                        <div class={css.marginAuto} />
+                        <Button onClick={() => this.state.pos = "top"}>
+                            Top
+                        </Button>
+                        <Button onClick={() => this.state.pos = "fill"}>
+                            Fill
+                        </Button>
+                        <Button onClick={() => this.state.pos = "bottom"}>
+                            Bottom
+                        </Button>
+                    </div>
+
+                    <div class={css.hbox(8)}>
+                        <Button onClick={() => component.instance.forceUpdate()}>
+                            Rerender
+                        </Button>
+                        {componentButtons.size > 0 && Array.from(componentButtons.values()).map(({ title, callback }) => (
+                            <Button onClick={() => callback(component)}>
+                                {title}
+                            </Button>
+                        ))}
+                    </div>
+
+
+                    <InputLabelURL hot label="Filter" url={pathFilter} />
+
+                    <h4>Parent Paths ({lastWatches.parentPaths.size})</h4>
+                    <div class={css.vbox(2)}>
+                        {Array.from(lastWatches.parentPaths).map(path =>
+                            <div class={css.hbox(4).wrap.button} onClick={() => {
+                                console.log(path);
+                                return navigator.clipboard.writeText(path);
+                            }}>
+                                {nicerPath(getPathFromStr(path)).map(x => <span class={css.pad2(3, 0).hsl(0, 0, 80)}>{x}</span>)}
+
+                                <span class={css.pad2(3, 2).hsl(180, 75, 75)}>({authorityStorage.getPathsFromParent(path)?.size})</span>
+                            </div>
+                        )}
+                    </div>
+                    <h4>Paths ({lastWatches.paths.size})</h4>
+                    {/* TODO: If we render a hierarchy, we will have space where the indent is, to show buttons. */}
+                    <div class={css.vbox(2).fillWidth}>
+                        {Array.from(lastWatches.paths).map(path => {
+                            if (filter && !path.toLowerCase().includes(filter)) {
+                                return undefined;
+                            }
+                            let value = pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(path));
+                            let valueStr = String(value);
+                            try {
+                                valueStr = String(JSON.stringify(value));
+                            } catch { }
+                            if (valueStr === "undefined" && typeof value === "function") {
+                                valueStr = String(value).slice(0, 100);
+                            }
+
+                            if (valueStr.length > 500) {
+                                valueStr = valueStr.slice(0, 500);
+                            }
+                            let pathArray = nicerPath(getPathFromStr(path));
+                            let breakOnWrites = PathValueProxyWatcher.BREAK_ON_WRITES.has(path);
+                            let breakOnReads = PathValueProxyWatcher.BREAK_ON_READS.has(path);
+                            let logOnRead = PathValueProxyWatcher.LOG_WRITES_INCLUDES.has(path);
+                            let fncBreak = PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.has(path);
+                            let selectedButton = css.hsl(120, 75, 75);
+                            return (
+                                <div class={css.hbox(10, 0).fillWidth.wrap}>
+                                    <button class={breakOnWrites && selectedButton || ""} onClick={() => {
+                                        if (breakOnWrites) {
+                                            PathValueProxyWatcher.BREAK_ON_WRITES.delete(path);
+                                        } else {
+                                            PathValueProxyWatcher.BREAK_ON_WRITES.add(path);
+                                        }
+                                        this.forceUpdate();
+                                    }}>
+                                        =
+                                    </button>
+                                    <button class={breakOnReads && selectedButton || ""} onClick={() => {
+                                        if (breakOnReads) {
+                                            PathValueProxyWatcher.BREAK_ON_READS.delete(path);
+                                        } else {
+                                            PathValueProxyWatcher.BREAK_ON_READS.add(path);
+                                        }
+                                        this.forceUpdate();
+                                    }}>
+                                        ===
+                                    </button>
+                                    <button class={logOnRead && selectedButton || ""} onClick={() => {
+                                        if (logOnRead) {
+                                            PathValueProxyWatcher.LOG_WRITES_INCLUDES.delete(path);
+                                        } else {
+                                            PathValueProxyWatcher.LOG_WRITES_INCLUDES.add(path);
+                                        }
+                                        this.forceUpdate();
+                                    }}>
+                                        Log
+                                    </button>
+                                    <button class={fncBreak && selectedButton || ""} title="Breaks on the caller of the function that mutates this. Only works on the second call (and only if the function is input predicted)." onClick={() => {
+                                        if (breakOnWrites) {
+                                            PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.delete(path);
+                                        } else {
+                                            PathValueProxyWatcher.SET_FUNCTION_WATCH_ON_WRITES.add(path);
+                                        }
+                                        this.forceUpdate();
+                                    }}>
+                                        Caller*
+                                    </button>
+                                    <div class={css.hbox(4).wrap.button} onClick={() => {
+                                        console.log(path);
+                                        return navigator.clipboard.writeText(path);
+                                    }}>
+                                        {pathArray.map((x, index) => <span class={css.pad2(3, 2).hsl(0, 0, 80)}>
+                                            {x}
+                                        </span>)}
+                                    </div>
+                                    <div class={css.pad2(3, 2).hsl(180, 75, 75).ellipsis.button} onClick={() => {
+                                        console.log(value);
+                                    }}>
+                                        {valueStr}
+                                    </div>
+                                </div>
+                            );
+                        })}
+                    </div>
+                </div>
+            </FullscreenModal>
+        );
+    }
+}

package/src/5-diagnostics/shared.ts

@@ -0,0 +1,26 @@
+import { isBufferType } from "socket-function/src/misc";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { PathValue } from "../0-path-value-core/pathValueCore";
+
+function getSingleSizeEstimateValue(node: PathValue) {
+    let value = pathValueSerializer.getPathValueOrBuffer(node);
+    if (isBufferType(value)) {
+        return value.byteLength;
+    }
+    if (typeof value === "string") {
+        return value.length * 2;
+    }
+    return 8;
+}
+/** NOTE: This will never be accurate, because of how strings are shared and compressed.
+ *      But, if we need a conservative (over-estimate) of the size of a PathValue, this works well enough.
+ */
+export function getSingleSizeEstimate(node: PathValue) {
+    return (
+        getSingleSizeEstimateValue(node)
+        + node.path.length * 2
+        + (node.source?.length || 0)
+        // Plus space for other misc fields
+        + 32
+    );
+}

package/src/5-diagnostics/synchronousLagTracking.ts

@@ -0,0 +1,47 @@
+import { runInfinitePoll } from "socket-function/src/batching";
+import { formatTime } from "socket-function/src/formatting/format";
+import { registerMeasureInfo } from "socket-function/src/profiling/measure";
+import { isNode } from "typesafecss";
+import { monitorEventLoopDelay } from "perf_hooks";
+import { registerNodeMetadata } from "./nodeMetadata";
+
+const POLL_INTERVAL = 350;
+
+export function trackSynchronousLag() {
+    if (isNode()) {
+        const histogram = monitorEventLoopDelay({});
+        histogram.enable();
+
+        registerMeasureInfo(() => {
+            const max = histogram.max;
+            const p99 = histogram.percentile(99);
+            const p50 = histogram.percentile(50);
+            histogram.reset();
+
+            return `BLOCKING LAG < ${formatTime(max / 1e6)} | 99% < ${formatTime(p99 / 1e6)} | 50% < ${formatTime(p50 / 1e6)}`;
+        });
+        registerNodeMetadata({
+            columnName: "Blocking Lag",
+            getValue: () => {
+                const max = histogram.max;
+                const p99 = histogram.percentile(99);
+                const p50 = histogram.percentile(50);
+                return [`< ${formatTime(max / 1e6)}`, `99% < ${formatTime(p99 / 1e6)}`, `50% < ${formatTime(p50 / 1e6)}`];
+            }
+        });
+    } else {
+        let lastLag = POLL_INTERVAL;
+        let time = Date.now();
+        runInfinitePoll(POLL_INTERVAL, function synchronousPoll() {
+            let newTime = Date.now();
+            let newLag = newTime - time;
+            lastLag = Math.max(lastLag, newLag);
+            time = newTime;
+        });
+        registerMeasureInfo(() => {
+            let curLag = lastLag;
+            lastLag = POLL_INTERVAL;
+            return `BLOCKING LAG < ${formatTime(curLag)}`;
+        });
+    }
+}

package/src/TestController.ts

@@ -0,0 +1,35 @@
+module.allowclient = true;
+
+import debugbreak from "debugbreak";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { IdentityController_getReconnectNodeId } from "./-c-identity/IdentityController";
+
+import { requiresNetworkTrustHook } from "./-d-trust/NetworkTrust2";
+import os from "os";
+
+class TestControllerBase {
+    async test() {
+        const caller = SocketFunction.getCaller();
+        console.log(`test called from ${caller.nodeId}`);
+        return {
+            nodeId: caller?.nodeId,
+            reconnectNodeId: IdentityController_getReconnectNodeId(caller),
+        };
+    }
+    async getProcessId() {
+        return process.pid;
+    }
+}
+
+
+export const TestController = SocketFunction.register(
+    "TestController-fbdd0aa9-a8ed-4d8c-80a8-8ab2f57c99db",
+    new TestControllerBase(),
+    () => ({
+        getProcessId: {},
+        test: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    })
+);

package/src/bits.ts

@@ -0,0 +1,86 @@
+
+/** Subtracts the smallest possible value from a number (a double). This makes it possible to convert an exclusive range end
+ *      to an inclusive range end, which is sometimes required (as in, < x is the same as <= minusEpsilon(x)).
+ */
+export function minusEpsilon(value: number) {
+    let high = getHighUint32(value);
+    let low = getLowUint32(value);
+
+    if (low === 0) {
+        low = 2 ** 32 - 1;
+        high--;
+    } else {
+        low--;
+    }
+
+    return setLowHighUint32(low, high);
+}
+const maxUint32 = 2 ** 32 - 1;
+export function addEpsilons(value: number, count: number) {
+    let high = getHighUint32(value);
+    let low = getLowUint32(value);
+
+    low += count;
+    if (low < 0) {
+        low += maxUint32;
+        high++;
+    } else if (low > maxUint32) {
+        low -= maxUint32;
+        high--;
+    }
+
+    return setLowHighUint32(low, high);
+}
+
+let conversionBuffer = new Float64Array(1);
+let conversionUint8Buffer = new Uint8Array(conversionBuffer.buffer);
+let conversionUint32Buffer = new Uint32Array(conversionBuffer.buffer);
+export function getHighUint32(num: number): number {
+    conversionBuffer[0] = num;
+    return conversionUint32Buffer[1];
+}
+export function getLowUint32(num: number): number {
+    conversionBuffer[0] = num;
+    return conversionUint32Buffer[0];
+}
+
+/** IMPORTANT! Beware of comparisons with 64 bit numbers. getFloat64_fromBytes(4294136438, 168) !== getFloat64_fromBytes(4294136438, 168).
+ *      USE is64BitEqual instead, OR, ensure the 2 highest bits are always 0.
+ */
+export function setLowHighUint32(low: number, high: number): number {
+    conversionUint32Buffer[0] = low;
+    conversionUint32Buffer[1] = high;
+    return conversionBuffer[0];
+}
+
+// Gets bits that can be stored in a number. Specifically, the first 62 bits,
+//  as 64 bits will not compare correctly when treated as a double.
+export function getShortNumber(buffer: Buffer): number {
+    let high = buffer.readUInt32BE(0) & 0x3FFFFFFF;
+    let low = buffer.readUInt32BE(4);
+    return setLowHighUint32(low, high);
+}
+/* Returns a number between 0 and 2**48 */
+export function getBufferInt(buffer: Buffer): number {
+    let num = 0;
+    for (let i = 0; i < Math.min(buffer.length, 6); i++) {
+        num = num * 256 + buffer[i];
+    }
+    return num;
+}
+const intMax = 2 ** 48;
+/** Returns a number between 0 (inclusive) and 1 (exclusive) */
+export function getBufferFraction(buffer: Buffer): number {
+    let int = getBufferInt(buffer);
+    return int / intMax;
+}
+
+/*
+export function numberToBase64(num: number): string {
+    conversionBuffer[0] = num;
+    return Buffer.from(conversionBuffer.buffer).toString("base64");
+}
+export function numberFromBase64(base64: string) {
+    return new Float64Array(Buffer.from(base64, "base64"))[0];
+}
+*/