querysub 0.490.0 → 0.492.0

package/src/3-path-functions/functionReplay.ts

@@ -0,0 +1,272 @@
+import { epochTime, PathValue } from "../0-path-value-core/pathValueCore";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { isTransparentValue, PathValueReadSource, proxyWatcher, runWithPathValueReadSource } from "../2-proxy/PathValueProxyWatcher";
+import { getPathFromStr, getParentPathStr } from "../path";
+import { CaptureRecord } from "./functionCaptureFormat";
+import { CallSpec, FunctionSpec, overrideCurrentCall } from "./PathFunctionRunner";
+import { getModuleFromSpec } from "./pathFunctionLoader";
+import { parseArgs } from "./PathFunctionHelpers";
+
+export function makeReplayCallSpec(record: CaptureRecord): CallSpec {
+    return {
+        DomainName: record.functionSpec.DomainName,
+        ModuleId: record.functionSpec.ModuleId,
+        CallId: record.callId,
+        FunctionId: record.functionSpec.FunctionId,
+        argsEncoded: record.argsEncoded,
+        callerMachineId: record.callerMachineId,
+        callerIP: record.callerIP,
+        runAtTime: record.runAtTime,
+    };
+}
+
+function makePathValue(path: string, value: unknown, time: PathValue["time"]): PathValue {
+    return {
+        path,
+        value,
+        time,
+        valid: true,
+        locks: [],
+        lockCount: 0,
+        isValueLazy: false,
+        isTransparent: isTransparentValue(value),
+        canGCValue: value === undefined,
+    };
+}
+
+// A PathValueReadSource backed entirely by a single captured run's reads. Because the capture
+//      recorded every value the run actually read, everything is treated as synced, so a replay
+//      finishes in a single synchronous pass with no real database.
+export function buildReplayReadSource(record: CaptureRecord): PathValueReadSource {
+    let values = new Map<string, PathValue>();
+    for (let read of record.reads) {
+        values.set(read.path, makePathValue(read.path, read.value, read.time));
+    }
+
+    // Direct-parent -> children index, so getKeys()/Object.keys() enumeration replays identically.
+    let parentIndex = new Map<string, Set<string>>();
+    let addToParentIndex = (path: string) => {
+        let parent = getParentPathStr(path);
+        let set = parentIndex.get(parent);
+        if (!set) {
+            set = new Set();
+            parentIndex.set(parent, set);
+        }
+        set.add(path);
+    };
+    for (let read of record.reads) addToParentIndex(read.path);
+    for (let read of record.undefinedReads) addToParentIndex(read.path);
+
+    let overrideStack: Map<string, PathValue>[] = [];
+    let readWithOverrides = (path: string): PathValue | undefined => {
+        for (let i = overrideStack.length - 1; i >= 0; i--) {
+            let value = overrideStack[i].get(path);
+            if (value) return value;
+        }
+        return values.get(path);
+    };
+
+    return {
+        getValueAtOrBeforeTime(pathStr) {
+            let stored = readWithOverrides(pathStr);
+            if (stored) return stored;
+            // Mirror real storage: a synced-but-empty path resolves to an epoch transparent value
+            //  (equivalent to "no value", but synced), so uncaptured reads don't stall the run.
+            return { path: pathStr, valid: true, time: epochTime, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true };
+        },
+        isSynced() {
+            return true;
+        },
+        isParentSynced() {
+            return true;
+        },
+        getPathsFromParent(pathStr) {
+            return parentIndex.get(pathStr);
+        },
+        temporaryOverride(overrides, code) {
+            if (!overrides || overrides.length === 0) return code();
+            let map = new Map<string, PathValue>();
+            for (let override of overrides) map.set(override.path, override);
+            overrideStack.push(map);
+            try {
+                return code();
+            } finally {
+                overrideStack.pop();
+            }
+        },
+        DEBUG_hasAnyValues(pathStr) {
+            return values.has(pathStr);
+        },
+    };
+}
+
+export interface ReplayResult {
+    writes: PathValue[];
+    result: unknown;
+}
+
+// Runs the function against a captured run's data in complete isolation, using a dry-run watcher so
+//      nothing is committed. Fully synchronous (the captured data is all "synced").
+export function replayCapturedCall(config: {
+    record: CaptureRecord;
+    baseFunction: Function;
+    args: unknown[];
+}): ReplayResult {
+    let { record, baseFunction, args } = config;
+    let readSource = buildReplayReadSource(record);
+    let callSpec = makeReplayCallSpec(record);
+
+    let captured: ReplayResult | { error: string } | undefined;
+    runWithPathValueReadSource(readSource, () => {
+        proxyWatcher.createWatcher({
+            dryRun: true,
+            canWrite: true,
+            runImmediately: true,
+            allowUnsyncedReads: true,
+            temporary: true,
+            skipPermissionsCheck: true,
+            nestedCalls: "ignore",
+            runAtTime: record.runAtTime,
+            maxLocksOverride: Number.MAX_SAFE_INTEGER,
+            debugName: `replay ${record.functionSpec.ModuleId}.${record.functionSpec.FunctionId} ${record.callId}`,
+            watchFunction: () => {
+                overrideCurrentCall({ spec: callSpec, fnc: record.functionSpec }, () => {
+                    baseFunction(...args);
+                });
+            },
+            onResultUpdated: (result, writes) => {
+                if ("error" in result) {
+                    captured = { error: result.error };
+                } else {
+                    captured = { writes: writes ?? [], result: result.result };
+                }
+            },
+        });
+    });
+
+    if (!captured) {
+        throw new Error(`Replay produced no result for ${record.callId}`);
+    }
+    if ("error" in captured) {
+        throw new Error(`Replay threw for ${record.functionSpec.ModuleId}.${record.functionSpec.FunctionId} (${record.callId}): ${captured.error}`);
+    }
+    return captured;
+}
+
+export interface ReplayComparison {
+    identical: boolean;
+    matched: number;
+    mismatched: { path: string; captured: unknown; replayed: unknown }[];
+    // In the capture, but the replay didn't write it.
+    missing: { path: string; captured: unknown }[];
+    // The replay wrote it, but the capture didn't have it.
+    extra: { path: string; replayed: unknown }[];
+}
+
+function stableStringify(value: unknown): string {
+    return JSON.stringify(value, (_key, val) => {
+        if (val && typeof val === "object" && !Array.isArray(val)) {
+            let sorted: { [key: string]: unknown } = {};
+            for (let key of Object.keys(val).sort()) {
+                sorted[key] = (val as { [key: string]: unknown })[key];
+            }
+            return sorted;
+        }
+        return val;
+    });
+}
+
+// Compares the writes a replay produced against the writes originally captured. Times/locks are
+//      ignored (they're assigned fresh on every run); only path -> value output state is compared.
+export function compareCapturedWrites(record: CaptureRecord, replayWrites: PathValue[]): ReplayComparison {
+    let capturedMap = new Map<string, unknown>();
+    for (let write of record.writes) capturedMap.set(write.path, write.value);
+
+    let replayMap = new Map<string, unknown>();
+    for (let write of replayWrites) replayMap.set(write.path, pathValueSerializer.getPathValue(write));
+
+    let matched = 0;
+    let mismatched: ReplayComparison["mismatched"] = [];
+    let missing: ReplayComparison["missing"] = [];
+    let extra: ReplayComparison["extra"] = [];
+
+    for (let [path, capturedValue] of capturedMap) {
+        if (!replayMap.has(path)) {
+            missing.push({ path, captured: capturedValue });
+            continue;
+        }
+        let replayedValue = replayMap.get(path);
+        if (stableStringify(capturedValue) === stableStringify(replayedValue)) {
+            matched++;
+        } else {
+            mismatched.push({ path, captured: capturedValue, replayed: replayedValue });
+        }
+    }
+    for (let [path, replayedValue] of replayMap) {
+        if (!capturedMap.has(path)) {
+            extra.push({ path, replayed: replayedValue });
+        }
+    }
+
+    return {
+        identical: mismatched.length === 0 && missing.length === 0 && extra.length === 0,
+        matched,
+        mismatched,
+        missing,
+        extra,
+    };
+}
+
+async function loadBaseFunction(spec: FunctionSpec): Promise<Function> {
+    let module = await getModuleFromSpec(spec);
+    let exportObj: any = module.exports;
+    for (let part of getPathFromStr(spec.exportPathStr)) {
+        exportObj = exportObj?.[part];
+    }
+    if (typeof exportObj !== "function") {
+        throw new Error(`Export at ${JSON.stringify(spec.exportPathStr)} in ${spec.FilePath} was not a function (it was ${typeof exportObj}). The captured code may not be loadable in this environment.`);
+    }
+    return exportObj as Function;
+}
+
+export interface ReplayRecordReport {
+    callId: string;
+    functionName: string;
+    ok: boolean;
+    error?: string;
+    comparison?: ReplayComparison;
+}
+
+export interface ReplaySummary {
+    total: number;
+    identical: number;
+    divergent: number;
+    errored: number;
+    reports: ReplayRecordReport[];
+}
+
+// Loads each captured function and replays it in isolation, comparing the replayed writes against
+//      the originally captured writes. Requires the app's modules to be loadable (import its deploy.ts
+//      before calling this, so development modules are registered).
+export async function replayCaptureRecords(records: CaptureRecord[]): Promise<ReplaySummary> {
+    let reports: ReplayRecordReport[] = [];
+    for (let record of records) {
+        let functionName = `${record.functionSpec.ModuleId}.${record.functionSpec.FunctionId}`;
+        try {
+            let baseFunction = await loadBaseFunction(record.functionSpec);
+            let args = parseArgs(makeReplayCallSpec(record));
+            let replay = replayCapturedCall({ record, baseFunction, args });
+            let comparison = compareCapturedWrites(record, replay.writes);
+            reports.push({ callId: record.callId, functionName, ok: comparison.identical, comparison });
+        } catch (e: any) {
+            reports.push({ callId: record.callId, functionName, ok: false, error: String(e?.stack || e) });
+        }
+    }
+    return {
+        total: reports.length,
+        identical: reports.filter(r => r.ok).length,
+        divergent: reports.filter(r => !r.ok && !r.error).length,
+        errored: reports.filter(r => r.error).length,
+        reports,
+    };
+}

package/src/4-querysub/Querysub.ts

@@ -18,7 +18,7 @@ import { RequireController, setRequireBootRequire } from "socket-function/requir
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getIdentityCAPromise, getOwnMachineId, getOwnThreadId, getThreadKeyCert, verifyMachineIdForPublicKey } from "sliftutils/misc/https/certs";
 import { getHostedIP, getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
-import { debugCoreMode, registerGetCompressNetwork, authorityStorage, enableDebugRejections } from "../0-path-value-core/pathValueCore";
+import { debugCoreMode, registerGetCompressNetwork, authorityStorage, enableDebugRejections, getNextTime } from "../0-path-value-core/pathValueCore";
 import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions, doProxyOptions } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
@@ -906,7 +906,7 @@ export class Querysub {
             globalThis.remapImportRequestsClientside = globalThis.remapImportRequestsClientside || [];
             globalThis.remapImportRequestsClientside.push(async (args) => {
                 try {
-                    let key: typeof identityStorageKey = "machineCA_11";
+                    let key: typeof identityStorageKey = "machineCA_14";
                     let storageValueJSON = localStorage.getItem(key);
                     if (!storageValueJSON) return args;
                     let storageValue = JSON.parse(storageValueJSON) as IdentityStorageType;
@@ -1204,6 +1204,9 @@ function getSyncedTimeUnique() {
 }
 
 function getSyncedTime() {
+    if (!Querysub.isInSyncedCall()) {
+        return Date.now();
+    }
     let setTime = getSetTime();
     if (setTime !== undefined) {
         return setTime;

package/src/diagnostics/managementPages.tsx

@@ -123,6 +123,17 @@ export async function registerManagementPages2(config: {
         controllerName: "DNSPageController",
         getModule: () => import("./misc-pages/DNSPage"),
     });
+    inputPages.push({
+        title: "Fnc Capture",
+        componentName: "FunctionCapturePage",
+        controllerName: "FunctionCaptureProxyController",
+        getModule: () => import("./misc-pages/FunctionCapturePage"),
+    });
+    inputPages.push({
+        title: "Capture Analyzer",
+        componentName: "FunctionCaptureAnalyzePage",
+        getModule: () => import("./misc-pages/FunctionCaptureAnalyzePage"),
+    });
     inputPages.push({
         title: "LOG VIEWER",
         componentName: "LogViewer3",

package/src/diagnostics/misc-pages/FunctionCaptureAnalyzePage.tsx

@@ -0,0 +1,220 @@
+module.allowclient = true;
+
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { Querysub, t } from "../../4-querysub/Querysub";
+import { Button } from "../../library-components/Button";
+import { formatNumber, formatTime, formatDateTime } from "socket-function/src/formatting/format";
+import { green, red } from "socket-function/src/formatting/logColors";
+import { getPathStr2 } from "../../path";
+import { sort } from "socket-function/src/misc";
+import { CaptureRecord, CaptureFileHeader, decodeCaptureFile } from "../../3-path-functions/functionCaptureFormat";
+
+// How many rows of a read/write list we show before requiring a "Show more" click.
+const ROW_LIMIT = 50;
+
+interface ParsedCapture {
+    header: CaptureFileHeader;
+    records: CaptureRecord[];
+    fileName: string;
+}
+
+export class FunctionCaptureAnalyzePage extends qreact.Component {
+    state = t.state({
+        error: t.string(""),
+        data: t.atomic<ParsedCapture>(),
+        expandedGroups: t.lookup(t.boolean),
+        expandedCalls: t.lookup(t.boolean),
+        // Section id => how many rows to show.
+        rowLimits: t.lookup(t.number),
+    });
+
+    private async loadFile(file: File | undefined) {
+        if (!file) return;
+        try {
+            let buffer = Buffer.from(await file.arrayBuffer());
+            let { header, records } = decodeCaptureFile(buffer);
+            Querysub.localCommit(() => {
+                this.state.data = { header, records, fileName: file.name };
+                this.state.error = "";
+                for (let key of Object.keys(this.state.expandedGroups)) delete this.state.expandedGroups[key];
+                for (let key of Object.keys(this.state.expandedCalls)) delete this.state.expandedCalls[key];
+            });
+            console.log(green(`Loaded capture ${file.name}: ${records.length} records`));
+        } catch (e: any) {
+            Querysub.localCommit(() => {
+                this.state.error = String(e?.stack || e);
+            });
+            console.error(red(`Failed to parse capture: ${e.stack}`));
+        }
+    }
+
+    render() {
+        let data = this.state.data;
+        return <div className={css.pad2(10).vbox(14).fillWidth}>
+            <h1>Function Capture Analyzer</h1>
+            {this.renderDropZone()}
+            {this.state.error && <div className={css.colorhsl(0, 70, 60).whiteSpace("pre-wrap")}>{this.state.error}</div>}
+            {data && this.renderData(data)}
+        </div>;
+    }
+
+    private renderDropZone() {
+        return <div
+            className={css.fillWidth.pad2(24).bord2(210, 30, 45, 2, "dashed").hsl(210, 40, 88).button.textAlign("center")}
+            onDragOver={e => e.preventDefault()}
+            onDrop={e => {
+                e.preventDefault();
+                void this.loadFile(e.dataTransfer?.files?.[0]);
+            }}
+            onClick={() => {
+                let input = document.createElement("input");
+                input.type = "file";
+                input.accept = ".qsfncap";
+                input.onchange = () => void this.loadFile(input.files?.[0]);
+                input.click();
+            }}
+        >
+            Drop a .qsfncap capture file here, or click to choose one.
+        </div>;
+    }
+
+    private renderData(data: ParsedCapture) {
+        let groups = groupRecords(data.records);
+        return <div className={css.vbox(12).fillWidth}>
+            <div className={css.vbox(2)}>
+                <div className={css.fontWeight("bold")}>{data.fileName}</div>
+                <div className={css.fontSize(12).colorhsl(0, 0, 60)}>
+                    version {data.header.version} · {formatNumber(data.records.length)} calls · {data.header.domainName || "?"} · created {formatDateTime(data.header.createdAt)}
+                </div>
+            </div>
+            <div className={css.vbox(1).fillWidth}>
+                {groups.map(group => this.renderGroup(group))}
+            </div>
+        </div>;
+    }
+
+    private renderGroup(group: FunctionGroup) {
+        let expanded = !!this.state.expandedGroups[group.key];
+        return <div className={css.vbox(1).fillWidth.borderBottom("1px solid hsla(0,0%,0%,0.15)")}>
+            <div
+                className={css.hbox(16).fillWidth.pad2(8, 6).button}
+                onClick={() => Querysub.localCommit(() => { this.state.expandedGroups[group.key] = !expanded; })}
+            >
+                <div className={css.width(16)}>{expanded ? "▾" : "▸"}</div>
+                <div className={css.width(420).vbox(2)}>
+                    <div className={css.fontWeight("bold")}>{group.functionName}</div>
+                    <div className={css.fontSize(11).colorhsl(0, 0, 50)}>{group.filePath}</div>
+                </div>
+                <div className={css.width(90)}>{formatNumber(group.records.length)} calls</div>
+                <div className={css.width(140)}>avg {formatTime(group.avgEvalTime)}</div>
+            </div>
+            {expanded &&
+                <div className={css.vbox(1).fillWidth.pad2(24, 0)}>
+                    {group.records.map((record, i) => this.renderCall(group.key, record, i))}
+                </div>
+            }
+        </div>;
+    }
+
+    private renderCall(groupKey: string, record: CaptureRecord, index: number) {
+        let callKey = getPathStr2(groupKey, record.callId);
+        let expanded = !!this.state.expandedCalls[callKey];
+        return <div className={css.vbox(2).fillWidth.borderBottom("1px solid hsla(0,0%,0%,0.1)")}>
+            <div
+                className={css.hbox(16).fillWidth.pad2(6, 4).button}
+                onClick={() => Querysub.localCommit(() => { this.state.expandedCalls[callKey] = !expanded; })}
+            >
+                <div className={css.width(16)}>{expanded ? "▾" : "▸"}</div>
+                <div className={css.width(220).ellipsis}>call #{index + 1} · {record.callId}</div>
+                <div className={css.width(120)}>{formatNumber(record.reads.length)} reads</div>
+                <div className={css.width(120)}>{formatNumber(record.writes.length)} writes</div>
+                <div className={css.width(120)}>eval {formatTime(record.evalTime)}</div>
+                <div className={css.width(180)}>runAt {formatDateTime(record.runAtTime.time)}</div>
+            </div>
+            {expanded && this.renderCallDetail(callKey, record)}
+        </div>;
+    }
+
+    private renderCallDetail(callKey: string, record: CaptureRecord) {
+        return <div className={css.vbox(10).fillWidth.pad2(24, 6)}>
+            <div className={css.fontSize(12).colorhsl(0, 0, 60)}>args: {shortValue(record.argsEncoded)}</div>
+            {this.renderSection(getPathStr2(callKey, "reads"), "Reads", record.reads.map(r => `${r.path}  =  ${shortValue(r.value)}`))}
+            {record.undefinedReads.length > 0 &&
+                this.renderSection(getPathStr2(callKey, "undef"), "Undefined reads", record.undefinedReads.map(r => r.path))}
+            {record.parentKeyReads.length > 0 &&
+                this.renderSection(getPathStr2(callKey, "keys"), "Enumerated (getKeys)", record.parentKeyReads)}
+            {this.renderSection(getPathStr2(callKey, "writes"), "Writes", record.writes.map(w => `${w.path}  =  ${shortValue(w.value)}${w.isTransparent ? "  (transparent)" : ""}`))}
+        </div>;
+    }
+
+    private renderSection(sectionId: string, title: string, rows: string[]) {
+        let limit = ROW_LIMIT;
+        if (sectionId in this.state.rowLimits) {
+            limit = Number(this.state.rowLimits[sectionId]);
+        }
+        let shown = rows.slice(0, limit);
+        let remaining = rows.length - shown.length;
+        return <div className={css.vbox(2).fillWidth}>
+            <div className={css.fontWeight("bold")}>{title} ({formatNumber(rows.length)})</div>
+            <div className={css.vbox(1).fillWidth.fontFamily("monospace").fontSize(12)}>
+                {shown.map(row => <div className={css.whiteSpace("pre-wrap").wordBreak("break-all")}>{row}</div>)}
+            </div>
+            {remaining > 0 &&
+                <Button onClick={() => Querysub.localCommit(() => { this.state.rowLimits[sectionId] = limit + ROW_LIMIT; })}>
+                    Show {Math.min(ROW_LIMIT, remaining)} more ({formatNumber(remaining)} remaining)
+                </Button>
+            }
+        </div>;
+    }
+}
+
+interface FunctionGroup {
+    key: string;
+    functionName: string;
+    filePath: string;
+    avgEvalTime: number;
+    records: CaptureRecord[];
+}
+
+function groupRecords(records: CaptureRecord[]): FunctionGroup[] {
+    let map = new Map<string, CaptureRecord[]>();
+    for (let record of records) {
+        let key = getPathStr2(record.functionSpec.ModuleId, record.functionSpec.FunctionId);
+        let list = map.get(key);
+        if (!list) {
+            list = [];
+            map.set(key, list);
+        }
+        list.push(record);
+    }
+    let groups: FunctionGroup[] = [];
+    for (let [key, list] of map) {
+        let spec = list[0].functionSpec;
+        let totalEval = list.reduce((sum, r) => sum + r.evalTime, 0);
+        let exportName = spec.exportPathStr ? spec.exportPathStr.split("/").slice(1).join(".") : spec.FunctionId;
+        groups.push({
+            key,
+            functionName: `${spec.ModuleId} · ${exportName}`,
+            filePath: spec.FilePath,
+            avgEvalTime: totalEval / list.length,
+            records: list,
+        });
+    }
+    sort(groups, group => -group.records.length);
+    return groups;
+}
+
+function shortValue(value: unknown): string {
+    let text: string;
+    try {
+        text = typeof value === "string" ? value : JSON.stringify(value);
+    } catch {
+        text = String(value);
+    }
+    if (text === undefined) text = String(value);
+    if (text.length > 300) {
+        text = text.slice(0, 300) + `… (${formatNumber(text.length)} chars)`;
+    }
+    return text;
+}