querysub 0.491.0 → 0.492.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.491.0",
+  "version": "0.492.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -43,9 +43,31 @@ import { isClient } from "../config2";
 
 const DEFAULT_MAX_LOCKS = 1000;
 
-// After this time we allow proxies to be reordered, even if there's flags that tell them not to be. 
+// After this time we allow proxies to be reordered, even if there's flags that tell them not to be.
 const MAX_PROXY_REORDER_BLOCK_TIME = timeInSecond * 10;
 
+// The proxy watcher reads all of its synced data through this indirection instead of importing
+//      authorityStorage directly, so the data source can be swapped out. This is what lets us replay
+//      captured function runs against an in-memory dataset in complete isolation (see functionReplay.ts).
+export interface PathValueReadSource {
+    getValueAtOrBeforeTime(pathStr: string, time: Time | undefined): PathValue | undefined;
+    isSynced(pathStr: string): boolean;
+    isParentSynced(pathStr: string): boolean;
+    getPathsFromParent(pathStr: string): Set<string> | undefined;
+    temporaryOverride<T>(overrides: PathValue[] | undefined, code: () => T): T;
+    DEBUG_hasAnyValues(pathStr: string): boolean;
+}
+let currentReadSource: PathValueReadSource = authorityStorage;
+export function runWithPathValueReadSource<T>(source: PathValueReadSource, code: () => T): T {
+    let prev = currentReadSource;
+    currentReadSource = source;
+    try {
+        return code();
+    } finally {
+        currentReadSource = prev;
+    }
+}
+
 let nextSeqNum = 1;
 let nextOrderSeqNum = 1;
 
@@ -566,7 +588,7 @@ export class PathValueProxyWatcher {
             if (!watcher.permissionsChecker.checkPermissions(pathStr).allowed) {
                 if (
                     !watcher.hasAnyUnsyncedAccesses()
-                    && authorityStorage.DEBUG_hasAnyValues(pathStr)
+                    && currentReadSource.DEBUG_hasAnyValues(pathStr)
                     // HACK: Don't show warnings for some framework paths, because they are filling up the console logs
                     //  and don't really matter. We could just not request them, but at depth 3 is valid,
                     //  so we kind of have to request that. And at depth 2, it would require special case code,
@@ -589,14 +611,14 @@ export class PathValueProxyWatcher {
 
         const currentReadTime = watcher.options.forceReadLatest ? undefined : watcher.currentReadTime;
 
-        let pathValue = authorityStorage.getValueAtOrBeforeTime(pathStr, currentReadTime);
+        let pathValue = currentReadSource.getValueAtOrBeforeTime(pathStr, currentReadTime);
         if (!watcher.options.noSyncing) {
             // NOTE: If we have any value, we are always synced (that's what synced means, as if we aren't syncing,
             //  we delete any values, to prevent stale values from being used).
             if (!pathValue) {
                 // NOTE: We might not have a value simply due to reading too far back in time,
                 //  so we have to call isSynced just to be sure it is actually unsynced
-                if (!authorityStorage.isSynced(pathStr)) {
+                if (!currentReadSource.isSynced(pathStr)) {
                     for (let checker of this.isUnsyncCheckers) {
                         checker.value++;
                     }
@@ -604,7 +626,7 @@ export class PathValueProxyWatcher {
                 }
             }
             if (syncParentKeys) {
-                if (!authorityStorage.isParentSynced(pathStr)) {
+                if (!currentReadSource.isParentSynced(pathStr)) {
                     watcher.pendingUnsyncedParentAccesses.add(pathStr);
                 }
             }
@@ -940,7 +962,7 @@ export class PathValueProxyWatcher {
             return getKeys(pathValue?.value) as string[];
         }
 
-        let childPaths = authorityStorage.getPathsFromParent(pathStr);
+        let childPaths = currentReadSource.getPathsFromParent(pathStr);
 
         // We need to also get keys from pendingWrites
         {
@@ -996,7 +1018,7 @@ export class PathValueProxyWatcher {
         }
 
         if (symbol === syncedSymbol) {
-            return { value: authorityStorage.isSynced(pathStr) };
+            return { value: currentReadSource.isSynced(pathStr) };
         }
 
         // Proxies should be considered atomic, at least for the purpose of other proxies!
@@ -1066,7 +1088,7 @@ export class PathValueProxyWatcher {
             let { watchFunction, ...mostOptions } = options;
             doProxyOptions(mostOptions, () => {
                 try {
-                    let result = authorityStorage.temporaryOverride(options.overrides, () =>
+                    let result = currentReadSource.temporaryOverride(options.overrides, () =>
                         options.watchFunction()
                     );
                     // Clone, otherwise proxies get out of the watcher, which can result in accesses outside
@@ -1353,7 +1375,7 @@ export class PathValueProxyWatcher {
                         }
                     },
                     code() {
-                        return authorityStorage.temporaryOverride(options.overrides, () =>
+                        return currentReadSource.temporaryOverride(options.overrides, () =>
                             runCodeWithDatabase(proxy, baseFunction)
                         );
                     },
@@ -1457,8 +1479,8 @@ export class PathValueProxyWatcher {
                         setTimeout(() => {
                             if (watcher.syncRunCount !== syncRunCount) return;
                             if (watcher.disposed) return;
-                            let remainingUnsynced = Array.from(watcher.lastUnsyncedAccesses).filter(x => !authorityStorage.isSynced(x));
-                            let remainingUnsyncedParent = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !authorityStorage.isParentSynced(x));
+                            let remainingUnsynced = Array.from(watcher.lastUnsyncedAccesses).filter(x => !currentReadSource.isSynced(x));
+                            let remainingUnsyncedParent = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !currentReadSource.isParentSynced(x));
                             console.warn(red(`Did not sync ${watcher.debugName} after 30 seconds. Either we had a lot synchronous block, or we will never received the values we are waiting for.`), { remainingUnsynced, remainingUnsyncedParent }, watcher.options.watchFunction);
                         }, 30 * 1000);
                         setTimeout(() => {
@@ -1472,8 +1494,8 @@ export class PathValueProxyWatcher {
                             //      we have had since this watcher last synced. If it is > 50% of the time...
                             //      then synchronous lag is the issue.
 
-                            let reallyUnsyncedAccesses = Array.from(watcher.lastUnsyncedAccesses).filter(x => !authorityStorage.isSynced(x));
-                            let reallyUnsyncedParentAccesses = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !authorityStorage.isParentSynced(x));
+                            let reallyUnsyncedAccesses = Array.from(watcher.lastUnsyncedAccesses).filter(x => !currentReadSource.isSynced(x));
+                            let reallyUnsyncedParentAccesses = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !currentReadSource.isParentSynced(x));
 
                             if (reallyUnsyncedAccesses.length !== 0 || reallyUnsyncedParentAccesses.length !== 0) {
                                 let notWatchingUnsynced = reallyUnsyncedAccesses.filter(x => !remoteWatcher.debugIsWatchingPath(x));
@@ -1705,12 +1727,12 @@ export class PathValueProxyWatcher {
             if (watcher.options.commitAllRuns) return false;
             // NOTE: We COULD remove any synced values from lastUnsyncedAccesses, however... we will generally sync all values at once, so we don't really need to optimize the cascading case here. Also... deleting values requires cloning while we iterate, as well as mutating the set, which probably makes the non-cascading case slower.
             for (let path of watcher.lastUnsyncedAccesses) {
-                if (!authorityStorage.isSynced(path)) {
+                if (!currentReadSource.isSynced(path)) {
                     return true;
                 }
             }
             for (let path of watcher.lastUnsyncedParentAccesses) {
-                if (!authorityStorage.isParentSynced(path)) {
+                if (!currentReadSource.isParentSynced(path)) {
                     return true;
                 }
             }
@@ -1723,13 +1745,13 @@ export class PathValueProxyWatcher {
         function logUnsynced() {
             let anyLogged = false;
             for (let path of watcher.lastUnsyncedAccesses) {
-                if (!authorityStorage.isSynced(path)) {
+                if (!currentReadSource.isSynced(path)) {
                     console.log(yellow(`    Waiting for ${path}`));
                     anyLogged = true;
                 }
             }
             for (let path of watcher.lastUnsyncedParentAccesses) {
-                if (!authorityStorage.isParentSynced(path)) {
+                if (!currentReadSource.isParentSynced(path)) {
                     console.log(yellow(`    Waiting for parent ${path}`));
                     anyLogged = true;
                 }
@@ -1997,7 +2019,7 @@ export class PathValueProxyWatcher {
     public async commitFunction<Result = void>(
         options: Omit<WatcherOptions<Result>, "onResultUpdated" | "onWriteCommitted">,
         config?: {
-            onWritesCommitted?: (writes: PathValue[]) => void;
+            onWritesCommitted?: (writes: PathValue[], watcher: SyncWatcher) => void;
         }
     ): Promise<Result> {
         options = {
@@ -2029,7 +2051,7 @@ export class PathValueProxyWatcher {
                     onResult(result.result);
                 }
                 if (writes && config?.onWritesCommitted) {
-                    config.onWritesCommitted(writes);
+                    config.onWritesCommitted(writes, watcher);
                 }
             }
         });
@@ -2131,12 +2153,12 @@ export class PathValueProxyWatcher {
     public reuseLastWatches() {
         let watcher = this.getTriggeredWatcher();
         for (let path of watcher.lastUnsyncedAccesses) {
-            if (!authorityStorage.isSynced(path)) {
+            if (!currentReadSource.isSynced(path)) {
                 watcher.pendingUnsyncedAccesses.add(path);
             }
         }
         for (let path of watcher.lastUnsyncedParentAccesses) {
-            if (!authorityStorage.isParentSynced(path)) {
+            if (!currentReadSource.isParentSynced(path)) {
                 watcher.pendingUnsyncedParentAccesses.add(path);
             }
         }

package/src/3-path-functions/PathFunctionRunner.ts

@@ -24,6 +24,7 @@ import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
 import type { DeployProgress } from "../4-deploy/deployFunctions";
 import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
 import { PathRouter } from "../0-path-value-core/PathRouter";
+import { FunctionCaptureController, isCapturing, recordCapturedCall } from "./functionCapture";
 setImmediate(() => import("../4-querysub/Querysub"));
 
 let functionCallOuterCount = 0;
@@ -229,6 +230,7 @@ export class PathFunctionRunner {
         filterSelector?: FilterSelector;
     }) {
         SocketFunction.expose(FunctionPreloadController);
+        SocketFunction.expose(FunctionCaptureController);
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
@@ -777,8 +779,26 @@ export class PathFunctionRunner {
                         }
                     },
                 }, {
-                    onWritesCommitted(writes) {
+                    onWritesCommitted(writes, watcher) {
                         finalWrites = writes;
+                        // Capture only runs that actually authored the Result (not nooped/partial runs),
+                        //  so a replayed call maps to exactly one recorded output state.
+                        if (isCapturing()) {
+                            let resultsPathStr = getProxyPath(() => functionSchema()[callSpec.DomainName].PathFunctionRunner[callSpec.ModuleId].Results[callSpec.CallId]);
+                            let resultWrite = writes.find(w => w.path === resultsPathStr);
+                            if (resultWrite) {
+                                recordCapturedCall({
+                                    callSpec,
+                                    functionSpec,
+                                    writes,
+                                    watcher,
+                                    resultsPathStr,
+                                    resultWriteTime: resultWrite.time,
+                                    timeTaken: Date.now() - startTime,
+                                    evalTime,
+                                });
+                            }
+                        }
                     },
                 });
             } catch (e: any) {

package/src/3-path-functions/functionCapture.ts

@@ -0,0 +1,213 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { runInfinitePoll } from "socket-function/src/batching";
+import { timeInSecond } from "socket-function/src/misc";
+import { authorityStorage, compareTime, PathValue, Time } from "../0-path-value-core/pathValueCore";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getDomain } from "../config";
+import { CaptureFileHeader, CaptureRecord, CAPTURE_FORMAT_VERSION, encodeCaptureFile, encodeCaptureRecord } from "./functionCaptureFormat";
+import type { SyncWatcher } from "../2-proxy/PathValueProxyWatcher";
+import type { CallSpec, FunctionSpec } from "./PathFunctionRunner";
+
+// How long to wait after a run commits its Result before treating that run as final and flushing
+//      it into the capture buffer. Long enough to catch a fast rejection (which re-runs the call and
+//      supersedes this Result), but well under the Result path's event expiry, so the re-check below
+//      can still read the value to confirm it's still ours.
+export const CAPTURE_FLUSH_DELAY = timeInSecond * 5;
+const CAPTURE_FLUSH_POLL_INTERVAL = timeInSecond;
+
+export interface CaptureStatus {
+    capturing: boolean;
+    // Records confirmed final + still awaiting the flush delay.
+    callCount: number;
+    finalizedCount: number;
+    pendingCount: number;
+    // Bytes of the finalized record frames (excludes header + pending).
+    byteCount: number;
+    startedAt: number;
+}
+
+interface PendingCapture {
+    record: CaptureRecord;
+    resultsPathStr: string;
+    resultWriteTime: Time;
+}
+
+let capturing = false;
+let captureStartedAt = 0;
+let finalizedFrames: Buffer[] = [];
+let finalizedCount = 0;
+let finalizedBytes = 0;
+let pendingCaptures: PendingCapture[] = [];
+
+export function isCapturing() {
+    return capturing;
+}
+
+function resetCaptureBuffer() {
+    finalizedFrames = [];
+    finalizedCount = 0;
+    finalizedBytes = 0;
+    pendingCaptures = [];
+}
+
+function readValue(pathValue: PathValue): unknown {
+    return pathValueSerializer.getPathValue(pathValue, "noMutate");
+}
+
+export function recordCapturedCall(config: {
+    callSpec: CallSpec;
+    functionSpec: FunctionSpec;
+    writes: PathValue[];
+    watcher: SyncWatcher;
+    resultsPathStr: string;
+    resultWriteTime: Time;
+    timeTaken: number;
+    evalTime: number;
+}): void {
+    if (!capturing) return;
+
+    let { callSpec, functionSpec, writes, watcher } = config;
+
+    let reads: CaptureRecord["reads"] = [];
+    for (let values of watcher.pendingAccesses.values()) {
+        for (let { pathValue } of values.values()) {
+            reads.push({ path: pathValue.path, value: readValue(pathValue), time: pathValue.time });
+        }
+    }
+
+    let undefinedReads: CaptureRecord["undefinedReads"] = [];
+    for (let [readTime, paths] of watcher.pendingEpochAccesses) {
+        for (let path of paths) {
+            undefinedReads.push({ path, readTime });
+        }
+    }
+
+    let parentKeyReads = Array.from(watcher.lastWatches.parentPaths);
+
+    let writeRecords: CaptureRecord["writes"] = writes.map(w => ({
+        path: w.path,
+        value: readValue(w),
+        time: w.time,
+        isTransparent: w.isTransparent,
+        event: w.event,
+    }));
+
+    let record: CaptureRecord = {
+        callId: callSpec.CallId,
+        functionSpec,
+        argsEncoded: callSpec.argsEncoded,
+        runAtTime: callSpec.runAtTime,
+        callerMachineId: callSpec.callerMachineId,
+        callerIP: callSpec.callerIP,
+        reads,
+        undefinedReads,
+        parentKeyReads,
+        writes: writeRecords,
+        timeTaken: config.timeTaken,
+        evalTime: config.evalTime,
+        capturedAt: Date.now(),
+    };
+
+    pendingCaptures.push({
+        record,
+        resultsPathStr: config.resultsPathStr,
+        resultWriteTime: config.resultWriteTime,
+    });
+}
+
+// A record is only kept if the Result path still holds the exact write from this run. If it was
+//      rejected or clobbered by another runner, the current value's time won't match, and we drop it.
+function isStillOurResult(pending: PendingCapture): boolean {
+    let current = authorityStorage.getValueAtOrBeforeTime(pending.resultsPathStr, undefined);
+    if (!current) return false;
+    return compareTime(current.time, pending.resultWriteTime) === 0;
+}
+
+function finalizePending(force: boolean) {
+    if (pendingCaptures.length === 0) return;
+    let now = Date.now();
+    let stillPending: PendingCapture[] = [];
+    for (let pending of pendingCaptures) {
+        if (!force && now - pending.record.capturedAt < CAPTURE_FLUSH_DELAY) {
+            stillPending.push(pending);
+            continue;
+        }
+        if (!isStillOurResult(pending)) continue;
+        let frame = encodeCaptureRecord(pending.record);
+        finalizedFrames.push(frame);
+        finalizedCount++;
+        finalizedBytes += frame.length;
+    }
+    pendingCaptures = stillPending;
+}
+
+let flushPollStarted = false;
+function ensureFlushPoll() {
+    if (flushPollStarted) return;
+    flushPollStarted = true;
+    runInfinitePoll(CAPTURE_FLUSH_POLL_INTERVAL, () => finalizePending(false));
+}
+
+function getStatus(): CaptureStatus {
+    return {
+        capturing,
+        callCount: finalizedCount + pendingCaptures.length,
+        finalizedCount,
+        pendingCount: pendingCaptures.length,
+        byteCount: finalizedBytes,
+        startedAt: captureStartedAt,
+    };
+}
+
+class FunctionCaptureControllerBase {
+    public async startCapture(): Promise<CaptureStatus> {
+        resetCaptureBuffer();
+        captureStartedAt = Date.now();
+        capturing = true;
+        ensureFlushPoll();
+        return getStatus();
+    }
+
+    public async getStatus(): Promise<CaptureStatus> {
+        return getStatus();
+    }
+
+    // Stops capturing, force-finalizes anything still within the flush delay, and returns the
+    //      encoded capture file. The buffer is retained until clearCapture so it can be re-downloaded.
+    public async stopCapture(): Promise<Buffer> {
+        capturing = false;
+        finalizePending(true);
+        let header: CaptureFileHeader = {
+            version: CAPTURE_FORMAT_VERSION,
+            createdAt: Date.now(),
+            domainName: getDomain(),
+            recordCount: finalizedCount,
+        };
+        return encodeCaptureFile(header, finalizedFrames);
+    }
+
+    public async clearCapture(): Promise<CaptureStatus> {
+        capturing = false;
+        resetCaptureBuffer();
+        captureStartedAt = 0;
+        return getStatus();
+    }
+}
+
+export const FunctionCaptureController = SocketFunction.register(
+    "FunctionCaptureController-2f1b8c4a-6e9d-4a71-9b2e-8c5f0d3a1e77",
+    new FunctionCaptureControllerBase(),
+    () => ({
+        startCapture: {},
+        getStatus: {},
+        stopCapture: {},
+        clearCapture: {},
+    }),
+    () => ({
+        hooks: [requiresNetworkTrustHook],
+    }),
+    {
+        noAutoExpose: true,
+    }
+);

package/src/3-path-functions/functionCaptureFormat.ts

@@ -0,0 +1,103 @@
+import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
+import type { Time } from "../0-path-value-core/pathValueCore";
+import type { FunctionSpec } from "./PathFunctionRunner";
+
+// Custom binary container for captured function runs. Layout:
+//      [MAGIC (8 bytes)]
+//      [uint32 LE headerLength][header JSON (utf8)]
+//      repeated: [uint32 LE recordLength][record (CBOR)]
+// The header is JSON (not CBOR) so future readers can pull version + metadata without
+//      understanding the record encoding, which lets us evolve the record format while
+//      staying backwards compatible.
+export const CAPTURE_MAGIC = Buffer.from("QSFNCAP\0", "latin1");
+export const CAPTURE_FORMAT_VERSION = 1;
+
+export interface CaptureFileHeader {
+    version: number;
+    createdAt: number;
+    domainName?: string;
+    nodeId?: string;
+    entryPoint?: string;
+    recordCount: number;
+    // Free-form metadata, so we can add fields without a version bump.
+    [key: string]: unknown;
+}
+
+// One captured, fully-resolved function run. Enough to replay it in isolation and compare
+//      the output state.
+export interface CaptureRecord {
+    callId: string;
+    functionSpec: FunctionSpec;
+    argsEncoded: string;
+    // The explicit timestamp the run executed at. Drives all determinism on replay
+    //      (Querysub.time()/Querysub.nextId() are derived from it).
+    runAtTime: Time;
+    callerMachineId: string;
+    callerIP: string;
+
+    // Values the proxy resolved for each read, keyed by the value's own write time so the
+    //      read can be reconstructed via getValueAtOrBeforeTime on replay.
+    reads: { path: string; value: unknown; time: Time }[];
+    // Reads that resolved to no value (undefined) at the given read time.
+    undefinedReads: { path: string; readTime?: Time }[];
+    // Paths whose child keys were enumerated (getKeys / Object.keys), needed so lookup
+    //      enumeration replays identically.
+    parentKeyReads: string[];
+
+    // Committed writes (the output state to compare against on replay).
+    writes: { path: string; value: unknown; time: Time; isTransparent?: boolean; event?: boolean }[];
+
+    timeTaken: number;
+    evalTime: number;
+    capturedAt: number;
+}
+
+function writeUInt32(value: number): Buffer {
+    let buffer = Buffer.alloc(4);
+    buffer.writeUInt32LE(value, 0);
+    return buffer;
+}
+
+export function encodeCaptureRecord(record: CaptureRecord): Buffer {
+    let body = encodeCborx(record);
+    return Buffer.concat([writeUInt32(body.length), body]);
+}
+
+export function encodeCaptureFile(header: CaptureFileHeader, recordFrames: Buffer[]): Buffer {
+    let headerBuffer = Buffer.from(JSON.stringify(header), "utf8");
+    return Buffer.concat([CAPTURE_MAGIC, writeUInt32(headerBuffer.length), headerBuffer, ...recordFrames]);
+}
+
+export function decodeCaptureFile(buffer: Buffer): { header: CaptureFileHeader; records: CaptureRecord[] } {
+    if (buffer.length < CAPTURE_MAGIC.length + 4) {
+        throw new Error(`Capture file too small to be valid (${buffer.length} bytes)`);
+    }
+    let magic = buffer.subarray(0, CAPTURE_MAGIC.length);
+    if (!magic.equals(CAPTURE_MAGIC)) {
+        throw new Error(`Capture file has invalid magic. Expected ${JSON.stringify(CAPTURE_MAGIC.toString("latin1"))}, was ${JSON.stringify(magic.toString("latin1"))}`);
+    }
+    let offset = CAPTURE_MAGIC.length;
+    let headerLength = buffer.readUInt32LE(offset);
+    offset += 4;
+    if (offset + headerLength > buffer.length) {
+        throw new Error(`Capture file header length (${headerLength}) exceeds file size (${buffer.length})`);
+    }
+    let header = JSON.parse(buffer.subarray(offset, offset + headerLength).toString("utf8")) as CaptureFileHeader;
+    offset += headerLength;
+
+    let records: CaptureRecord[] = [];
+    while (offset < buffer.length) {
+        if (offset + 4 > buffer.length) {
+            throw new Error(`Capture file truncated reading record length at offset ${offset}`);
+        }
+        let recordLength = buffer.readUInt32LE(offset);
+        offset += 4;
+        if (offset + recordLength > buffer.length) {
+            throw new Error(`Capture file truncated reading record body at offset ${offset} (length ${recordLength}, file size ${buffer.length})`);
+        }
+        let record = decodeCborx<CaptureRecord>(buffer.subarray(offset, offset + recordLength));
+        records.push(record);
+        offset += recordLength;
+    }
+    return { header, records };
+}

package/src/3-path-functions/functionReplay.ts

@@ -0,0 +1,272 @@
+import { epochTime, PathValue } from "../0-path-value-core/pathValueCore";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+import { isTransparentValue, PathValueReadSource, proxyWatcher, runWithPathValueReadSource } from "../2-proxy/PathValueProxyWatcher";
+import { getPathFromStr, getParentPathStr } from "../path";
+import { CaptureRecord } from "./functionCaptureFormat";
+import { CallSpec, FunctionSpec, overrideCurrentCall } from "./PathFunctionRunner";
+import { getModuleFromSpec } from "./pathFunctionLoader";
+import { parseArgs } from "./PathFunctionHelpers";
+
+export function makeReplayCallSpec(record: CaptureRecord): CallSpec {
+    return {
+        DomainName: record.functionSpec.DomainName,
+        ModuleId: record.functionSpec.ModuleId,
+        CallId: record.callId,
+        FunctionId: record.functionSpec.FunctionId,
+        argsEncoded: record.argsEncoded,
+        callerMachineId: record.callerMachineId,
+        callerIP: record.callerIP,
+        runAtTime: record.runAtTime,
+    };
+}
+
+function makePathValue(path: string, value: unknown, time: PathValue["time"]): PathValue {
+    return {
+        path,
+        value,
+        time,
+        valid: true,
+        locks: [],
+        lockCount: 0,
+        isValueLazy: false,
+        isTransparent: isTransparentValue(value),
+        canGCValue: value === undefined,
+    };
+}
+
+// A PathValueReadSource backed entirely by a single captured run's reads. Because the capture
+//      recorded every value the run actually read, everything is treated as synced, so a replay
+//      finishes in a single synchronous pass with no real database.
+export function buildReplayReadSource(record: CaptureRecord): PathValueReadSource {
+    let values = new Map<string, PathValue>();
+    for (let read of record.reads) {
+        values.set(read.path, makePathValue(read.path, read.value, read.time));
+    }
+
+    // Direct-parent -> children index, so getKeys()/Object.keys() enumeration replays identically.
+    let parentIndex = new Map<string, Set<string>>();
+    let addToParentIndex = (path: string) => {
+        let parent = getParentPathStr(path);
+        let set = parentIndex.get(parent);
+        if (!set) {
+            set = new Set();
+            parentIndex.set(parent, set);
+        }
+        set.add(path);
+    };
+    for (let read of record.reads) addToParentIndex(read.path);
+    for (let read of record.undefinedReads) addToParentIndex(read.path);
+
+    let overrideStack: Map<string, PathValue>[] = [];
+    let readWithOverrides = (path: string): PathValue | undefined => {
+        for (let i = overrideStack.length - 1; i >= 0; i--) {
+            let value = overrideStack[i].get(path);
+            if (value) return value;
+        }
+        return values.get(path);
+    };
+
+    return {
+        getValueAtOrBeforeTime(pathStr) {
+            let stored = readWithOverrides(pathStr);
+            if (stored) return stored;
+            // Mirror real storage: a synced-but-empty path resolves to an epoch transparent value
+            //  (equivalent to "no value", but synced), so uncaptured reads don't stall the run.
+            return { path: pathStr, valid: true, time: epochTime, locks: [], lockCount: 0, value: undefined, canGCValue: true, isTransparent: true };
+        },
+        isSynced() {
+            return true;
+        },
+        isParentSynced() {
+            return true;
+        },
+        getPathsFromParent(pathStr) {
+            return parentIndex.get(pathStr);
+        },
+        temporaryOverride(overrides, code) {
+            if (!overrides || overrides.length === 0) return code();
+            let map = new Map<string, PathValue>();
+            for (let override of overrides) map.set(override.path, override);
+            overrideStack.push(map);
+            try {
+                return code();
+            } finally {
+                overrideStack.pop();
+            }
+        },
+        DEBUG_hasAnyValues(pathStr) {
+            return values.has(pathStr);
+        },
+    };
+}
+
+export interface ReplayResult {
+    writes: PathValue[];
+    result: unknown;
+}
+
+// Runs the function against a captured run's data in complete isolation, using a dry-run watcher so
+//      nothing is committed. Fully synchronous (the captured data is all "synced").
+export function replayCapturedCall(config: {
+    record: CaptureRecord;
+    baseFunction: Function;
+    args: unknown[];
+}): ReplayResult {
+    let { record, baseFunction, args } = config;
+    let readSource = buildReplayReadSource(record);
+    let callSpec = makeReplayCallSpec(record);
+
+    let captured: ReplayResult | { error: string } | undefined;
+    runWithPathValueReadSource(readSource, () => {
+        proxyWatcher.createWatcher({
+            dryRun: true,
+            canWrite: true,
+            runImmediately: true,
+            allowUnsyncedReads: true,
+            temporary: true,
+            skipPermissionsCheck: true,
+            nestedCalls: "ignore",
+            runAtTime: record.runAtTime,
+            maxLocksOverride: Number.MAX_SAFE_INTEGER,
+            debugName: `replay ${record.functionSpec.ModuleId}.${record.functionSpec.FunctionId} ${record.callId}`,
+            watchFunction: () => {
+                overrideCurrentCall({ spec: callSpec, fnc: record.functionSpec }, () => {
+                    baseFunction(...args);
+                });
+            },
+            onResultUpdated: (result, writes) => {
+                if ("error" in result) {
+                    captured = { error: result.error };
+                } else {
+                    captured = { writes: writes ?? [], result: result.result };
+                }
+            },
+        });
+    });
+
+    if (!captured) {
+        throw new Error(`Replay produced no result for ${record.callId}`);
+    }
+    if ("error" in captured) {
+        throw new Error(`Replay threw for ${record.functionSpec.ModuleId}.${record.functionSpec.FunctionId} (${record.callId}): ${captured.error}`);
+    }
+    return captured;
+}
+
+export interface ReplayComparison {
+    identical: boolean;
+    matched: number;
+    mismatched: { path: string; captured: unknown; replayed: unknown }[];
+    // In the capture, but the replay didn't write it.
+    missing: { path: string; captured: unknown }[];
+    // The replay wrote it, but the capture didn't have it.
+    extra: { path: string; replayed: unknown }[];
+}
+
+function stableStringify(value: unknown): string {
+    return JSON.stringify(value, (_key, val) => {
+        if (val && typeof val === "object" && !Array.isArray(val)) {
+            let sorted: { [key: string]: unknown } = {};
+            for (let key of Object.keys(val).sort()) {
+                sorted[key] = (val as { [key: string]: unknown })[key];
+            }
+            return sorted;
+        }
+        return val;
+    });
+}
+
+// Compares the writes a replay produced against the writes originally captured. Times/locks are
+//      ignored (they're assigned fresh on every run); only path -> value output state is compared.
+export function compareCapturedWrites(record: CaptureRecord, replayWrites: PathValue[]): ReplayComparison {
+    let capturedMap = new Map<string, unknown>();
+    for (let write of record.writes) capturedMap.set(write.path, write.value);
+
+    let replayMap = new Map<string, unknown>();
+    for (let write of replayWrites) replayMap.set(write.path, pathValueSerializer.getPathValue(write));
+
+    let matched = 0;
+    let mismatched: ReplayComparison["mismatched"] = [];
+    let missing: ReplayComparison["missing"] = [];
+    let extra: ReplayComparison["extra"] = [];
+
+    for (let [path, capturedValue] of capturedMap) {
+        if (!replayMap.has(path)) {
+            missing.push({ path, captured: capturedValue });
+            continue;
+        }
+        let replayedValue = replayMap.get(path);
+        if (stableStringify(capturedValue) === stableStringify(replayedValue)) {
+            matched++;
+        } else {
+            mismatched.push({ path, captured: capturedValue, replayed: replayedValue });
+        }
+    }
+    for (let [path, replayedValue] of replayMap) {
+        if (!capturedMap.has(path)) {
+            extra.push({ path, replayed: replayedValue });
+        }
+    }
+
+    return {
+        identical: mismatched.length === 0 && missing.length === 0 && extra.length === 0,
+        matched,
+        mismatched,
+        missing,
+        extra,
+    };
+}
+
+async function loadBaseFunction(spec: FunctionSpec): Promise<Function> {
+    let module = await getModuleFromSpec(spec);
+    let exportObj: any = module.exports;
+    for (let part of getPathFromStr(spec.exportPathStr)) {
+        exportObj = exportObj?.[part];
+    }
+    if (typeof exportObj !== "function") {
+        throw new Error(`Export at ${JSON.stringify(spec.exportPathStr)} in ${spec.FilePath} was not a function (it was ${typeof exportObj}). The captured code may not be loadable in this environment.`);
+    }
+    return exportObj as Function;
+}
+
+export interface ReplayRecordReport {
+    callId: string;
+    functionName: string;
+    ok: boolean;
+    error?: string;
+    comparison?: ReplayComparison;
+}
+
+export interface ReplaySummary {
+    total: number;
+    identical: number;
+    divergent: number;
+    errored: number;
+    reports: ReplayRecordReport[];
+}
+
+// Loads each captured function and replays it in isolation, comparing the replayed writes against
+//      the originally captured writes. Requires the app's modules to be loadable (import its deploy.ts
+//      before calling this, so development modules are registered).
+export async function replayCaptureRecords(records: CaptureRecord[]): Promise<ReplaySummary> {
+    let reports: ReplayRecordReport[] = [];
+    for (let record of records) {
+        let functionName = `${record.functionSpec.ModuleId}.${record.functionSpec.FunctionId}`;
+        try {
+            let baseFunction = await loadBaseFunction(record.functionSpec);
+            let args = parseArgs(makeReplayCallSpec(record));
+            let replay = replayCapturedCall({ record, baseFunction, args });
+            let comparison = compareCapturedWrites(record, replay.writes);
+            reports.push({ callId: record.callId, functionName, ok: comparison.identical, comparison });
+        } catch (e: any) {
+            reports.push({ callId: record.callId, functionName, ok: false, error: String(e?.stack || e) });
+        }
+    }
+    return {
+        total: reports.length,
+        identical: reports.filter(r => r.ok).length,
+        divergent: reports.filter(r => !r.ok && !r.error).length,
+        errored: reports.filter(r => r.error).length,
+        reports,
+    };
+}

package/src/4-querysub/Querysub.ts

@@ -18,7 +18,7 @@ import { RequireController, setRequireBootRequire } from "socket-function/requir
 import { cache, cacheLimited, lazy } from "socket-function/src/caching";
 import { getIdentityCAPromise, getOwnMachineId, getOwnThreadId, getThreadKeyCert, verifyMachineIdForPublicKey } from "sliftutils/misc/https/certs";
 import { getHostedIP, getSNICerts, publishMachineARecords } from "../-e-certs/EdgeCertController";
-import { debugCoreMode, registerGetCompressNetwork, authorityStorage, enableDebugRejections } from "../0-path-value-core/pathValueCore";
+import { debugCoreMode, registerGetCompressNetwork, authorityStorage, enableDebugRejections, getNextTime } from "../0-path-value-core/pathValueCore";
 import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions, doProxyOptions } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
@@ -1204,6 +1204,9 @@ function getSyncedTimeUnique() {
 }
 
 function getSyncedTime() {
+    if (!Querysub.isInSyncedCall()) {
+        return Date.now();
+    }
     let setTime = getSetTime();
     if (setTime !== undefined) {
         return setTime;

package/src/diagnostics/managementPages.tsx

@@ -123,6 +123,17 @@ export async function registerManagementPages2(config: {
         controllerName: "DNSPageController",
         getModule: () => import("./misc-pages/DNSPage"),
     });
+    inputPages.push({
+        title: "Fnc Capture",
+        componentName: "FunctionCapturePage",
+        controllerName: "FunctionCaptureProxyController",
+        getModule: () => import("./misc-pages/FunctionCapturePage"),
+    });
+    inputPages.push({
+        title: "Capture Analyzer",
+        componentName: "FunctionCaptureAnalyzePage",
+        getModule: () => import("./misc-pages/FunctionCaptureAnalyzePage"),
+    });
     inputPages.push({
         title: "LOG VIEWER",
         componentName: "LogViewer3",

package/src/diagnostics/misc-pages/FunctionCaptureAnalyzePage.tsx

@@ -0,0 +1,220 @@
+module.allowclient = true;
+
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { Querysub, t } from "../../4-querysub/Querysub";
+import { Button } from "../../library-components/Button";
+import { formatNumber, formatTime, formatDateTime } from "socket-function/src/formatting/format";
+import { green, red } from "socket-function/src/formatting/logColors";
+import { getPathStr2 } from "../../path";
+import { sort } from "socket-function/src/misc";
+import { CaptureRecord, CaptureFileHeader, decodeCaptureFile } from "../../3-path-functions/functionCaptureFormat";
+
+// How many rows of a read/write list we show before requiring a "Show more" click.
+const ROW_LIMIT = 50;
+
+interface ParsedCapture {
+    header: CaptureFileHeader;
+    records: CaptureRecord[];
+    fileName: string;
+}
+
+export class FunctionCaptureAnalyzePage extends qreact.Component {
+    state = t.state({
+        error: t.string(""),
+        data: t.atomic<ParsedCapture>(),
+        expandedGroups: t.lookup(t.boolean),
+        expandedCalls: t.lookup(t.boolean),
+        // Section id => how many rows to show.
+        rowLimits: t.lookup(t.number),
+    });
+
+    private async loadFile(file: File | undefined) {
+        if (!file) return;
+        try {
+            let buffer = Buffer.from(await file.arrayBuffer());
+            let { header, records } = decodeCaptureFile(buffer);
+            Querysub.localCommit(() => {
+                this.state.data = { header, records, fileName: file.name };
+                this.state.error = "";
+                for (let key of Object.keys(this.state.expandedGroups)) delete this.state.expandedGroups[key];
+                for (let key of Object.keys(this.state.expandedCalls)) delete this.state.expandedCalls[key];
+            });
+            console.log(green(`Loaded capture ${file.name}: ${records.length} records`));
+        } catch (e: any) {
+            Querysub.localCommit(() => {
+                this.state.error = String(e?.stack || e);
+            });
+            console.error(red(`Failed to parse capture: ${e.stack}`));
+        }
+    }
+
+    render() {
+        let data = this.state.data;
+        return <div className={css.pad2(10).vbox(14).fillWidth}>
+            <h1>Function Capture Analyzer</h1>
+            {this.renderDropZone()}
+            {this.state.error && <div className={css.colorhsl(0, 70, 60).whiteSpace("pre-wrap")}>{this.state.error}</div>}
+            {data && this.renderData(data)}
+        </div>;
+    }
+
+    private renderDropZone() {
+        return <div
+            className={css.fillWidth.pad2(24).bord2(210, 30, 45, 2, "dashed").hsl(210, 40, 88).button.textAlign("center")}
+            onDragOver={e => e.preventDefault()}
+            onDrop={e => {
+                e.preventDefault();
+                void this.loadFile(e.dataTransfer?.files?.[0]);
+            }}
+            onClick={() => {
+                let input = document.createElement("input");
+                input.type = "file";
+                input.accept = ".qsfncap";
+                input.onchange = () => void this.loadFile(input.files?.[0]);
+                input.click();
+            }}
+        >
+            Drop a .qsfncap capture file here, or click to choose one.
+        </div>;
+    }
+
+    private renderData(data: ParsedCapture) {
+        let groups = groupRecords(data.records);
+        return <div className={css.vbox(12).fillWidth}>
+            <div className={css.vbox(2)}>
+                <div className={css.fontWeight("bold")}>{data.fileName}</div>
+                <div className={css.fontSize(12).colorhsl(0, 0, 60)}>
+                    version {data.header.version} · {formatNumber(data.records.length)} calls · {data.header.domainName || "?"} · created {formatDateTime(data.header.createdAt)}
+                </div>
+            </div>
+            <div className={css.vbox(1).fillWidth}>
+                {groups.map(group => this.renderGroup(group))}
+            </div>
+        </div>;
+    }
+
+    private renderGroup(group: FunctionGroup) {
+        let expanded = !!this.state.expandedGroups[group.key];
+        return <div className={css.vbox(1).fillWidth.borderBottom("1px solid hsla(0,0%,0%,0.15)")}>
+            <div
+                className={css.hbox(16).fillWidth.pad2(8, 6).button}
+                onClick={() => Querysub.localCommit(() => { this.state.expandedGroups[group.key] = !expanded; })}
+            >
+                <div className={css.width(16)}>{expanded ? "▾" : "▸"}</div>
+                <div className={css.width(420).vbox(2)}>
+                    <div className={css.fontWeight("bold")}>{group.functionName}</div>
+                    <div className={css.fontSize(11).colorhsl(0, 0, 50)}>{group.filePath}</div>
+                </div>
+                <div className={css.width(90)}>{formatNumber(group.records.length)} calls</div>
+                <div className={css.width(140)}>avg {formatTime(group.avgEvalTime)}</div>
+            </div>
+            {expanded &&
+                <div className={css.vbox(1).fillWidth.pad2(24, 0)}>
+                    {group.records.map((record, i) => this.renderCall(group.key, record, i))}
+                </div>
+            }
+        </div>;
+    }
+
+    private renderCall(groupKey: string, record: CaptureRecord, index: number) {
+        let callKey = getPathStr2(groupKey, record.callId);
+        let expanded = !!this.state.expandedCalls[callKey];
+        return <div className={css.vbox(2).fillWidth.borderBottom("1px solid hsla(0,0%,0%,0.1)")}>
+            <div
+                className={css.hbox(16).fillWidth.pad2(6, 4).button}
+                onClick={() => Querysub.localCommit(() => { this.state.expandedCalls[callKey] = !expanded; })}
+            >
+                <div className={css.width(16)}>{expanded ? "▾" : "▸"}</div>
+                <div className={css.width(220).ellipsis}>call #{index + 1} · {record.callId}</div>
+                <div className={css.width(120)}>{formatNumber(record.reads.length)} reads</div>
+                <div className={css.width(120)}>{formatNumber(record.writes.length)} writes</div>
+                <div className={css.width(120)}>eval {formatTime(record.evalTime)}</div>
+                <div className={css.width(180)}>runAt {formatDateTime(record.runAtTime.time)}</div>
+            </div>
+            {expanded && this.renderCallDetail(callKey, record)}
+        </div>;
+    }
+
+    private renderCallDetail(callKey: string, record: CaptureRecord) {
+        return <div className={css.vbox(10).fillWidth.pad2(24, 6)}>
+            <div className={css.fontSize(12).colorhsl(0, 0, 60)}>args: {shortValue(record.argsEncoded)}</div>
+            {this.renderSection(getPathStr2(callKey, "reads"), "Reads", record.reads.map(r => `${r.path}  =  ${shortValue(r.value)}`))}
+            {record.undefinedReads.length > 0 &&
+                this.renderSection(getPathStr2(callKey, "undef"), "Undefined reads", record.undefinedReads.map(r => r.path))}
+            {record.parentKeyReads.length > 0 &&
+                this.renderSection(getPathStr2(callKey, "keys"), "Enumerated (getKeys)", record.parentKeyReads)}
+            {this.renderSection(getPathStr2(callKey, "writes"), "Writes", record.writes.map(w => `${w.path}  =  ${shortValue(w.value)}${w.isTransparent ? "  (transparent)" : ""}`))}
+        </div>;
+    }
+
+    private renderSection(sectionId: string, title: string, rows: string[]) {
+        let limit = ROW_LIMIT;
+        if (sectionId in this.state.rowLimits) {
+            limit = Number(this.state.rowLimits[sectionId]);
+        }
+        let shown = rows.slice(0, limit);
+        let remaining = rows.length - shown.length;
+        return <div className={css.vbox(2).fillWidth}>
+            <div className={css.fontWeight("bold")}>{title} ({formatNumber(rows.length)})</div>
+            <div className={css.vbox(1).fillWidth.fontFamily("monospace").fontSize(12)}>
+                {shown.map(row => <div className={css.whiteSpace("pre-wrap").wordBreak("break-all")}>{row}</div>)}
+            </div>
+            {remaining > 0 &&
+                <Button onClick={() => Querysub.localCommit(() => { this.state.rowLimits[sectionId] = limit + ROW_LIMIT; })}>
+                    Show {Math.min(ROW_LIMIT, remaining)} more ({formatNumber(remaining)} remaining)
+                </Button>
+            }
+        </div>;
+    }
+}
+
+interface FunctionGroup {
+    key: string;
+    functionName: string;
+    filePath: string;
+    avgEvalTime: number;
+    records: CaptureRecord[];
+}
+
+function groupRecords(records: CaptureRecord[]): FunctionGroup[] {
+    let map = new Map<string, CaptureRecord[]>();
+    for (let record of records) {
+        let key = getPathStr2(record.functionSpec.ModuleId, record.functionSpec.FunctionId);
+        let list = map.get(key);
+        if (!list) {
+            list = [];
+            map.set(key, list);
+        }
+        list.push(record);
+    }
+    let groups: FunctionGroup[] = [];
+    for (let [key, list] of map) {
+        let spec = list[0].functionSpec;
+        let totalEval = list.reduce((sum, r) => sum + r.evalTime, 0);
+        let exportName = spec.exportPathStr ? spec.exportPathStr.split("/").slice(1).join(".") : spec.FunctionId;
+        groups.push({
+            key,
+            functionName: `${spec.ModuleId} · ${exportName}`,
+            filePath: spec.FilePath,
+            avgEvalTime: totalEval / list.length,
+            records: list,
+        });
+    }
+    sort(groups, group => -group.records.length);
+    return groups;
+}
+
+function shortValue(value: unknown): string {
+    let text: string;
+    try {
+        text = typeof value === "string" ? value : JSON.stringify(value);
+    } catch {
+        text = String(value);
+    }
+    if (text === undefined) text = String(value);
+    if (text.length > 300) {
+        text = text.slice(0, 300) + `… (${formatNumber(text.length)} chars)`;
+    }
+    return text;
+}

package/src/diagnostics/misc-pages/FunctionCapturePage.tsx

@@ -0,0 +1,192 @@
+module.allowclient = true;
+
+import { qreact } from "../../4-dom/qreact";
+import { SocketFunction } from "socket-function/SocketFunction";
+import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
+import { css } from "typesafecss";
+import { Querysub, t } from "../../4-querysub/Querysub";
+import { Button } from "../../library-components/Button";
+import { ATag } from "../../library-components/ATag";
+import { formatDateTime, formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { green, red } from "socket-function/src/formatting/logColors";
+import { assertIsManagementUser, managementPageURL } from "../managementPages";
+import { getControllerNodeIdList } from "../../-g-core-values/NodeCapabilities";
+import { errorToUndefined } from "../../errors";
+import { CaptureStatus, FunctionCaptureController } from "../../3-path-functions/functionCapture";
+
+interface RunnerStatus {
+    nodeId: string;
+    entryPoint: string;
+    status?: CaptureStatus;
+}
+
+const REFRESH_INTERVAL = 1000;
+
+export class FunctionCapturePage extends qreact.Component {
+    state = t.state({
+        statuses: t.atomic<RunnerStatus[]>(),
+        busy: t.lookup(t.boolean),
+        error: t.string(""),
+    });
+
+    private intervalId: ReturnType<typeof setInterval> | undefined;
+
+    componentDidMount() {
+        void this.refresh();
+        this.intervalId = setInterval(() => void this.refresh(), REFRESH_INTERVAL);
+    }
+    componentWillUnmount() {
+        if (this.intervalId) clearInterval(this.intervalId);
+    }
+
+    private async refresh() {
+        try {
+            let statuses = await FunctionCaptureProxyController.nodes[getBrowserUrlNode()].getStatuses();
+            Querysub.localCommit(() => {
+                this.state.statuses = statuses;
+                this.state.error = "";
+            });
+        } catch (e: any) {
+            Querysub.localCommit(() => {
+                this.state.error = String(e?.stack || e);
+            });
+        }
+    }
+
+    render() {
+        let statuses = this.state.statuses;
+        return <div className={css.pad2(10).vbox(14).fillWidth}>
+            <h1>Function Capture</h1>
+            <div>Turn on capturing to record fully-resolved function runs (reads, writes, timings) on a runner. Stop to download a capture file, then analyze it in the <ATag values={[{ param: managementPageURL, value: "FunctionCaptureAnalyzePage" }]}>Capture Analyzer</ATag>.</div>
+            {this.state.error && <div className={css.colorhsl(0, 70, 60)}>{this.state.error}</div>}
+            {!statuses && <div>Loading runners…</div>}
+            {statuses && statuses.length === 0 && <div>No function runners found.</div>}
+            {statuses && statuses.length > 0 &&
+                <div className={css.vbox(1).fillWidth}>
+                    <div className={css.hbox(20).fillWidth.borderBottom("1px solid hsla(0,0%,0%,0.3)").pad2(6, 4).fontWeight("bold")}>
+                        <div className={css.width(280)}>Runner</div>
+                        <div className={css.width(90)}>Capturing</div>
+                        <div className={css.width(90)}>Calls</div>
+                        <div className={css.width(120)}>Size</div>
+                        <div className={css.width(140)}>Started</div>
+                        <div>Actions</div>
+                    </div>
+                    {statuses.map(runner => this.renderRunner(runner))}
+                </div>
+            }
+        </div>;
+    }
+
+    private renderRunner(runner: RunnerStatus) {
+        let status = runner.status;
+        let busy = runner.nodeId in this.state.busy;
+        return <div className={css.hbox(20).fillWidth.borderBottom("1px solid hsla(0,0%,0%,0.15)").pad2(6, 4)}>
+            <div className={css.width(280).vbox(2)}>
+                <div>{runner.entryPoint}</div>
+                <div className={css.fontSize(11).colorhsl(0, 0, 45).ellipsis.maxWidth(280)}>{runner.nodeId}</div>
+            </div>
+            <div className={css.width(90)}>{!status ? "?" : status.capturing ? "● live" : "off"}</div>
+            <div className={css.width(90)}>{status ? formatNumber(status.callCount) : "-"}{status && status.pendingCount > 0 && ` (+${status.pendingCount})`}</div>
+            <div className={css.width(120)}>{status ? `${formatNumber(status.byteCount)} B` : "-"}</div>
+            <div className={css.width(140)}>{status && status.startedAt ? formatTime(Date.now() - status.startedAt) + " ago" : "-"}</div>
+            <div className={css.hbox(8)}>
+                {status && !status.capturing &&
+                    <Button disabled={busy} onClick={() => this.runAction(runner, "start")}>Start</Button>
+                }
+                {status && status.capturing &&
+                    <Button disabled={busy} onClick={() => this.runAction(runner, "stop")}>Stop &amp; Download</Button>
+                }
+                {status && !status.capturing && (status.callCount > 0) &&
+                    <Button disabled={busy} onClick={() => this.downloadCapture(runner)}>Download</Button>
+                }
+                {status && !status.capturing && (status.callCount > 0) &&
+                    <Button disabled={busy} onClick={() => this.runAction(runner, "clear")}>Clear</Button>
+                }
+            </div>
+        </div>;
+    }
+
+    private async runAction(runner: RunnerStatus, action: "start" | "stop" | "clear") {
+        if (Querysub.fastRead(() => runner.nodeId in this.state.busy)) return;
+        Querysub.localCommit(() => { this.state.busy[runner.nodeId] = true; });
+        try {
+            let proxy = FunctionCaptureProxyController.nodes[getBrowserUrlNode()];
+            if (action === "start") {
+                await proxy.startCapture({ nodeId: runner.nodeId });
+            } else if (action === "clear") {
+                await proxy.clearCapture({ nodeId: runner.nodeId });
+            } else if (action === "stop") {
+                let buffer = await proxy.stopCapture({ nodeId: runner.nodeId });
+                downloadBuffer(buffer, runner);
+            }
+            console.log(green(`${action} capture on ${runner.entryPoint}`));
+        } catch (e: any) {
+            console.error(red(`Failed to ${action} capture on ${runner.nodeId}: ${e.stack}`));
+        } finally {
+            Querysub.localCommit(() => { delete this.state.busy[runner.nodeId]; });
+            void this.refresh();
+        }
+    }
+
+    private async downloadCapture(runner: RunnerStatus) {
+        if (Querysub.fastRead(() => runner.nodeId in this.state.busy)) return;
+        Querysub.localCommit(() => { this.state.busy[runner.nodeId] = true; });
+        try {
+            let buffer = await FunctionCaptureProxyController.nodes[getBrowserUrlNode()].stopCapture({ nodeId: runner.nodeId });
+            downloadBuffer(buffer, runner);
+        } catch (e: any) {
+            console.error(red(`Failed to download capture on ${runner.nodeId}: ${e.stack}`));
+        } finally {
+            Querysub.localCommit(() => { delete this.state.busy[runner.nodeId]; });
+            void this.refresh();
+        }
+    }
+}
+
+function downloadBuffer(buffer: Buffer, runner: RunnerStatus) {
+    let blob = new Blob([buffer]);
+    let url = URL.createObjectURL(blob);
+    let anchor = document.createElement("a");
+    anchor.href = url;
+    let label = runner.entryPoint.replace(/[^A-Za-z0-9_.-]+/g, "_");
+    anchor.download = `function-capture-${label}-${formatDateTime(Date.now())}.qsfncap`;
+    document.body.appendChild(anchor);
+    anchor.click();
+    document.body.removeChild(anchor);
+    URL.revokeObjectURL(url);
+}
+
+// Management-side proxy: pages can't talk to runner processes directly, so this endpoint (running
+//      where management pages are hosted) fans out to the runner nodes over trusted server-to-server calls.
+class FunctionCaptureProxyControllerBase {
+    public async getStatuses(): Promise<RunnerStatus[]> {
+        let runners = await getControllerNodeIdList(FunctionCaptureController);
+        return await Promise.all(runners.map(async runner => {
+            let status = await errorToUndefined(FunctionCaptureController.nodes[runner.nodeId].getStatus());
+            return { nodeId: runner.nodeId, entryPoint: runner.entryPoint, status };
+        }));
+    }
+    public async startCapture(config: { nodeId: string }): Promise<CaptureStatus> {
+        return await FunctionCaptureController.nodes[config.nodeId].startCapture();
+    }
+    public async clearCapture(config: { nodeId: string }): Promise<CaptureStatus> {
+        return await FunctionCaptureController.nodes[config.nodeId].clearCapture();
+    }
+    public async stopCapture(config: { nodeId: string }): Promise<Buffer> {
+        return await FunctionCaptureController.nodes[config.nodeId].stopCapture();
+    }
+}
+
+export const FunctionCaptureProxyController = SocketFunction.register(
+    "FunctionCaptureProxyController-9c4e2b71-5a83-4d19-8e0f-3b6a7c2d1f42",
+    new FunctionCaptureProxyControllerBase(),
+    () => ({
+        getStatuses: {},
+        startCapture: {},
+        clearCapture: {},
+        stopCapture: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+);