querysub 0.451.0 → 0.453.0

package/src/-g-core-values/NodeCapabilities.ts

@@ -12,6 +12,7 @@ import { delay } from "socket-function/src/batching";
 import debugbreak from "debugbreak";
 import { formatTime } from "socket-function/src/formatting/format";
 import type { DebugFunctionShardInfo } from "../3-path-functions/PathFunctionRunner";
+import type { AuthoritySpec } from "../0-path-value-core/PathRouter";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { isNoNetwork } from "../config";
 import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
@@ -94,10 +95,10 @@ export async function getControllerNodeIdList(
     await Promise.all(nodeIdsToTest.map(async nodeId => {
         let result = await doesNodeExposeController(nodeId, controller);
         if (result) {
-            let entryPoint = await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
+            let metadata = await NodeCapabilitiesController.nodes[nodeId].getMetadata();
             passedNodeIds.set(nodeId, {
                 machineId: getMachineId(nodeId),
-                entryPoint,
+                entryPoint: metadata.entryPoint,
             });
         }
     }));
@@ -117,38 +118,43 @@ export async function getControllerNodeIdList(
 
 
 export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<boolean> {
-    let exposedControllers = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getExposedControllers());
+    let metadata = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getMetadata());
 
-    if (exposedControllers?.includes(controller._classGuid)) {
+    if (metadata?.exposedControllers.includes(controller._classGuid)) {
         return true;
     }
     return false;
 }
 
 const startupTime = Date.now();
+
+export type NodeMetadata = {
+    entryPoint: string;
+    startupTime: number;
+    processId: number;
+    functionRunnerShards: DebugFunctionShardInfo[];
+    authoritySpec: AuthoritySpec;
+    exposedControllers: string[];
+    trueTimeOffset: number;
+};
+
 class NodeCapabilitiesControllerBase {
-    public async getExposedControllers() {
-        return Array.from(SocketFunction.exposedClasses);
-    }
-    public async getEntryPoint() {
-        return process.argv[1];
-    }
-    public async getStartupTime() {
-        return startupTime;
+    public async getMetadata(): Promise<NodeMetadata> {
+        let { authorityLookup } = await import("../0-path-value-core/AuthorityLookup");
+        return {
+            entryPoint: process.argv[1],
+            startupTime,
+            processId: process.pid,
+            functionRunnerShards: getFunctionRunnerShards(),
+            authoritySpec: authorityLookup.getOurSpec(),
+            exposedControllers: Array.from(SocketFunction.exposedClasses),
+            trueTimeOffset: getTrueTimeOffset(),
+        };
     }
+
     public async getMemoryUsage() {
         return process.memoryUsage();
     }
-    public async getProcessId() {
-        return process.pid;
-    }
-    public async getFunctionRunnerShards() {
-        return getFunctionRunnerShards();
-    }
-
-    public async getTrueTimeOffset() {
-        return getTrueTimeOffset();
-    }
 
     public async getInspectURL() {
         return await getDebuggerUrl();
@@ -188,13 +194,8 @@ export const NodeCapabilitiesController = SocketFunction.register(
     "NodeCapabilitiesController-399b7067-75c3-4d92-8be3-8470bde32d3c",
     new NodeCapabilitiesControllerBase(),
     () => ({
-        getExposedControllers: {},
-        getEntryPoint: {},
-        getStartupTime: {},
+        getMetadata: {},
         getMemoryUsage: {},
-        getProcessId: {},
-        getFunctionRunnerShards: {},
-        getTrueTimeOffset: {},
         getInspectURL: { hooks: [requiresNetworkTrustHook] },
         exposeExternalDebugPortOnce: { hooks: [requiresNetworkTrustHook] },
     }),

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -245,7 +245,31 @@ export function createArchiveLocker2(config: {
                     newTransaction.ops.push({ type: "delete", key: obj.file });
                 }
             }
+            // Per-op trace so a "written too slowly" crash on a specific file can be
+            //  correlated against the exact transaction that touched it. The aggregate
+            //  "Joining N => M" log doesn't name the files. We log before the attempt and
+            //  again after with the resulting status (including "rejected").
+            const describeOp = (op: typeof newTransaction.ops[number]): string => {
+                let detail = "";
+                if (op.key.endsWith(".data") || op.key.endsWith(".data.locked")) {
+                    try {
+                        detail = ` source=${pathValueArchives.decodeDataPath(op.key).sourceType}`;
+                    } catch (e) {
+                        detail = ` decodeError=${(e as Error).message ?? e}`;
+                    }
+                }
+                return `${op.type} ${op.key}${detail}`;
+            };
+            for (let op of newTransaction.ops) {
+                console.info(`Transaction attempt: ${describeOp(op)}`);
+            }
+
             let status = await locker.addTransaction(newTransaction);
+
+            for (let op of newTransaction.ops) {
+                console.info(`Transaction ${status}: ${describeOp(op)}`);
+            }
+
             if (status === "accepted") {
                 let newFiles = new Set<string>();
                 for (let file of files) {

package/src/0-path-value-core/pathValueCore.ts

@@ -68,7 +68,7 @@ export const MAX_CHANGE_AGE = MAX_ACCEPTED_CHANGE_AGE * 2;
 /** Extra time we keep clientside prediction rejections for, to give us time to receive the actual values. */
 export const CLIENTSIDE_PREDICT_LEEWAY = 500;
 
-/** Any PathValues which take longer than this to wrist should be rejected, so... we have
+/** Any PathValues which take longer than this to write should be rejected, so... we have
  *      to write well before this time.
  *      - This has to be at least MAX_CHANGE_AGE * 4.5 + the time to serialize and
  *          send our data to remote storage.

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -1472,8 +1472,8 @@ export class PathValueProxyWatcher {
                                 let notWatchingUnsyncedParent = reallyUnsyncedParentAccesses.filter(x => !remoteWatcher.debugIsWatchingPath(x));
                                 if (notWatchingUnsynced.length !== 0 || notWatchingUnsyncedParent.length !== 0) {
                                     console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("NOT REMOTE WATCHING REQUIRED PATHS")}. This means our sync or unsync (likely unsync) logic is broken, in remoteWatcher/clientWatcher. OR, there were no read nodes when we tried to sync (we don't handle missing read nodes correctly at the moment)`), { notWatchingUnsynced, notWatchingUnsyncedParent }, watcher.options.watchFunction);
-                                    debugbreak(2);
-                                    debugger;
+                                    // debugbreak(2);
+                                    // debugger;
                                 } else {
                                     console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT RECEIVE PATH VALUES")}. This means PathValueServer is not responding to watches, either to specific paths, or for all paths`), { reallyUnsyncedAccesses, reallyUnsyncedParentAccesses }, watcher.options.watchFunction);
                                     // debugbreak(2);
@@ -1481,12 +1481,12 @@ export class PathValueProxyWatcher {
                                 }
                             } else if (watcher.lastSpecialPromiseUnsynced) {
                                 console.warn((`${yellow("WATCHER SLOW TO SYNC")} ${watcher.debugName} ${magenta("DEPENDENT PROMISE NEVER RESOLVED")}. This promise might resolve, but it probably won't. Slow promises should be detached from the watcher system and use multiple watchers/writes, instead of blocking on a promise.`), watcher.lastSpecialPromiseUnsyncedReason, watcher.options.watchFunction);
-                                debugbreak(2);
-                                debugger;
+                                // debugbreak(2);
+                                // debugger;
                             } else {
                                 console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT TRIGGER WATCHER")}. This means either ProxyWatcher is broken (and isn't triggering when it should, or isn't watching when it should), or ClientWatcher/PathWatcher are broken and are not properly informing callers of watchers.`), { lastUnsyncedAccesses: watcher.lastUnsyncedAccesses, lastUnsyncedParentAccesses: watcher.lastUnsyncedParentAccesses }, watcher.options.watchFunction);
-                                debugbreak(2);
-                                debugger;
+                                // debugbreak(2);
+                                // debugger;
                             }
                         }, 60000);
                     }

package/src/4-querysub/Querysub.ts

@@ -44,7 +44,7 @@ import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { logMeasureTable, measureBlock, measureFnc, measureWrap, startMeasure } from "socket-function/src/profiling/measure";
 import { delay } from "socket-function/src/batching";
 import { MaybePromise } from "socket-function/src/types";
-import { devDebugbreak, getDomain, isBootstrapOnly, isDynamicallyLoading, isPublic, noSyncing } from "../config";
+import { devDebugbreak, getDomain, isBootstrapOnly, isDynamicallyLoading, isPublic, isRecovery, noSyncing } from "../config";
 import { Schema2, Schema2T, t } from "../2-proxy/schema2";
 import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
 import yargs, { check } from "yargs";
@@ -1035,19 +1035,21 @@ export class Querysub {
             return module;
         }
 
-        RequireController.addMapGetModules(async (result, args) => {
-            let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
-            if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
-                await isAllowedToSeeSource(configObj?.signedIdentity);
-                //console.log(red(`Not allowed to see source`));
-                for (let [key, value] of Object.entries(result.modules)) {
-                    result.modules[key] = stripSource(value);
+        if (!isRecovery()) {
+            RequireController.addMapGetModules(async (result, args) => {
+                let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
+                if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
+                    await isAllowedToSeeSource(configObj?.signedIdentity);
+                    //console.log(red(`Not allowed to see source`));
+                    for (let [key, value] of Object.entries(result.modules)) {
+                        result.modules[key] = stripSource(value);
+                    }
+                } else {
+                    //console.log(green(`Allowed to see source`));
                 }
-            } else {
-                //console.log(green(`Allowed to see source`));
-            }
-            return result;
-        });
+                return result;
+            });
+        }
     }
 
     public static async hostService(name: string, port = 0) {

package/src/archiveapps/archiveGCEntry.tsx

@@ -29,6 +29,7 @@ async function main() {
     await Querysub.hostService("gc");
 
     if (yargObj.watch) {
+        console.log("Running in watch mode.");
         await runInfinitePollCallAtStart(timeInDay, runAliveCheckerIteration);
     } else {
         try {

package/src/archiveapps/archiveJoinEntry.ts

@@ -2,7 +2,7 @@ import "../inject";
 
 import { logErrors } from "../errors";
 import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
-import { PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
+import { ARCHIVE_FLUSH_LIMIT, PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
 import { runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
@@ -41,7 +41,12 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
                 valueFiles = valueFiles.filter(x => {
                     let obj = pathValueArchives.decodeDataPath(x.file);
                     if (!obj.minTime) return false;
-                    return obj.sourceType === "genesis";
+                    if (obj.sourceType === "genesis") {
+                        // Has to be old enough
+                        return Date.now() - obj.time < ARCHIVE_FLUSH_LIMIT;
+                    }
+                    // Anything else can be merged immediately.
+                    return true;
                 });
             }
             let withinTimeRangeCount = valueFiles.length;
@@ -123,6 +128,7 @@ async function main() {
     await Querysub.hostService("join");
 
     if (yargObj.watch) {
+        console.log("Running in watch mode.");
         await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
     } else {
         try {

package/src/deployManager/LaunchTrackingHeader.tsx

@@ -0,0 +1,65 @@
+module.allowclient = true;
+
+import { qreact } from "../4-dom/qreact";
+import { css } from "typesafecss";
+import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
+import { MachineServiceController } from "./machineSchema";
+
+const SINCE_DAYS = 2;
+const CRASH_HUE = 0;
+const OTHER_HUE = 210;
+
+export class LaunchTrackingHeader extends qreact.Component {
+    render() {
+        let summaries = MachineServiceController(getBrowserUrlNode()).getRecentLaunches(SINCE_DAYS);
+        if (!summaries) return undefined;
+        if (summaries.length === 0) return undefined;
+
+        let totalCrashes = 0;
+        let totalOther = 0;
+        let perKey = new Map<string, { crashes: number; other: number }>();
+        for (let s of summaries) {
+            let isCrash = s.reason === "crashed";
+            if (isCrash) totalCrashes++;
+            else totalOther++;
+            let entry = perKey.get(s.serviceKey);
+            if (!entry) {
+                entry = { crashes: 0, other: 0 };
+                perKey.set(s.serviceKey, entry);
+            }
+            if (isCrash) entry.crashes++;
+            else entry.other++;
+        }
+
+        let ranked: { serviceKey: string; crashes: number; other: number }[] = [];
+        for (let [serviceKey, c] of perKey) {
+            ranked.push({ serviceKey, crashes: c.crashes, other: c.other });
+        }
+        ranked.sort((a, b) => {
+            if (b.crashes !== a.crashes) return b.crashes - a.crashes;
+            return (b.crashes + b.other) - (a.crashes + a.other);
+        });
+        let top = ranked[0];
+
+        let title = `Launches in last ${SINCE_DAYS} days: ${totalCrashes} crashed, ${totalOther} other`;
+        for (let r of ranked) {
+            title += `\n${r.serviceKey}: ${r.crashes} crashed, ${r.other} other`;
+        }
+
+        return <div title={title} className={css.hbox(6).colorhsl(0, 0, 20)}>
+            <span>🚀</span>
+            <span>
+                <span className={css.colorhsl(CRASH_HUE, 70, 35).boldStyle}>{totalCrashes}</span>
+                <span className={css.colorhsl(0, 0, 55)}>|</span>
+                <span className={css.colorhsl(OTHER_HUE, 65, 35).boldStyle}>{totalOther}</span>
+            </span>
+            {top && top.crashes > 0 &&
+                <span className={css.colorhsl(0, 0, 40)}>
+                    (<span className={css.colorhsl(CRASH_HUE, 70, 35)}>{top.crashes}</span>
+                    <span className={css.colorhsl(0, 0, 55)}>|</span>
+                    <span className={css.colorhsl(OTHER_HUE, 65, 35)}>{top.other}</span>)
+                </span>
+            }
+        </div>;
+    }
+}

package/src/deployManager/machineApplyMainCode.ts

@@ -4,7 +4,7 @@ import { measureWrap } from "socket-function/src/profiling/measure";
 import { getOwnMachineId } from "../-a-auth/certs";
 import { forceRemoveNode, getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { Querysub } from "../4-querysub/QuerysubController";
-import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs } from "./machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs, recordLaunch } from "./machineSchema";
 import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
@@ -183,25 +183,21 @@ export async function streamScreenOutput(config: {
             const pipeFile = `${root}${screenName}/pipe.txt`;
             const tailScript = `${root}${screenName}/smart_tail.sh`;
 
-            // Create a smart tail script that handles file truncation
+            // Create a smart tail script that handles file truncation. It does
+            // NOT emit the initial backlog - that is read and sent directly by
+            // this process (as a single message) below. The script is given a
+            // start byte offset and only emits content appended after it.
             await fs.promises.writeFile(tailScript, `#!/bin/bash
 PIPE_FILE="$1"
+START_POS="$2"
 
-# Initialize position tracking
-CURRENT_POS=0
+# Position tracking starts at the offset we already read directly.
+CURRENT_POS=$START_POS
+CURRENT_SIZE=$START_POS
 LAST_MTIME=""
 
-# Read initial content and get file size
 if [ -f "$PIPE_FILE" ]; then
-    CURRENT_SIZE=$(stat -c%s "$PIPE_FILE" 2>/dev/null || wc -c < "$PIPE_FILE")
     LAST_MTIME=$(stat -c%Y "$PIPE_FILE" 2>/dev/null || stat -f%m "$PIPE_FILE" 2>/dev/null || echo "0")
-    # Output initial content like tail would
-    if [ $CURRENT_SIZE -gt 0 ]; then
-        cat "$PIPE_FILE"
-        CURRENT_POS=$CURRENT_SIZE
-    fi
-else
-    CURRENT_SIZE=0
 fi
 
 # Poll for file changes every 250ms
@@ -237,8 +233,21 @@ done`);
 
             await runPromise(`chmod +x ${tailScript}`);
 
-            // Use our smart tail script instead of regular tail
-            childProcess = spawn("bash", [tailScript, pipeFile], {
+            // Read the existing backlog ourselves and deliver it as a single
+            // onData call. If we let the tail script cat it, the stream chunks
+            // it and runInSerial dribbles it out one round-trip at a time.
+            let initialContent = "";
+            let initialByteSize = 0;
+            try {
+                initialContent = await fs.promises.readFile(pipeFile, "utf8");
+                initialByteSize = Buffer.byteLength(initialContent, "utf8");
+            } catch {
+                // pipe.txt may not exist yet; the tail script will pick it up.
+            }
+
+            // The tail script emits only content appended after initialByteSize,
+            // so the backlog we just read is never sent twice.
+            childProcess = spawn("bash", [tailScript, pipeFile, String(initialByteSize)], {
                 stdio: "pipe",
             });
 
@@ -262,6 +271,13 @@ done`);
                 started.reject(err);
             });
 
+            if (initialContent) {
+                // Queued synchronously here, before any stdout 'data' event can
+                // fire, so the backlog is always delivered ahead of new output.
+                started.resolve();
+                void onDataWrapped(initialContent);
+            }
+
             await started.promise;
         } catch (e) {
             void stop();
@@ -725,6 +741,21 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
 
                 await fs.promises.writeFile(parameterPath, newParametersString);
 
+                let launchReason: "crashed" | "update";
+                if (!sameParameters) {
+                    launchReason = "update";
+                } else {
+                    launchReason = "crashed";
+                }
+                void recordLaunch({
+                    serviceId: config.serviceId,
+                    serviceKey: config.parameters.key,
+                    screenName,
+                    machineId,
+                    reason: launchReason,
+                    time: Date.now(),
+                });
+
                 await runScreenCommand({
                     screenName,
                     command: config.parameters.command,
@@ -839,6 +870,92 @@ async function getPPID(pid: string) {
     }
 }
 
+// Node prints this when the process has exited but the V8 inspector is still
+//  holding it open waiting for a debugger client to detach. If a debugger client
+//  drops uncleanly the process can sit on this forever, holding the screen and
+//  preventing the supervisor from spawning a fresh instance. The watcher below
+//  polls each screen's pipe.txt for this phrase and sends Ctrl+C to break it out.
+const DEBUGGER_DISCONNECT_MESSAGE = "Waiting for the debugger to disconnect...";
+
+// To avoid false positives from app code that happens to log a string
+//  containing the phrase, the wedge is only reported when:
+//    1. the phrase is on its own line (start-of-string or after a newline), AND
+//    2. nothing else has been logged after it (only trailing whitespace/EOF).
+//  In a real wedge, this line is the very last thing Node writes before going
+//  silent — so any later output rules out the wedge interpretation.
+const DEBUGGER_DISCONNECT_TAIL_PATTERN = /(?:^|\r?\n)Waiting for the debugger to disconnect\.\.\.\s*$/;
+
+// How often the wedge watcher scans every service's pipe.txt.
+const DEBUGGER_WEDGE_POLL_INTERVAL = timeInSecond * 15;
+// How much of the tail of pipe.txt to read on each scan. The phrase is short
+//  and appears near the very end of the log when a process is stuck, so a few
+//  KB is plenty and keeps scans cheap.
+const DEBUGGER_WEDGE_TAIL_BYTES = 8 * 1024;
+// Suppress repeated Ctrl+C bursts to the same screen while it tears down; if
+//  the first signal didn't break the wait, give it this long before we try again.
+const DEBUGGER_WEDGE_RESIGNAL_DELAY = timeInMinute;
+// After a first match, wait this long and re-check before signaling. If the
+//  process is still alive and just happened to log a message ending in the
+//  phrase, more output will appear in the meantime and the second check fails.
+const DEBUGGER_WEDGE_RECHECK_DELAY = timeInSecond * 3;
+
+// screenName -> last time we sent Ctrl+C because of a debugger wedge.
+const lastDebuggerWedgeSignal = new Map<string, number>();
+
+async function readPipeFileTail(pipeFile: string, maxBytes: number): Promise<string> {
+    let handle: fs.promises.FileHandle | undefined;
+    try {
+        handle = await fs.promises.open(pipeFile, "r");
+        let stat = await handle.stat();
+        let start = Math.max(0, stat.size - maxBytes);
+        let length = stat.size - start;
+        if (length <= 0) return "";
+        let buf = Buffer.alloc(length);
+        await handle.read(buf, 0, length, start);
+        return buf.toString("utf8");
+    } catch {
+        // File doesn't exist yet, or got truncated mid-read — treat as empty.
+        return "";
+    } finally {
+        if (handle) {
+            try {
+                await handle.close();
+            } catch {
+                // ignore
+            }
+        }
+    }
+}
+
+async function unwedgeStuckDebuggerScreens(): Promise<void> {
+    let prefix = getTmuxPrefix();
+    let screens = await getScreenState(false);
+    for (let { screenName } of screens) {
+        let pipeFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/pipe.txt";
+        let tail1 = await readPipeFileTail(pipeFile, DEBUGGER_WEDGE_TAIL_BYTES);
+        if (!DEBUGGER_DISCONNECT_TAIL_PATTERN.test(tail1)) continue;
+
+        // Confirm the process is actually wedged and not just briefly idle by
+        //  re-checking after a short delay. A live process will write more
+        //  output in this window, which moves the phrase away from the end
+        //  and fails the second match.
+        await delay(DEBUGGER_WEDGE_RECHECK_DELAY);
+        let tail2 = await readPipeFileTail(pipeFile, DEBUGGER_WEDGE_TAIL_BYTES);
+        if (!DEBUGGER_DISCONNECT_TAIL_PATTERN.test(tail2)) continue;
+
+        let last = lastDebuggerWedgeSignal.get(screenName) ?? 0;
+        if (Date.now() - last < DEBUGGER_WEDGE_RESIGNAL_DELAY) continue;
+        lastDebuggerWedgeSignal.set(screenName, Date.now());
+
+        console.warn(red(`Detected stuck "${DEBUGGER_DISCONNECT_MESSAGE}" in ${screenName} (confirmed across two checks ${DEBUGGER_WEDGE_RECHECK_DELAY}ms apart); sending Ctrl+C to unblock it.`));
+        try {
+            await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
+        } catch (e: any) {
+            console.warn(`Failed to send Ctrl+C to ${screenName}: ${e.stack ?? e}`);
+        }
+    }
+}
+
 export async function machineApplyMain() {
     let parentPID = process.argv[2];
     // Wait for the console to get shimmed
@@ -867,6 +984,14 @@ export async function machineApplyMain() {
     await Querysub.hostService("machine-apply");
     onServiceConfigChange(resyncServices);
 
+    runInfinitePoll(DEBUGGER_WEDGE_POLL_INTERVAL, async () => {
+        try {
+            await unwedgeStuckDebuggerScreens();
+        } catch (e: any) {
+            console.error(`Error in debugger-wedge watcher: ${e.stack ?? e}`);
+        }
+    });
+
     runInfinitePoll(timeInMinute * 3, async () => {
         //console.log(magenta(`Quick outdated check at ${new Date().toISOString()}`));
         // console.log(magenta("Likely outdated, resyncing now"));

package/src/deployManager/machineSchema.ts

@@ -1,4 +1,4 @@
-import { isNodeTrue, list, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { isNodeTrue, list, timeInDay, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { nestArchives } from "../-a-archives/archives";
 import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
 import { getDomain } from "../config";
@@ -103,6 +103,69 @@ export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machin
 export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
 export const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("machines/machine-configs/", getArchivesBackblaze(getDomain())));
 
+export type LaunchRecord = {
+    serviceId: string;
+    serviceKey: string;
+    screenName: string;
+    machineId: string;
+    reason: "crashed" | "update";
+    time: number;
+};
+
+export type LaunchSummary = {
+    time: number;
+    reason: string;
+    serviceKey: string;
+    key: string;
+};
+
+const launches = lazy(() => nestArchives("machines/launches/", getArchivesBackblaze(getDomain())));
+
+function formatLaunchDay(time: number): string {
+    let date = new Date(time);
+    let y = date.getUTCFullYear();
+    let m = String(date.getUTCMonth() + 1).padStart(2, "0");
+    let d = String(date.getUTCDate()).padStart(2, "0");
+    return `${y}-${m}-${d}`;
+}
+
+export async function recordLaunch(record: LaunchRecord) {
+    let day = formatLaunchDay(record.time);
+    let key = `${day}/${record.time}_${record.reason}_${record.serviceKey}`;
+    await launches().set(key, Buffer.from(JSON.stringify(record)));
+}
+
+function parseLaunchKey(key: string): LaunchSummary {
+    let rest = key;
+    let slash = key.indexOf("/");
+    if (slash >= 0) {
+        rest = key.slice(slash + 1);
+    }
+    let timeStr = "";
+    let afterTime = rest;
+    let firstUnderscore = rest.indexOf("_");
+    if (firstUnderscore >= 0) {
+        timeStr = rest.slice(0, firstUnderscore);
+        afterTime = rest.slice(firstUnderscore + 1);
+    }
+    let reason = afterTime;
+    let serviceKey = "";
+    let secondUnderscore = afterTime.indexOf("_");
+    if (secondUnderscore >= 0) {
+        reason = afterTime.slice(0, secondUnderscore);
+        serviceKey = afterTime.slice(secondUnderscore + 1);
+    }
+    let time = Number(timeStr);
+    if (!Number.isFinite(time)) {
+        console.warn(`Unparseable launch key (bad time): ${key}`);
+        time = Date.now();
+    }
+    if (reason !== "crashed" && reason !== "update") {
+        console.warn(`Launch key has unexpected reason "${reason}": ${key}`);
+    }
+    return { time, reason, serviceKey, key };
+}
+
 export function doRegisterNodeForMachineCleanup() {
     if (isNode()) {
         void SocketFunction.mountPromise.finally(() => {
@@ -371,6 +434,22 @@ export class MachineServiceControllerBase {
         });
     }
 
+    public async getRecentLaunches(sinceDays: number): Promise<LaunchSummary[]> {
+        let now = Date.now();
+        let dayStrings: string[] = [];
+        for (let i = 0; i <= sinceDays; i++) {
+            dayStrings.push(formatLaunchDay(now - i * timeInDay));
+        }
+        let keyLists = await Promise.all(dayStrings.map(day => launches().find(`${day}/`)));
+        let summaries: LaunchSummary[] = [];
+        for (let keys of keyLists) {
+            for (let key of keys) {
+                summaries.push(parseLaunchKey(key));
+            }
+        }
+        return summaries;
+    }
+
     public async deployFunctions(config: {
         functionSpecs: FunctionSpec[];
         prefixes: string[];
@@ -469,6 +548,7 @@ export const MachineServiceController = getSyncedController(
             getPendingFunctions: {},
             deployFunctions: {},
             getLiveFunctions: {},
+            getRecentLaunches: {},
         }),
         () => ({
             hooks: [assertIsManagementUser],
@@ -497,6 +577,7 @@ export const MachineServiceController = getSyncedController(
             getGitInfo: ["gitInfo"],
             getPendingFunctions: ["gitInfo"],
             getLiveFunctions: ["gitInfo"],
+            getRecentLaunches: ["launches"],
         }
     }
 );

package/src/diagnostics/NodeConnectionsPage.tsx

@@ -142,7 +142,7 @@ class NodeConnectionsControllerBase {
     }
 
     public async getEntryPoint_forBrowser(nodeId: string) {
-        return await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
+        return (await NodeCapabilitiesController.nodes[nodeId].getMetadata()).entryPoint;
     }
 
     public async getAllNodeIds() {