querysub 0.555.0 → 0.557.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.555.0",
+  "version": "0.557.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.39",
+    "sliftutils": "^1.7.40",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-0-hooks/hooks.ts

@@ -1,5 +1,5 @@
-import { MaybePromise } from "socket-function/src/types";
-import { CallInterceptor } from "../3-path-functions/PathFunctionHelpers";
+import type { MaybePromise } from "socket-function/src/types";
+import type { CallInterceptor } from "../3-path-functions/PathFunctionHelpers";
 import type { EdgeNodeConfig } from "../4-deploy/edgeNodes";
 import type { ExtraMetadata } from "../5-diagnostics/nodeMetadata";
 

package/src/0-path-value-core/PathWatcher.ts

@@ -50,6 +50,12 @@ function incrementWatcherSequence() {
     });
 }
 
+if (!registerResource) {
+    debugger;
+    require("debugbreak")(2);
+    debugger;
+}
+
 // WATCH CASES
 // 1) getOwnNodeId() is used to trigger pathValueClientWatcher (no one else should be using it)
 // 2) Use other nodes to immediately proxy to the other nodes

package/src/deployManager/components/ProcessesView.tsx

@@ -0,0 +1,169 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../4-dom/qreact";
+import { css } from "typesafecss";
+import { t } from "../../2-proxy/schema2";
+import { Querysub } from "../../4-querysub/Querysub";
+import { formatTime, formatDateTimeDetailed } from "socket-function/src/formatting/format";
+import { sort, timeInSecond, nextId } from "socket-function/src/misc";
+import { MachineController, watchProcessOutput, stopWatchingProcessOutput } from "../machineController";
+import type { ProcessRecord } from "../processLogs";
+import { Button } from "../../library-components/Button";
+import { parseAnsiColors } from "../../diagnostics/logs/ansiFormat";
+
+module.hotreload = true;
+
+const RUNNING_COLOR = { h: 130, s: 55, l: 88 };
+const DEAD_COLOR = { h: 0, s: 0, l: 92 };
+const OUTPUT_BUFFER_LIMIT = 1_000_000;
+const OUTPUT_BUFFER_KEPT = 100_000;
+const OUTPUT_MAX_HEIGHT = "40vh";
+
+/** One process's live output. Mounting starts the stream, unmounting stops it, so the watch lifetime is exactly the time it is on screen. */
+class ProcessOutput extends qreact.Component<{ record: ProcessRecord; machineNodeId: string }> {
+    state = t.state({
+        data: t.type(""),
+    });
+    private callbackId = nextId();
+    // Unmounting before the watch finishes registering would otherwise stop a callback that gets added right after, leaving it streaming into a component that is gone
+    private watching: Promise<void> = Promise.resolve();
+    private unmounted = false;
+    componentDidMount() {
+        // Props are synchronized state, so the plain values the async work needs are read here
+        let { folder, launchId } = this.props.record;
+        let nodeId = this.props.machineNodeId;
+        let callbackId = this.callbackId;
+        this.watching = (async () => {
+            await watchProcessOutput({
+                nodeId,
+                folder,
+                launchId,
+                callbackId,
+                onData: async (data: string) => {
+                    Querysub.commit(() => {
+                        let full = this.state.data + data;
+                        // Trimmed well below the limit, so trimming is rare - each trim jumps the scroll position
+                        if (full.length > OUTPUT_BUFFER_LIMIT) {
+                            full = full.slice(-OUTPUT_BUFFER_KEPT);
+                        }
+                        this.state.data = full;
+                    });
+                },
+            });
+            // Unmounted while we were registering, so stop the watch we just started
+            if (this.unmounted) {
+                await stopWatchingProcessOutput({ callbackId });
+            }
+        })();
+    }
+    componentWillUnmount() {
+        this.unmounted = true;
+        let callbackId = this.callbackId;
+        let watching = this.watching;
+        Querysub.onCommitFinished(async () => {
+            await watching;
+            await stopWatchingProcessOutput({ callbackId });
+        });
+    }
+    render() {
+        return <div className={
+            css.fontFamily("monospace").whiteSpace("pre-wrap").fillWidth
+                .maxHeight(OUTPUT_MAX_HEIGHT).overflow("auto").pad2(10, 8)
+                .hsl(0, 0, 12).colorhsl(0, 0, 90)
+        }>
+            {parseAnsiColors(this.state.data)}
+        </div>;
+    }
+}
+
+class ProcessRow extends qreact.Component<{ record: ProcessRecord; machineNodeId: string }> {
+    state = t.state({
+        watching: t.type(false),
+    });
+    render() {
+        let record = this.props.record;
+        let now = Querysub.nowDelayed(timeInSecond);
+        let running = record.deadTime === undefined;
+        let color = running && RUNNING_COLOR || DEAD_COLOR;
+        return <div className={css.vbox(6).fillWidth.pad2(10, 8).bord2(color.h, color.s, color.l - 20).hsl(color.h, color.s, color.l)}>
+            <div className={css.hbox(10).wrap.alignItems("center")}>
+                <div className={css.boldStyle}>{record.screenName}</div>
+                <div className={css.colorhsl(0, 0, 35)}>{record.serviceKey} #{record.index}</div>
+                <div className={css.hbox(4)}>
+                    <span>started</span>
+                    <span className={css.boldStyle}>{formatDateTimeDetailed(record.startTime)}</span>
+                    <span className={css.colorhsl(0, 0, 40)}>({formatTime(now - record.startTime)} ago)</span>
+                </div>
+                {running && <div>● running for {formatTime(now - record.startTime)}</div>}
+                {!running && <div className={css.hbox(4)}>
+                    <span>died</span>
+                    <span className={css.boldStyle}>{formatDateTimeDetailed(record.deadTime || 0)}</span>
+                    <span className={css.colorhsl(0, 0, 40)}>
+                        ({formatTime(now - (record.deadTime || 0))} ago, ran {formatTime((record.deadTime || 0) - record.startTime)})
+                    </span>
+                </div>}
+                {record.pid !== undefined && <div className={css.colorhsl(0, 0, 45)}>pid {record.pid}</div>}
+                <div className={css.flexGrow(1)} />
+                <Button flavor="tiny" onClick={() => this.state.watching = !this.state.watching}>
+                    {this.state.watching && "Hide output" || "Watch output"}
+                </Button>
+            </div>
+            {this.state.watching && <ProcessOutput record={record} machineNodeId={this.props.machineNodeId} />}
+        </div>;
+    }
+}
+
+class MachineProcesses extends qreact.Component<{ machineId: string; applyNodeId: string; serviceId?: string }> {
+    state = t.state({
+        showHistorical: t.type(false),
+    });
+    render() {
+        let records = MachineController(SocketFunction.browserNodeId()).listOtherProcesses({ nodeId: this.props.applyNodeId });
+        if (!records) return <div className={css.pad2(10, 8)}>{this.props.machineId}: loading processes...</div>;
+        let shown = records;
+        if (this.props.serviceId) {
+            shown = shown.filter(x => x.serviceId === this.props.serviceId);
+        }
+        let running = shown.filter(x => x.deadTime === undefined);
+        let historical = shown.filter(x => x.deadTime !== undefined);
+        sort(running, x => -x.startTime);
+        sort(historical, x => -(x.deadTime || 0));
+        return <div className={css.vbox(8).fillWidth}>
+            <div className={css.hbox(10).wrap.alignItems("center")}>
+                <div className={css.boldStyle}>{this.props.machineId}</div>
+                <div className={css.colorhsl(0, 0, 45)}>{running.length} running</div>
+                {historical.length > 0 && <Button flavor="tiny" onClick={() => this.state.showHistorical = !this.state.showHistorical}>
+                    {this.state.showHistorical && `Hide ${historical.length} historical` || `Show ${historical.length} historical`}
+                </Button>}
+            </div>
+            {running.map(record => <ProcessRow key={record.launchId} record={record} machineNodeId={this.props.applyNodeId} />)}
+            {this.state.showHistorical && historical.map(record =>
+                <ProcessRow key={record.launchId} record={record} machineNodeId={this.props.applyNodeId} />
+            )}
+        </div>;
+    }
+}
+
+/** Every machine's processes. The listing is cached per machine, so Invalidate empties it and each machine repopulates as its own call returns - watching a process streams live regardless. */
+export class ProcessesView extends qreact.Component<{
+    machines: { machineId: string; applyNodeId: string }[];
+    serviceId?: string;
+}> {
+    render() {
+        return <div className={css.vbox(14).fillWidth}>
+            <div className={css.hbox(10).alignItems("center")}>
+                <h3 className={css.flexGrow(1)}>Processes</h3>
+                <Button onClick={() => {
+                    MachineController(SocketFunction.browserNodeId()).listOtherProcesses.resetAll();
+                }}>
+                    Invalidate
+                </Button>
+            </div>
+            {this.props.machines.map(machine => <MachineProcesses
+                key={machine.machineId}
+                machineId={machine.machineId}
+                applyNodeId={machine.applyNodeId}
+                serviceId={this.props.serviceId}
+            />)}
+        </div>;
+    }
+}

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -8,30 +8,24 @@ import { currentViewParam, selectedServiceIdParam, selectedMachineIdParam } from
 import { formatDateTime, formatDateTimeDetailed, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 import { InputPicker } from "../../library-components/InputPicker";
 import { MachinePicker } from "./MachinePicker";
-import { deepCloneJSON, nextId, sort, timeInSecond } from "socket-function/src/misc";
+import { deepCloneJSON, sort, timeInSecond } from "socket-function/src/misc";
 import { InputLabel } from "../../library-components/InputLabel";
 import { Button } from "../../library-components/Button";
 import { isDefined } from "../../misc";
-import { watchScreenOutput, stopWatchingScreenOutput } from "../machineController";
+import { ProcessesView } from "./ProcessesView";
 import { getPathStr2 } from "../../path";
 import { ATag, Anchor } from "../../library-components/ATag";
 import { ScrollOnMount } from "../../library-components/ScrollOnMount";
-import { StickyBottomScroll } from "../../library-components/StickyBottomScroll";
 import { PrimitiveDisplay } from "../../diagnostics/logs/ObjectDisplay";
-import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
 import { RenderGitRefInfo, UpdateServiceButtons, bigEmoji, buttonStyle } from "./deployButtons";
 import { TypedConfigEditor } from "../../library-components/TypedConfigEditor";
 import { managementPageURL } from "../../diagnostics/managementPages";
 import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
-import { getScreenName } from "../machineApplyMainCode";
+import { getScreenName } from "../processManager";
 import { getOwnThreadId } from "../../-f-node-discovery/NodeDiscovery";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { showModal } from "../../5-diagnostics/Modal";
 
-// Trimmed well below the limit, so trimming is rare - each trim jumps the relative scroll position
-const OUTPUT_BUFFER_LIMIT = 1_000_000;
-const OUTPUT_BUFFER_KEPT = 100_000;
-
 export class ServiceDetailPage extends qreact.Component {
     state = t.state({
         // The editor's current value. Purely in the browser — nothing is written anywhere until a deploy is scheduled (or forced). Whether there are unsaved changes is DERIVED by comparing this to the deployed config, never tracked separately.
@@ -41,13 +35,6 @@ export class ServiceDetailPage extends qreact.Component {
         expandedErrors: t.lookup({
             expanded: t.type(false)
         }),
-        watchingOutputs: t.lookup({
-            isWatching: t.type(false),
-            data: t.type(""),
-            callbackId: t.type(""),
-            // The launch the buffered output belongs to. A new launch is a new process, so its output must not be appended to the previous one's.
-            launchTime: t.number(0),
-        }),
         // Milliseconds; 0 means no scheduled time picked yet (use Deploy Now instead)
         switchTime: t.number(0),
         // Seconds until the release goes live; 0 means use the default (DEFAULT_OVERLAP_TIME)
@@ -66,66 +53,6 @@ export class ServiceDetailPage extends qreact.Component {
         this.state.editorState = updatedConfig;
     }
 
-    private async startWatchingOutput(config: {
-        nodeId: string;
-        key: string;
-        index: number;
-        launchTime: number;
-    }) {
-        const { nodeId, key, index, launchTime } = config;
-        const outputKey = getPathStr2(key, index + "");
-        let callbackId = nextId();
-
-        // Drop the previous watch first: two live callbacks writing to one buffer is what interlaced the output of the old and new processes
-        let previousCallbackId = Querysub.localRead(() => this.state.watchingOutputs[outputKey].callbackId);
-        if (previousCallbackId) {
-            await stopWatchingScreenOutput({ callbackId: previousCallbackId });
-        }
-
-        Querysub.commit(() => {
-            // Cleared, so the buffer only ever holds the output of the launch it is watching
-            this.state.watchingOutputs[outputKey] = { isWatching: true, data: "", callbackId, launchTime };
-        });
-
-        await watchScreenOutput({
-            nodeId,
-            key,
-            index,
-            callbackId,
-            onData: async (data: string, dataConfig?: { reset?: boolean }) => {
-                Querysub.localCommit(() => {
-                    let watchingState = this.state.watchingOutputs[outputKey];
-                    // A callback that outlived its watch (a restart raced with in-flight data) must not write into the new process's buffer
-                    if (watchingState.callbackId !== callbackId) return;
-                    // The screen's process changed, so what we have belongs to a process that is gone
-                    let fullData = (dataConfig?.reset && "" || watchingState.data) + data;
-                    // Don't trim every time, otherwise the relative scroll position changes by too much
-                    if (fullData.length > OUTPUT_BUFFER_LIMIT) {
-                        fullData = fullData.slice(-OUTPUT_BUFFER_KEPT);
-                    }
-                    watchingState.data = fullData;
-                });
-            }
-        });
-    }
-
-    private async stopWatchingOutput(key: string, index: number) {
-        const outputKey = getPathStr2(key, index + "");
-
-        let callbackId = Querysub.localRead(() => {
-            const watchingState = this.state.watchingOutputs[outputKey];
-            let previousCallbackId = watchingState.callbackId;
-            watchingState.isWatching = false;
-            // Clearing this stops any in-flight data from landing in the buffer, and marks that there is no watch to stop next time
-            watchingState.callbackId = "";
-            return previousCallbackId;
-        });
-
-        Querysub.onCommitFinished(async () => {
-            await stopWatchingScreenOutput({ callbackId });
-        });
-    }
-
     // Deploys the editor's config (with parameters.releaseTime deciding when it goes live). The editor is NEVER reset — it is set to exactly what was deployed, so it keeps its content and simply compares as having no unsaved changes.
     private deployConfig(deployConfig: ServiceConfig) {
         // Do not let them update the serviceId, as that would break things
@@ -435,13 +362,8 @@ export class ServiceDetailPage extends qreact.Component {
                         }
 
                         let key = config.parameters.key;
-                        const outputKey = getPathStr2(key, index + "");
-                        const isWatching = this.state.watchingOutputs[outputKey].isWatching;
-                        let outputData = this.state.watchingOutputs[outputKey].data;
                         const screenName = getScreenName({ serviceKey: key, index });
 
-                        let launchTime = serviceInfo?.lastLaunchedTime || 0;
-
                         return <div key={machineId}
                             className={css.pad2(12).vbox(10).bord2(0, 0, 20).fillWidth + backgroundColor}
                         >
@@ -510,24 +432,6 @@ export class ServiceDetailPage extends qreact.Component {
                                 </Anchor>
 
 
-                                <div
-                                    className={css.button.pad2(16, 8).bord2(0, 0, 10) + (isWatching ? css.hsl(0, 70, 90) : css.hsl(120, 70, 90))
-                                    }
-                                    onClick={(e) => {
-                                        e.stopPropagation();
-                                        let applyNodeId = machineInfo.applyNodeId;
-                                        Querysub.onCommitFinished(() => {
-                                            if (isWatching) {
-                                                void this.stopWatchingOutput(key, index);
-                                            } else {
-                                                void this.startWatchingOutput({ nodeId: applyNodeId, key, index, launchTime });
-                                            }
-                                        });
-                                    }}
-                                >
-                                    {isWatching ? "Stop Watching Output" : "Watch Screen Output"}
-                                </div>
-
                                 <ATag values={getLogViewerParams({ __machineId: machineId })}>
                                     Machine Logs
                                 </ATag>
@@ -559,34 +463,15 @@ export class ServiceDetailPage extends qreact.Component {
                                 </div>
                             )}
 
-                            {isWatching &&
-                                <div
-                                    className={
-                                        css.pad2(8).bord2(0, 0, 10).hsl(0, 0, 10).colorhsl(0, 0, 100)
-                                            .whiteSpace("pre-wrap").fontFamily("monospace")
-                                            .overflowAuto
-                                            .height("60vh")
-                                            .vbox0
-                                            .fillWidth
-                                    }
-                                    onClick={e => e.stopPropagation()}
-                                >
-                                    <div className={css.flexShrink0}>
-                                        {(() => {
-                                            let parts = parseAnsiColors(outputData);
-                                            return parts.map(({ text, color }) => {
-                                                if (!color) return <span>{text}</span>;
-                                                let hue = rgbToHsl(color).h;
-                                                return <span className={css.hsl(hue, 60, 30)}>{text}</span>;
-                                            });
-                                        })()}
-                                    </div>
-                                    <StickyBottomScroll debugText={`Screen ${outputKey}`} time={Date.now()} />
-                                </div>
-                            }
                         </div>;
                     })}
                 </div>
+                <ProcessesView
+                    serviceId={selectedServiceId || ""}
+                    machines={machineStatuses
+                        .filter(x => x.machineInfo)
+                        .map(x => ({ machineId: x.machineId, applyNodeId: x.machineInfo!.applyNodeId }))}
+                />
             </div>}
 
             <div className={css.hbox(12).fillWidth}>

package/src/deployManager/machineController.ts

@@ -5,7 +5,8 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { timeInMinute } from "socket-function/src/misc";
 import { assertIsManagementUser } from "../diagnostics/managementPages";
 import { isNode } from "typesafecss";
-import { streamScreenOutput } from "./machineApplyMainCode";
+import { streamProcessOutput } from "./processManager";
+import { ProcessRecord, listProcessRecords } from "./processLogs";
 import { Querysub } from "../4-querysub/Querysub";
 import { getPathStr2 } from "../path";
 import { getGitURLLive, setGitRef } from "../4-deploy/git";
@@ -72,38 +73,43 @@ export const OnServiceChange = SocketFunction.register(
 
 class MachineControllerBase {
     // NOTE: We don't need to worry about escaping commands here. YES, the user CAN inject code into the key. But this system is literally for running arbitrary commands, so they could just write a serviceConfig and run anything they want, on all the machines...
-    public async streamScreenOutput(config: {
-        key: string;
-        index: number;
+    /** Every process this machine has a record of, running or dead. */
+    public async listProcesses(): Promise<ProcessRecord[]> {
+        return await listProcessRecords();
+    }
+    public async streamProcessOutput(config: {
+        folder: string;
+        launchId: string;
         callbackId: string;
     }): Promise<void> {
         let caller = SocketFunction.getCaller();
-        await streamScreenOutput({
-            key: config.key,
-            index: config.index,
-            onData: async (data, dataConfig) => {
+        await streamProcessOutput({
+            folder: config.folder,
+            launchId: config.launchId,
+            onData: async (data) => {
                 await MachineControllerClient.nodes[caller.nodeId].onScreenOutput({
-                    key: config.key,
-                    index: config.index,
+                    launchId: config.launchId,
                     data,
-                    reset: dataConfig?.reset,
                     callbackId: config.callbackId,
                 });
             },
         });
     }
-    // We need to forward it, as clients can't connect to servers directly (as they don't have real HTTP certificates).
-    public async watchOtherScreenOutput(config: {
+    // We need to forward these, as clients can't connect to the machines directly (they don't have real HTTP certificates).
+    public async listOtherProcesses(config: { nodeId: string }): Promise<ProcessRecord[]> {
+        return await MachineController(config.nodeId).listProcesses.promise();
+    }
+    public async watchOtherProcessOutput(config: {
         nodeId: string;
-        key: string;
-        index: number;
+        folder: string;
+        launchId: string;
         callbackId: string;
     }) {
         let caller = SocketFunction.getCaller();
         forwardedCallbacks.set(config.callbackId, caller.nodeId);
-        await MachineController(config.nodeId).streamScreenOutput.promise({
-            key: config.key,
-            index: config.index,
+        await MachineController(config.nodeId).streamProcessOutput.promise({
+            folder: config.folder,
+            launchId: config.launchId,
             callbackId: config.callbackId,
         });
     }
@@ -137,8 +143,10 @@ export const MachineController = getSyncedController(SocketFunction.register(
     "machine-controller-c3157d4a-580c-4e76-9dc9-072dd92e70af",
     () => new MachineControllerBase(),
     () => ({
-        streamScreenOutput: {},
-        watchOtherScreenOutput: {},
+        listProcesses: {},
+        streamProcessOutput: {},
+        listOtherProcesses: {},
+        watchOtherProcessOutput: {},
         deployMachineFromBrowser: {},
         deployMachine: {},
     }),
@@ -153,26 +161,25 @@ export const MachineController = getSyncedController(SocketFunction.register(
     reads: {},
 });
 
-let callbacks = new Map<string, (data: string, config?: { reset?: boolean }) => Promise<void>>();
-export async function watchScreenOutput(config: {
+let callbacks = new Map<string, (data: string) => Promise<void>>();
+export async function watchProcessOutput(config: {
     nodeId: string;
-    key: string;
-    index: number;
+    folder: string;
+    launchId: string;
     callbackId: string;
-    /** reset means the process behind the screen changed: replace what you have with this, don't append it. */
-    onData: (data: string, config?: { reset?: boolean }) => Promise<void>;
+    onData: (data: string) => Promise<void>;
 }) {
     let callbackId = config.callbackId;
     callbacks.set(callbackId, config.onData);
-    await MachineController(SocketFunction.browserNodeId()).watchOtherScreenOutput.promise({
+    await MachineController(SocketFunction.browserNodeId()).watchOtherProcessOutput.promise({
         nodeId: config.nodeId,
-        key: config.key,
-        index: config.index,
+        folder: config.folder,
+        launchId: config.launchId,
         callbackId,
     });
 }
 
-export async function stopWatchingScreenOutput(config: {
+export async function stopWatchingProcessOutput(config: {
     callbackId: string;
 }) {
     let callbackId = config.callbackId;
@@ -180,19 +187,15 @@ export async function stopWatchingScreenOutput(config: {
 }
 class MachineControllerClientBase {
     public async onScreenOutput(config: {
-        key: string;
-        index: number;
+        launchId: string;
         data: string;
-        reset?: boolean;
         callbackId: string;
     }): Promise<void> {
         let forwardToNodeId = forwardedCallbacks.get(config.callbackId);
         if (forwardToNodeId) {
             await MachineControllerClient.nodes[forwardToNodeId].onScreenOutput({
-                key: config.key,
-                index: config.index,
+                launchId: config.launchId,
                 data: config.data,
-                reset: config.reset,
                 callbackId: config.callbackId,
             });
             return;
@@ -202,7 +205,7 @@ class MachineControllerClientBase {
         if (!callback) {
             throw new Error(`Callback ${config.callbackId} not found (likely removed)`);
         }
-        await callback(config.data, { reset: config.reset });
+        await callback(config.data);
     }
 }
 // NOTE: THis is secure, because callbackId is random.

package/src/deployManager/processLogs.ts

@@ -0,0 +1,171 @@
+import os from "os";
+import fs from "fs";
+import path from "path";
+import { SERVICE_FOLDER, ServiceParameters } from "./machineSchema";
+
+// One log file per process, appended to until it passes LOG_SIZE_LIMIT, then emptied wholesale. Watchers see the file shrink and re-seek to the start. A watcher attaching in the moment right after a reset sees an empty log, which at this size is rare enough to be worth how much simpler this is than rolling files.
+const LOG_SIZE_LIMIT = 10 * 1024 * 1024;
+// Checked every this many lines, so the size check costs nothing per line
+const LOG_SIZE_CHECK_LINES = 1000;
+/** How long a dead process's log is kept before it is deleted. */
+export const DEAD_PROCESS_RETENTION = 3 * 24 * 60 * 60 * 1000;
+
+const PROCESS_FOLDER = "processes";
+const RECORD_SUFFIX = ".json";
+const LOG_SUFFIX = ".log";
+const PIPE_SCRIPT_SUFFIX = ".pipe.sh";
+const TAIL_SCRIPT_SUFFIX = ".tail.sh";
+
+/** One launch of one service instance: the process, the configuration it was launched with, and its output. Written when the process is launched, and updated only to record its pid and then its death. */
+export type ProcessRecord = {
+    launchId: string;
+    /** The instance folder this process runs out of. A release's future process shares its canonical instance's folder, so this is what owns the logs, not the tmux session name. */
+    folder: string;
+    /** The tmux session, which a takeover renames from the future name to the canonical one */
+    screenName: string;
+    serviceId: string;
+    serviceKey: string;
+    /** Which instance of the service this is, on this machine */
+    index: number;
+    machineId: string;
+    /** The start time the OS reports for the pid, which is also half of the launchId */
+    startTime: number;
+    /** When we first noticed the process was gone. Absent while it is running. */
+    deadTime?: number;
+    pid?: number;
+    /** The parameters it was launched with, with template variables already resolved */
+    parameters: ServiceParameters;
+};
+
+export function getProcessLogPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + LOG_SUFFIX);
+}
+function getRecordPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + RECORD_SUFFIX);
+}
+// The scripts live beside the log so a process's entire footprint is one set of files, deleted together
+export function getPipeScriptPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + PIPE_SCRIPT_SUFFIX);
+}
+export function getTailScriptPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + TAIL_SCRIPT_SUFFIX);
+}
+
+/** A process identifies itself: its pid, plus the start time the OS reports for it. The pid alone is reused, but a pid running since a specific second is one process, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
+export function createLaunchId(pid: string, startTime: number): string {
+    return `${pid}-${startTime}`;
+}
+export function parseLaunchId(launchId: string): { pid: string; startTime: number } {
+    let separator = launchId.lastIndexOf("-");
+    return {
+        pid: launchId.slice(0, separator),
+        startTime: parseInt(launchId.slice(separator + 1)) || 0,
+    };
+}
+
+/** The shell a screen's output is piped through: appends every line, and empties the file once it grows past the limit. */
+export function getLogPipeScript(logPath: string): string {
+    return `#!/bin/bash
+line_count=0
+while IFS= read -r line; do
+    echo "$line" >> "${logPath}"
+    ((line_count++))
+    if (( line_count % ${LOG_SIZE_CHECK_LINES} == 0 )); then
+        size=$(stat -c%s "${logPath}" 2>/dev/null || stat -f%z "${logPath}" 2>/dev/null || echo 0)
+        if [ "$size" -gt ${LOG_SIZE_LIMIT} ]; then
+            : > "${logPath}"
+        fi
+    fi
+done`;
+}
+
+/** Streams everything appended to a log after the given offset. A file smaller than our offset was emptied by the size limit, so we start over from the front of it. */
+export function getLogTailScript(logPath: string): string {
+    return `#!/bin/bash
+CURRENT_POS="$1"
+
+while true; do
+    sleep 0.25
+    if [ -f "${logPath}" ]; then
+        SIZE=$(stat -c%s "${logPath}" 2>/dev/null || stat -f%z "${logPath}" 2>/dev/null || echo 0)
+        if [ "$SIZE" -lt "$CURRENT_POS" ]; then
+            CURRENT_POS=0
+        fi
+        if [ "$SIZE" -gt "$CURRENT_POS" ]; then
+            tail -c +$((CURRENT_POS + 1)) "${logPath}" | head -c $((SIZE - CURRENT_POS))
+            CURRENT_POS=$SIZE
+        fi
+    fi
+done`;
+}
+
+export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
+    await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
+    await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));
+}
+
+async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
+    let processFolder = path.join(folder, PROCESS_FOLDER);
+    let files: string[];
+    try {
+        files = await fs.promises.readdir(processFolder);
+    } catch {
+        return [];
+    }
+    let records: ProcessRecord[] = [];
+    for (let file of files) {
+        if (!file.endsWith(RECORD_SUFFIX)) continue;
+        records.push(JSON.parse(await fs.promises.readFile(path.join(processFolder, file), "utf8")) as ProcessRecord);
+    }
+    return records;
+}
+
+/** Every process this machine has a record of, running or dead, across every service. */
+export async function listProcessRecords(): Promise<ProcessRecord[]> {
+    let root = os.homedir() + "/" + SERVICE_FOLDER;
+    let instanceFolders: string[];
+    try {
+        instanceFolders = await fs.promises.readdir(root);
+    } catch {
+        return [];
+    }
+    let records: ProcessRecord[] = [];
+    for (let instanceFolder of instanceFolders) {
+        records.push(...await readFolderRecords(path.join(root, instanceFolder)));
+    }
+    return records;
+}
+
+async function removeProcessRecord(record: ProcessRecord): Promise<void> {
+    let files = [
+        getRecordPath(record.folder, record.launchId),
+        getProcessLogPath(record.folder, record.launchId),
+        getPipeScriptPath(record.folder, record.launchId),
+        getTailScriptPath(record.folder, record.launchId),
+    ];
+    for (let file of files) {
+        try {
+            await fs.promises.unlink(file);
+        } catch {
+            // Already gone, which is the state we wanted
+        }
+    }
+}
+
+/** Marks every record whose process is gone as dead, and deletes the ones that have been dead past the retention. */
+export async function syncProcessRecords(config: {
+    isAlive: (record: ProcessRecord) => Promise<boolean>;
+    now: number;
+}): Promise<void> {
+    for (let record of await listProcessRecords()) {
+        if (record.deadTime === undefined && !await config.isAlive(record)) {
+            record.deadTime = config.now;
+            console.log(`Process ${record.launchId} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
+            await writeProcessRecord(record);
+        }
+        if (record.deadTime !== undefined && config.now - record.deadTime > DEAD_PROCESS_RETENTION) {
+            console.log(`Deleting the log of process ${record.launchId} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
+            await removeProcessRecord(record);
+        }
+    }
+}

package/src/deployManager/processManager.ts

@@ -0,0 +1,369 @@
+import os from "os";
+import fs from "fs";
+import path from "path";
+import { spawn, ChildProcess } from "child_process";
+import { lazy } from "socket-function/src/caching";
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { delay, runInSerial } from "socket-function/src/batching";
+import { red, green } from "socket-function/src/formatting/logColors";
+import { runPromise } from "../functional/runCommand";
+import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
+import { fsExistsAsync } from "../fs";
+import { PromiseObj } from "../promise";
+import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
+import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, writeProcessRecord } from "./processLogs";
+
+// Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
+
+const SCREEN_SUFFIX = "-dply";
+export function getScreenName(config: { serviceKey: string; index: number }): string {
+    return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
+}
+// The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
+export function getFutureScreenName(canonicalScreenName: string): string {
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + "-future" + SCREEN_SUFFIX;
+}
+
+
+async function removeOldNodeId(screenName: string) {
+    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
+    if (await fsExistsAsync(nodeIdFile)) {
+        let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
+        console.log(green(`Removing node if for dead service on ${nodeIdFile}, node id ${nodeId}`));
+        await fs.promises.unlink(nodeIdFile);
+        await forceRemoveNode(nodeId);
+    }
+}
+
+export const getTmuxPrefix = lazy(() => {
+    if (os.platform() === "win32") {
+        return "C:/cygwin64/bin/";
+        // C:/cygwin64/bin/tmux new -s server1 -d
+    }
+    return "";
+});
+function textTableLineToObj(text: string): Record<string, string>[] {
+    /*
+      PID    PPID    PGID     WINPID   TTY         UID    STIME COMMAND
+     1312    1284    1312      56996  pty2      197609 09:24:47 /usr/bin/bash
+     1313    1284    1313      56997  pty3      197609 09:24:48 /usr/bin/vim
+    =>
+    [{ PID: "1312", PPID: "1284", ...}, { PID: "1313", PPID: "1284", ...}]
+    */
+    let lines = text.split("\n").filter(line => line.trim().length > 0);
+    if (lines.length < 2) {
+        return [];
+    }
+
+    let headerLine = lines[0];
+    let dataLines = lines.slice(1);
+
+    // Parse column positions from header
+    let headerWords = headerLine.match(/\S+/g) || [];
+    if (headerWords.length === 0) return [];
+
+    // Find column boundaries: start at 0, then at end of each header word (except last), then start of last header, then end of line
+    let boundaries: number[] = [0];
+    let searchPos = 0;
+    for (let i = 0; i < headerWords.length; i++) {
+        let word = headerWords[i];
+        searchPos = headerLine.indexOf(word, searchPos) + word.length;
+        boundaries.push(searchPos);
+    }
+    boundaries[boundaries.length - 1] = Number.MAX_SAFE_INTEGER;
+
+    // Create column definitions using boundaries
+    let columns: { name: string; start: number; end: number }[] = [];
+    for (let i = 0; i < headerWords.length; i++) {
+        columns.push({
+            name: headerWords[i],
+            start: boundaries[i],
+            end: boundaries[i + 1]
+        });
+    }
+
+    // Extract values from all data lines
+    let results: Record<string, string>[] = [];
+    for (let dataLine of dataLines) {
+        let result: Record<string, string> = {};
+        for (let column of columns) {
+            let value = dataLine.substring(column.start, column.end).trim();
+            result[column.name] = value;
+        }
+        results.push(result);
+    }
+
+    return results;
+}
+const getLinuxChildPids = measureWrap(async function getLinuxChildPids(pid: string): Promise<{ PID: string; PPID: string; CMD: string }[]> {
+    let prefix = getTmuxPrefix();
+    if (os.platform() === "win32") {
+        let table = await runPromise(`${prefix}ps`, { quiet: true });
+        let obj = textTableLineToObj(table) as { PID: string; PPID: string; CMD: string }[];
+        return obj.filter(x => x.PPID === pid);
+    } else {
+        let table = await runPromise(`ps -eo pid,ppid,cmd`, { quiet: true });
+        let obj = textTableLineToObj(table) as { PPID: string; PID: string; CMD: string }[];
+        return obj.filter(x => x.PPID === pid);
+    }
+});
+// One process-table read for the whole machine — a pid is "running something" when any process has it as a parent. Used by getScreenState so a resync doesn't spawn one ps per screen (which made the release boundary ticks wait on every idle screen first).
+const getAllParentPids = measureWrap(async function getAllParentPids(): Promise<Set<string>> {
+    let prefix = getTmuxPrefix();
+    let table = os.platform() === "win32"
+        ? await runPromise(`${prefix}ps`, { quiet: true })
+        : await runPromise(`ps -eo pid,ppid,cmd`, { quiet: true });
+    let obj = textTableLineToObj(table) as { PID: string; PPID: string }[];
+    return new Set(obj.map(x => x.PPID));
+});
+/** The start time the OS reports for a pid, as an epoch time in milliseconds like every other time we store. Asked of the machine rather than remembered, so it stays true across our restarts and catches a reused pid. Returns 0 when the process is gone. */
+export const getProcessStartTime = measureWrap(async function getProcessStartTime(pid: string): Promise<number> {
+    // ps only resolves to the second, so this is second-accurate - which is plenty to tell two processes on the same pid apart
+    let started = await runPromise(`date -d "$(ps -o lstart= -p ${pid})" +%s`, { quiet: true });
+    return (parseInt(started.trim()) || 0) * 1000;
+});
+
+export const isScreenRunningProcess = measureWrap(async function isScreenRunningProcess(pid: string): Promise<boolean> {
+    try {
+        let bashChildPids = await getLinuxChildPids(pid);
+        for (let childPid of bashChildPids) {
+            console.log(`Screen pid ${pid} is running ${childPid.CMD}`);
+            return true;
+        }
+        console.log(`Screen pid ${pid} is not running anything.`);
+        return false;
+    } catch (e: any) {
+        console.warn(`Error checking if screen is running for ${pid}: ${e.stack}`);
+        return false;
+    }
+});
+export const getScreenState = measureWrap(async function getScreenState(populateIsProcessRunning: boolean = true): Promise<{
+    screenName: string;
+    isProcessRunning: boolean;
+    pid: string;
+}[]> {
+    const prefix = getTmuxPrefix();
+    const delimit = "::::";
+    // Use list-sessions instead of list-panes -a to avoid zombie sessions
+    // 2>/dev/null suppresses "no server running" errors, -r prevents xargs from running if input is empty
+    let screenList = (await runPromise(`${prefix}tmux list-sessions -F "#{session_name}" 2>/dev/null | xargs -r -I {} tmux list-panes -t {} -F "{}${delimit}#{pane_pid}"`))
+        .split("\n")
+        .filter(x => x.includes(delimit))
+        .map(x => ({
+            screenName: x.split(delimit)[0].trim(),
+            pid: x.split(delimit)[1].trim(),
+            isProcessRunning: false,
+        }))
+        .filter(x => x.screenName.endsWith(SCREEN_SUFFIX))
+        ;
+
+    if (populateIsProcessRunning) {
+        let parentPids = await getAllParentPids();
+        for (let x of screenList) {
+            x.isProcessRunning = parentPids.has(x.pid);
+        }
+    }
+
+    return screenList;
+});
+
+export const runScreenCommand = measureWrap(async function runScreenCommand(config: {
+    screenName: string;
+    command: string;
+    // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
+    folder?: string;
+    // Identifies the process this launch creates, so its log and its configuration are stored against it
+    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
+}): Promise<string> {
+    let prefix = getTmuxPrefix();
+    let screenName = config.screenName;
+
+    try {
+        // Throw if it already exists
+        await runPromise(`${prefix}tmux new -s ${screenName} -d`);
+    } catch { }
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'echo "Updating running command at ${new Date().toISOString()}"' Enter`);
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
+    await delay(1000);
+
+
+    let screens = await getScreenState();
+    let screen = screens.find(x => x.screenName === screenName);
+    let pid = screen?.pid;
+    if (pid && await isScreenRunningProcess(pid)) {
+        // It doesn't want to die. Wait longer, but it it just won't die, kill the screen
+        console.warn(`Screen ${screenName} is not dying, giving it another 30 seconds`);
+        for (let i = 0; i < 6; i++) {
+            await delay(5);
+            if (!await isScreenRunningProcess(pid)) {
+                break;
+            }
+        }
+        if (pid && await isScreenRunningProcess(pid)) {
+            console.warn(`Screen ${screenName} is still running, killing it forcefully`);
+            await killScreen({ screenName });
+            if (pid && await isScreenRunningProcess(pid)) {
+                console.error(`I don't know what happened. The screen won't die. We can't do much else, I guess we'll just ignore it...`);
+            } else {
+                // Nested, to create the screen again.
+                return await runScreenCommand({
+                    screenName,
+                    command: config.command,
+                    folder: config.folder,
+                    record: config.record,
+                });
+            }
+        }
+    }
+    await removeOldNodeId(screenName);
+    let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
+
+    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running.
+    let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
+    if (!panePid) {
+        throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
+    }
+    let startTime = await getProcessStartTime(panePid);
+    let launchId = createLaunchId(panePid, startTime);
+    await writeProcessRecord({
+        ...config.record,
+        launchId,
+        folder,
+        screenName,
+        pid: parseInt(panePid) || undefined,
+        startTime,
+    });
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
+    let command = `#!/bin/bash
+${config.command}
+`;
+    await fs.promises.writeFile(folder + "command.sh", command);
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
+
+    await setupPipePane({ screenName, folder, launchId });
+    return launchId;
+});
+
+// Points the screen's pipe-pane at this launch's own log file.
+async function setupPipePane(config: { screenName: string; folder: string; launchId: string }) {
+    let prefix = getTmuxPrefix();
+    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let pipeScript = getPipeScriptPath(config.folder, config.launchId);
+    await fs.promises.mkdir(path.dirname(logPath), { recursive: true });
+    await fs.promises.writeFile(pipeScript, getLogPipeScript(logPath));
+    await runPromise(`chmod +x ${pipeScript}`);
+    await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
+}
+
+export const killScreen = measureWrap(async function killScreen(config: {
+    screenName: string;
+    // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
+    skipNodeIdRemoval?: boolean;
+}) {
+    console.log(red(`Killing screen ${config.screenName}`));
+    let prefix = getTmuxPrefix();
+    // Try ctrl+c a few times first
+    let pid = (await getScreenState(false)).find(x => x.screenName === config.screenName)?.pid;
+    for (let i = 0; i < 5; i++) {
+        if (!pid || !await isScreenRunningProcess(pid)) {
+            break;
+        }
+        await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
+        await delay(5000);
+    }
+    await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
+    if (!config.skipNodeIdRemoval) {
+        await removeOldNodeId(config.screenName);
+    }
+});
+
+/** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */
+export async function streamProcessOutput(config: {
+    folder: string;
+    launchId: string;
+    onData: (data: string) => Promise<void>;
+}) {
+    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let serialOnData = runInSerial(config.onData);
+    let stopped = false;
+    let childProcess: ChildProcess | undefined;
+
+    let pendingDataCalls = 0;
+    const MAX_PENDING_CALLS = 100;
+
+    async function stop() {
+        if (stopped) return;
+        stopped = true;
+        if (childProcess) {
+            childProcess.kill();
+        }
+    }
+
+    const onDataWrapped = async (data: string) => {
+        pendingDataCalls++;
+        if (pendingDataCalls > MAX_PENDING_CALLS) {
+            console.error(`Too many queued onData calls for ${config.launchId}, stopping stream.`);
+            await stop();
+            return;
+        }
+        try {
+            await serialOnData(data);
+        } catch (e: any) {
+            console.log(`Callback for stream output ${config.launchId} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
+            await stop();
+        } finally {
+            pendingDataCalls--;
+        }
+    };
+
+    try {
+        let tailScript = getTailScriptPath(config.folder, config.launchId);
+        await fs.promises.writeFile(tailScript, getLogTailScript(logPath));
+        await runPromise(`chmod +x ${tailScript}`);
+
+        // Read what is already there ourselves and deliver it as one call - letting the tail script cat it
+        // makes runInSerial dribble it out one round trip at a time.
+        let initialContent = "";
+        try {
+            initialContent = await fs.promises.readFile(logPath, "utf8");
+        } catch {
+            // The process may not have written anything yet; the tail script picks it up when it does.
+        }
+
+        childProcess = spawn("bash", [tailScript, String(Buffer.byteLength(initialContent, "utf8"))], {
+            stdio: "pipe",
+        });
+
+        let started = new PromiseObj<void>();
+
+        childProcess.stdout?.on("data", (data) => {
+            if (stopped) return;
+            started.resolve();
+            void onDataWrapped(data.toString());
+        });
+        // Give it some time to error out, otherwise, just start
+        setTimeout(() => started.resolve(), 200);
+
+        childProcess.stderr?.on("data", (data) => {
+            if (stopped) return;
+            void onDataWrapped(red(data.toString()));
+        });
+
+        childProcess.on("error", async (err) => {
+            if (stopped) return;
+            started.reject(err);
+        });
+
+        if (initialContent) {
+            // Queued synchronously here, before any stdout "data" event can fire, so it always lands first
+            started.resolve();
+            void onDataWrapped(initialContent);
+        }
+
+        await started.promise;
+    } catch (e) {
+        void stop();
+        throw e;
+    }
+}
+

package/src/diagnostics/trackResources.ts

@@ -3,7 +3,6 @@ import { lazy } from "socket-function/src/caching";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { blue } from "socket-function/src/formatting/logColors";
 import { isNode } from "socket-function/src/misc";
-import { registerPeriodic } from "./periodic";
 import { registerMeasureInfo } from "socket-function/src/profiling/measure";
 import { logNodeStateStats } from "../-0-hooks/hooks";
 
@@ -89,4 +88,7 @@ registerMeasureInfo(() => {
     return `MEM ${formatNumber(getUsedHeapSize())}B+${formatNumber(getBufferUsage())}B/${formatNumber(getHeapSize())}B `;
 });
 
-registerPeriodic(logResourcesNow);
+setImmediate(async () => {
+    let { registerPeriodic } = await import("./periodic");
+    registerPeriodic(logResourcesNow);
+});