querysub 0.556.0 → 0.558.0

package/src/deployManager/machineController.ts

@@ -5,7 +5,8 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { timeInMinute } from "socket-function/src/misc";
 import { assertIsManagementUser } from "../diagnostics/managementPages";
 import { isNode } from "typesafecss";
-import { streamScreenOutput } from "./machineApplyMainCode";
+import { streamProcessOutput } from "./processManager";
+import { ProcessRecord, listProcessRecords } from "./processLogs";
 import { Querysub } from "../4-querysub/Querysub";
 import { getPathStr2 } from "../path";
 import { getGitURLLive, setGitRef } from "../4-deploy/git";
@@ -72,38 +73,43 @@ export const OnServiceChange = SocketFunction.register(
 
 class MachineControllerBase {
     // NOTE: We don't need to worry about escaping commands here. YES, the user CAN inject code into the key. But this system is literally for running arbitrary commands, so they could just write a serviceConfig and run anything they want, on all the machines...
-    public async streamScreenOutput(config: {
-        key: string;
-        index: number;
+    /** Every process this machine has a record of, running or dead. */
+    public async listProcesses(): Promise<ProcessRecord[]> {
+        return await listProcessRecords();
+    }
+    public async streamProcessOutput(config: {
+        folder: string;
+        launchId: string;
         callbackId: string;
     }): Promise<void> {
         let caller = SocketFunction.getCaller();
-        await streamScreenOutput({
-            key: config.key,
-            index: config.index,
-            onData: async (data, dataConfig) => {
+        await streamProcessOutput({
+            folder: config.folder,
+            launchId: config.launchId,
+            onData: async (data) => {
                 await MachineControllerClient.nodes[caller.nodeId].onScreenOutput({
-                    key: config.key,
-                    index: config.index,
+                    launchId: config.launchId,
                     data,
-                    reset: dataConfig?.reset,
                     callbackId: config.callbackId,
                 });
             },
         });
     }
-    // We need to forward it, as clients can't connect to servers directly (as they don't have real HTTP certificates).
-    public async watchOtherScreenOutput(config: {
+    // We need to forward these, as clients can't connect to the machines directly (they don't have real HTTP certificates).
+    public async listOtherProcesses(config: { nodeId: string }): Promise<ProcessRecord[]> {
+        return await MachineController(config.nodeId).listProcesses.promise();
+    }
+    public async watchOtherProcessOutput(config: {
         nodeId: string;
-        key: string;
-        index: number;
+        folder: string;
+        launchId: string;
         callbackId: string;
     }) {
         let caller = SocketFunction.getCaller();
         forwardedCallbacks.set(config.callbackId, caller.nodeId);
-        await MachineController(config.nodeId).streamScreenOutput.promise({
-            key: config.key,
-            index: config.index,
+        await MachineController(config.nodeId).streamProcessOutput.promise({
+            folder: config.folder,
+            launchId: config.launchId,
             callbackId: config.callbackId,
         });
     }
@@ -137,8 +143,10 @@ export const MachineController = getSyncedController(SocketFunction.register(
     "machine-controller-c3157d4a-580c-4e76-9dc9-072dd92e70af",
     () => new MachineControllerBase(),
     () => ({
-        streamScreenOutput: {},
-        watchOtherScreenOutput: {},
+        listProcesses: {},
+        streamProcessOutput: {},
+        listOtherProcesses: {},
+        watchOtherProcessOutput: {},
         deployMachineFromBrowser: {},
         deployMachine: {},
     }),
@@ -153,26 +161,25 @@ export const MachineController = getSyncedController(SocketFunction.register(
     reads: {},
 });
 
-let callbacks = new Map<string, (data: string, config?: { reset?: boolean }) => Promise<void>>();
-export async function watchScreenOutput(config: {
+let callbacks = new Map<string, (data: string) => Promise<void>>();
+export async function watchProcessOutput(config: {
     nodeId: string;
-    key: string;
-    index: number;
+    folder: string;
+    launchId: string;
     callbackId: string;
-    /** reset means the process behind the screen changed: replace what you have with this, don't append it. */
-    onData: (data: string, config?: { reset?: boolean }) => Promise<void>;
+    onData: (data: string) => Promise<void>;
 }) {
     let callbackId = config.callbackId;
     callbacks.set(callbackId, config.onData);
-    await MachineController(SocketFunction.browserNodeId()).watchOtherScreenOutput.promise({
+    await MachineController(SocketFunction.browserNodeId()).watchOtherProcessOutput.promise({
         nodeId: config.nodeId,
-        key: config.key,
-        index: config.index,
+        folder: config.folder,
+        launchId: config.launchId,
         callbackId,
     });
 }
 
-export async function stopWatchingScreenOutput(config: {
+export async function stopWatchingProcessOutput(config: {
     callbackId: string;
 }) {
     let callbackId = config.callbackId;
@@ -180,19 +187,15 @@ export async function stopWatchingScreenOutput(config: {
 }
 class MachineControllerClientBase {
     public async onScreenOutput(config: {
-        key: string;
-        index: number;
+        launchId: string;
         data: string;
-        reset?: boolean;
         callbackId: string;
     }): Promise<void> {
         let forwardToNodeId = forwardedCallbacks.get(config.callbackId);
         if (forwardToNodeId) {
             await MachineControllerClient.nodes[forwardToNodeId].onScreenOutput({
-                key: config.key,
-                index: config.index,
+                launchId: config.launchId,
                 data: config.data,
-                reset: config.reset,
                 callbackId: config.callbackId,
             });
             return;
@@ -202,7 +205,7 @@ class MachineControllerClientBase {
         if (!callback) {
             throw new Error(`Callback ${config.callbackId} not found (likely removed)`);
         }
-        await callback(config.data, { reset: config.reset });
+        await callback(config.data);
     }
 }
 // NOTE: THis is secure, because callbackId is random.

package/src/deployManager/processLogs.ts

@@ -0,0 +1,171 @@
+import os from "os";
+import fs from "fs";
+import path from "path";
+import { SERVICE_FOLDER, ServiceParameters } from "./machineSchema";
+
+// One log file per process, appended to until it passes LOG_SIZE_LIMIT, then emptied wholesale. Watchers see the file shrink and re-seek to the start. A watcher attaching in the moment right after a reset sees an empty log, which at this size is rare enough to be worth how much simpler this is than rolling files.
+const LOG_SIZE_LIMIT = 10 * 1024 * 1024;
+// Checked every this many lines, so the size check costs nothing per line
+const LOG_SIZE_CHECK_LINES = 1000;
+/** How long a dead process's log is kept before it is deleted. */
+export const DEAD_PROCESS_RETENTION = 3 * 24 * 60 * 60 * 1000;
+
+const PROCESS_FOLDER = "processes";
+const RECORD_SUFFIX = ".json";
+const LOG_SUFFIX = ".log";
+const PIPE_SCRIPT_SUFFIX = ".pipe.sh";
+const TAIL_SCRIPT_SUFFIX = ".tail.sh";
+
+/** One launch of one service instance: the process, the configuration it was launched with, and its output. Written when the process is launched, and updated only to record its pid and then its death. */
+export type ProcessRecord = {
+    launchId: string;
+    /** The instance folder this process runs out of. A release's future process shares its canonical instance's folder, so this is what owns the logs, not the tmux session name. */
+    folder: string;
+    /** The tmux session, which a takeover renames from the future name to the canonical one */
+    screenName: string;
+    serviceId: string;
+    serviceKey: string;
+    /** Which instance of the service this is, on this machine */
+    index: number;
+    machineId: string;
+    /** The start time the OS reports for the pid, which is also half of the launchId */
+    startTime: number;
+    /** When we first noticed the process was gone. Absent while it is running. */
+    deadTime?: number;
+    pid?: number;
+    /** The parameters it was launched with, with template variables already resolved */
+    parameters: ServiceParameters;
+};
+
+export function getProcessLogPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + LOG_SUFFIX);
+}
+function getRecordPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + RECORD_SUFFIX);
+}
+// The scripts live beside the log so a process's entire footprint is one set of files, deleted together
+export function getPipeScriptPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + PIPE_SCRIPT_SUFFIX);
+}
+export function getTailScriptPath(folder: string, launchId: string): string {
+    return path.join(folder, PROCESS_FOLDER, launchId + TAIL_SCRIPT_SUFFIX);
+}
+
+/** A process identifies itself: its pid, plus the start time the OS reports for it. The pid alone is reused, but a pid running since a specific second is one process, and both halves can be re-checked against the machine at any time instead of trusted from our own bookkeeping. */
+export function createLaunchId(pid: string, startTime: number): string {
+    return `${pid}-${startTime}`;
+}
+export function parseLaunchId(launchId: string): { pid: string; startTime: number } {
+    let separator = launchId.lastIndexOf("-");
+    return {
+        pid: launchId.slice(0, separator),
+        startTime: parseInt(launchId.slice(separator + 1)) || 0,
+    };
+}
+
+/** The shell a screen's output is piped through: appends every line, and empties the file once it grows past the limit. */
+export function getLogPipeScript(logPath: string): string {
+    return `#!/bin/bash
+line_count=0
+while IFS= read -r line; do
+    echo "$line" >> "${logPath}"
+    ((line_count++))
+    if (( line_count % ${LOG_SIZE_CHECK_LINES} == 0 )); then
+        size=$(stat -c%s "${logPath}" 2>/dev/null || stat -f%z "${logPath}" 2>/dev/null || echo 0)
+        if [ "$size" -gt ${LOG_SIZE_LIMIT} ]; then
+            : > "${logPath}"
+        fi
+    fi
+done`;
+}
+
+/** Streams everything appended to a log after the given offset. A file smaller than our offset was emptied by the size limit, so we start over from the front of it. */
+export function getLogTailScript(logPath: string): string {
+    return `#!/bin/bash
+CURRENT_POS="$1"
+
+while true; do
+    sleep 0.25
+    if [ -f "${logPath}" ]; then
+        SIZE=$(stat -c%s "${logPath}" 2>/dev/null || stat -f%z "${logPath}" 2>/dev/null || echo 0)
+        if [ "$SIZE" -lt "$CURRENT_POS" ]; then
+            CURRENT_POS=0
+        fi
+        if [ "$SIZE" -gt "$CURRENT_POS" ]; then
+            tail -c +$((CURRENT_POS + 1)) "${logPath}" | head -c $((SIZE - CURRENT_POS))
+            CURRENT_POS=$SIZE
+        fi
+    fi
+done`;
+}
+
+export async function writeProcessRecord(record: ProcessRecord): Promise<void> {
+    await fs.promises.mkdir(path.join(record.folder, PROCESS_FOLDER), { recursive: true });
+    await fs.promises.writeFile(getRecordPath(record.folder, record.launchId), JSON.stringify(record));
+}
+
+async function readFolderRecords(folder: string): Promise<ProcessRecord[]> {
+    let processFolder = path.join(folder, PROCESS_FOLDER);
+    let files: string[];
+    try {
+        files = await fs.promises.readdir(processFolder);
+    } catch {
+        return [];
+    }
+    let records: ProcessRecord[] = [];
+    for (let file of files) {
+        if (!file.endsWith(RECORD_SUFFIX)) continue;
+        records.push(JSON.parse(await fs.promises.readFile(path.join(processFolder, file), "utf8")) as ProcessRecord);
+    }
+    return records;
+}
+
+/** Every process this machine has a record of, running or dead, across every service. */
+export async function listProcessRecords(): Promise<ProcessRecord[]> {
+    let root = os.homedir() + "/" + SERVICE_FOLDER;
+    let instanceFolders: string[];
+    try {
+        instanceFolders = await fs.promises.readdir(root);
+    } catch {
+        return [];
+    }
+    let records: ProcessRecord[] = [];
+    for (let instanceFolder of instanceFolders) {
+        records.push(...await readFolderRecords(path.join(root, instanceFolder)));
+    }
+    return records;
+}
+
+async function removeProcessRecord(record: ProcessRecord): Promise<void> {
+    let files = [
+        getRecordPath(record.folder, record.launchId),
+        getProcessLogPath(record.folder, record.launchId),
+        getPipeScriptPath(record.folder, record.launchId),
+        getTailScriptPath(record.folder, record.launchId),
+    ];
+    for (let file of files) {
+        try {
+            await fs.promises.unlink(file);
+        } catch {
+            // Already gone, which is the state we wanted
+        }
+    }
+}
+
+/** Marks every record whose process is gone as dead, and deletes the ones that have been dead past the retention. */
+export async function syncProcessRecords(config: {
+    isAlive: (record: ProcessRecord) => Promise<boolean>;
+    now: number;
+}): Promise<void> {
+    for (let record of await listProcessRecords()) {
+        if (record.deadTime === undefined && !await config.isAlive(record)) {
+            record.deadTime = config.now;
+            console.log(`Process ${record.launchId} (${record.screenName}) is no longer running, marking it dead. Its log is deleted at ${new Date(record.deadTime + DEAD_PROCESS_RETENTION).toISOString()}`);
+            await writeProcessRecord(record);
+        }
+        if (record.deadTime !== undefined && config.now - record.deadTime > DEAD_PROCESS_RETENTION) {
+            console.log(`Deleting the log of process ${record.launchId} (${record.screenName}), dead since ${new Date(record.deadTime).toISOString()}`);
+            await removeProcessRecord(record);
+        }
+    }
+}