querysub 0.556.0 → 0.558.0

package/src/deployManager/processManager.ts

@@ -0,0 +1,369 @@
+import os from "os";
+import fs from "fs";
+import path from "path";
+import { spawn, ChildProcess } from "child_process";
+import { lazy } from "socket-function/src/caching";
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { delay, runInSerial } from "socket-function/src/batching";
+import { red, green } from "socket-function/src/formatting/logColors";
+import { runPromise } from "../functional/runCommand";
+import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
+import { fsExistsAsync } from "../fs";
+import { PromiseObj } from "../promise";
+import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
+import { ProcessRecord, createLaunchId, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, writeProcessRecord } from "./processLogs";
+
+// Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
+
+const SCREEN_SUFFIX = "-dply";
+export function getScreenName(config: { serviceKey: string; index: number }): string {
+    return `${config.serviceKey}-${config.index}${SCREEN_SUFFIX}`.replace(/[^a-zA-Z0-9\-_]/g, "_");
+}
+// The new version's screen during a release, in the SAME folder as the canonical screen: created (just echoing when it will start) shortly before releaseTime, started at releaseTime, and renamed to the canonical screen name once the old screen is killed at releaseTime + overlapTime.
+export function getFutureScreenName(canonicalScreenName: string): string {
+    return canonicalScreenName.slice(0, -SCREEN_SUFFIX.length) + "-future" + SCREEN_SUFFIX;
+}
+
+
+async function removeOldNodeId(screenName: string) {
+    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
+    if (await fsExistsAsync(nodeIdFile)) {
+        let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
+        console.log(green(`Removing node if for dead service on ${nodeIdFile}, node id ${nodeId}`));
+        await fs.promises.unlink(nodeIdFile);
+        await forceRemoveNode(nodeId);
+    }
+}
+
+export const getTmuxPrefix = lazy(() => {
+    if (os.platform() === "win32") {
+        return "C:/cygwin64/bin/";
+        // C:/cygwin64/bin/tmux new -s server1 -d
+    }
+    return "";
+});
+function textTableLineToObj(text: string): Record<string, string>[] {
+    /*
+      PID    PPID    PGID     WINPID   TTY         UID    STIME COMMAND
+     1312    1284    1312      56996  pty2      197609 09:24:47 /usr/bin/bash
+     1313    1284    1313      56997  pty3      197609 09:24:48 /usr/bin/vim
+    =>
+    [{ PID: "1312", PPID: "1284", ...}, { PID: "1313", PPID: "1284", ...}]
+    */
+    let lines = text.split("\n").filter(line => line.trim().length > 0);
+    if (lines.length < 2) {
+        return [];
+    }
+
+    let headerLine = lines[0];
+    let dataLines = lines.slice(1);
+
+    // Parse column positions from header
+    let headerWords = headerLine.match(/\S+/g) || [];
+    if (headerWords.length === 0) return [];
+
+    // Find column boundaries: start at 0, then at end of each header word (except last), then start of last header, then end of line
+    let boundaries: number[] = [0];
+    let searchPos = 0;
+    for (let i = 0; i < headerWords.length; i++) {
+        let word = headerWords[i];
+        searchPos = headerLine.indexOf(word, searchPos) + word.length;
+        boundaries.push(searchPos);
+    }
+    boundaries[boundaries.length - 1] = Number.MAX_SAFE_INTEGER;
+
+    // Create column definitions using boundaries
+    let columns: { name: string; start: number; end: number }[] = [];
+    for (let i = 0; i < headerWords.length; i++) {
+        columns.push({
+            name: headerWords[i],
+            start: boundaries[i],
+            end: boundaries[i + 1]
+        });
+    }
+
+    // Extract values from all data lines
+    let results: Record<string, string>[] = [];
+    for (let dataLine of dataLines) {
+        let result: Record<string, string> = {};
+        for (let column of columns) {
+            let value = dataLine.substring(column.start, column.end).trim();
+            result[column.name] = value;
+        }
+        results.push(result);
+    }
+
+    return results;
+}
+const getLinuxChildPids = measureWrap(async function getLinuxChildPids(pid: string): Promise<{ PID: string; PPID: string; CMD: string }[]> {
+    let prefix = getTmuxPrefix();
+    if (os.platform() === "win32") {
+        let table = await runPromise(`${prefix}ps`, { quiet: true });
+        let obj = textTableLineToObj(table) as { PID: string; PPID: string; CMD: string }[];
+        return obj.filter(x => x.PPID === pid);
+    } else {
+        let table = await runPromise(`ps -eo pid,ppid,cmd`, { quiet: true });
+        let obj = textTableLineToObj(table) as { PPID: string; PID: string; CMD: string }[];
+        return obj.filter(x => x.PPID === pid);
+    }
+});
+// One process-table read for the whole machine — a pid is "running something" when any process has it as a parent. Used by getScreenState so a resync doesn't spawn one ps per screen (which made the release boundary ticks wait on every idle screen first).
+const getAllParentPids = measureWrap(async function getAllParentPids(): Promise<Set<string>> {
+    let prefix = getTmuxPrefix();
+    let table = os.platform() === "win32"
+        ? await runPromise(`${prefix}ps`, { quiet: true })
+        : await runPromise(`ps -eo pid,ppid,cmd`, { quiet: true });
+    let obj = textTableLineToObj(table) as { PID: string; PPID: string }[];
+    return new Set(obj.map(x => x.PPID));
+});
+/** The start time the OS reports for a pid, as an epoch time in milliseconds like every other time we store. Asked of the machine rather than remembered, so it stays true across our restarts and catches a reused pid. Returns 0 when the process is gone. */
+export const getProcessStartTime = measureWrap(async function getProcessStartTime(pid: string): Promise<number> {
+    // ps only resolves to the second, so this is second-accurate - which is plenty to tell two processes on the same pid apart
+    let started = await runPromise(`date -d "$(ps -o lstart= -p ${pid})" +%s`, { quiet: true });
+    return (parseInt(started.trim()) || 0) * 1000;
+});
+
+export const isScreenRunningProcess = measureWrap(async function isScreenRunningProcess(pid: string): Promise<boolean> {
+    try {
+        let bashChildPids = await getLinuxChildPids(pid);
+        for (let childPid of bashChildPids) {
+            console.log(`Screen pid ${pid} is running ${childPid.CMD}`);
+            return true;
+        }
+        console.log(`Screen pid ${pid} is not running anything.`);
+        return false;
+    } catch (e: any) {
+        console.warn(`Error checking if screen is running for ${pid}: ${e.stack}`);
+        return false;
+    }
+});
+export const getScreenState = measureWrap(async function getScreenState(populateIsProcessRunning: boolean = true): Promise<{
+    screenName: string;
+    isProcessRunning: boolean;
+    pid: string;
+}[]> {
+    const prefix = getTmuxPrefix();
+    const delimit = "::::";
+    // Use list-sessions instead of list-panes -a to avoid zombie sessions
+    // 2>/dev/null suppresses "no server running" errors, -r prevents xargs from running if input is empty
+    let screenList = (await runPromise(`${prefix}tmux list-sessions -F "#{session_name}" 2>/dev/null | xargs -r -I {} tmux list-panes -t {} -F "{}${delimit}#{pane_pid}"`))
+        .split("\n")
+        .filter(x => x.includes(delimit))
+        .map(x => ({
+            screenName: x.split(delimit)[0].trim(),
+            pid: x.split(delimit)[1].trim(),
+            isProcessRunning: false,
+        }))
+        .filter(x => x.screenName.endsWith(SCREEN_SUFFIX))
+        ;
+
+    if (populateIsProcessRunning) {
+        let parentPids = await getAllParentPids();
+        for (let x of screenList) {
+            x.isProcessRunning = parentPids.has(x.pid);
+        }
+    }
+
+    return screenList;
+});
+
+export const runScreenCommand = measureWrap(async function runScreenCommand(config: {
+    screenName: string;
+    command: string;
+    // Defaults to the folder derived from screenName; a future screen passes its canonical screen's folder
+    folder?: string;
+    // Identifies the process this launch creates, so its log and its configuration are stored against it
+    record: Omit<ProcessRecord, "launchId" | "folder" | "screenName" | "startTime">;
+}): Promise<string> {
+    let prefix = getTmuxPrefix();
+    let screenName = config.screenName;
+
+    try {
+        // Throw if it already exists
+        await runPromise(`${prefix}tmux new -s ${screenName} -d`);
+    } catch { }
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'echo "Updating running command at ${new Date().toISOString()}"' Enter`);
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
+    await delay(1000);
+
+
+    let screens = await getScreenState();
+    let screen = screens.find(x => x.screenName === screenName);
+    let pid = screen?.pid;
+    if (pid && await isScreenRunningProcess(pid)) {
+        // It doesn't want to die. Wait longer, but it it just won't die, kill the screen
+        console.warn(`Screen ${screenName} is not dying, giving it another 30 seconds`);
+        for (let i = 0; i < 6; i++) {
+            await delay(5);
+            if (!await isScreenRunningProcess(pid)) {
+                break;
+            }
+        }
+        if (pid && await isScreenRunningProcess(pid)) {
+            console.warn(`Screen ${screenName} is still running, killing it forcefully`);
+            await killScreen({ screenName });
+            if (pid && await isScreenRunningProcess(pid)) {
+                console.error(`I don't know what happened. The screen won't die. We can't do much else, I guess we'll just ignore it...`);
+            } else {
+                // Nested, to create the screen again.
+                return await runScreenCommand({
+                    screenName,
+                    command: config.command,
+                    folder: config.folder,
+                    record: config.record,
+                });
+            }
+        }
+    }
+    await removeOldNodeId(screenName);
+    let folder = config.folder || os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
+
+    // The pane we are about to run the command in identifies the process: its pid, and the start time the OS reports for that pid. Both can be re-checked against the machine later, so nothing downstream has to trust our bookkeeping about what is still running.
+    let panePid = (await getScreenState(false)).find(x => x.screenName === screenName)?.pid;
+    if (!panePid) {
+        throw new Error(`Screen ${screenName} does not exist after creating it, so there is no process to run the command in`);
+    }
+    let startTime = await getProcessStartTime(panePid);
+    let launchId = createLaunchId(panePid, startTime);
+    await writeProcessRecord({
+        ...config.record,
+        launchId,
+        folder,
+        screenName,
+        pid: parseInt(panePid) || undefined,
+        startTime,
+    });
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
+    let command = `#!/bin/bash
+${config.command}
+`;
+    await fs.promises.writeFile(folder + "command.sh", command);
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
+
+    await setupPipePane({ screenName, folder, launchId });
+    return launchId;
+});
+
+// Points the screen's pipe-pane at this launch's own log file.
+async function setupPipePane(config: { screenName: string; folder: string; launchId: string }) {
+    let prefix = getTmuxPrefix();
+    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let pipeScript = getPipeScriptPath(config.folder, config.launchId);
+    await fs.promises.mkdir(path.dirname(logPath), { recursive: true });
+    await fs.promises.writeFile(pipeScript, getLogPipeScript(logPath));
+    await runPromise(`chmod +x ${pipeScript}`);
+    await runPromise(`${prefix}tmux pipe-pane -t ${config.screenName} 'bash ${pipeScript}'`);
+}
+
+export const killScreen = measureWrap(async function killScreen(config: {
+    screenName: string;
+    // During a takeover the folder's nodeId file already belongs to the NEW process, so the old screen's kill must not remove it
+    skipNodeIdRemoval?: boolean;
+}) {
+    console.log(red(`Killing screen ${config.screenName}`));
+    let prefix = getTmuxPrefix();
+    // Try ctrl+c a few times first
+    let pid = (await getScreenState(false)).find(x => x.screenName === config.screenName)?.pid;
+    for (let i = 0; i < 5; i++) {
+        if (!pid || !await isScreenRunningProcess(pid)) {
+            break;
+        }
+        await runPromise(`${prefix}tmux send-keys -t ${config.screenName} 'C-c' Enter`);
+        await delay(5000);
+    }
+    await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
+    if (!config.skipNodeIdRemoval) {
+        await removeOldNodeId(config.screenName);
+    }
+});
+
+/** Streams one process's log: everything already in it, then everything appended after. One process, one file - nothing here has to reason about which process a byte came from. */
+export async function streamProcessOutput(config: {
+    folder: string;
+    launchId: string;
+    onData: (data: string) => Promise<void>;
+}) {
+    let logPath = getProcessLogPath(config.folder, config.launchId);
+    let serialOnData = runInSerial(config.onData);
+    let stopped = false;
+    let childProcess: ChildProcess | undefined;
+
+    let pendingDataCalls = 0;
+    const MAX_PENDING_CALLS = 100;
+
+    async function stop() {
+        if (stopped) return;
+        stopped = true;
+        if (childProcess) {
+            childProcess.kill();
+        }
+    }
+
+    const onDataWrapped = async (data: string) => {
+        pendingDataCalls++;
+        if (pendingDataCalls > MAX_PENDING_CALLS) {
+            console.error(`Too many queued onData calls for ${config.launchId}, stopping stream.`);
+            await stop();
+            return;
+        }
+        try {
+            await serialOnData(data);
+        } catch (e: any) {
+            console.log(`Callback for stream output ${config.launchId} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
+            await stop();
+        } finally {
+            pendingDataCalls--;
+        }
+    };
+
+    try {
+        let tailScript = getTailScriptPath(config.folder, config.launchId);
+        await fs.promises.writeFile(tailScript, getLogTailScript(logPath));
+        await runPromise(`chmod +x ${tailScript}`);
+
+        // Read what is already there ourselves and deliver it as one call - letting the tail script cat it
+        // makes runInSerial dribble it out one round trip at a time.
+        let initialContent = "";
+        try {
+            initialContent = await fs.promises.readFile(logPath, "utf8");
+        } catch {
+            // The process may not have written anything yet; the tail script picks it up when it does.
+        }
+
+        childProcess = spawn("bash", [tailScript, String(Buffer.byteLength(initialContent, "utf8"))], {
+            stdio: "pipe",
+        });
+
+        let started = new PromiseObj<void>();
+
+        childProcess.stdout?.on("data", (data) => {
+            if (stopped) return;
+            started.resolve();
+            void onDataWrapped(data.toString());
+        });
+        // Give it some time to error out, otherwise, just start
+        setTimeout(() => started.resolve(), 200);
+
+        childProcess.stderr?.on("data", (data) => {
+            if (stopped) return;
+            void onDataWrapped(red(data.toString()));
+        });
+
+        childProcess.on("error", async (err) => {
+            if (stopped) return;
+            started.reject(err);
+        });
+
+        if (initialContent) {
+            // Queued synchronously here, before any stdout "data" event can fire, so it always lands first
+            started.resolve();
+            void onDataWrapped(initialContent);
+        }
+
+        await started.promise;
+    } catch (e) {
+        void stop();
+        throw e;
+    }
+}
+

package/src/deployManager/setupMachineMain.ts

@@ -1,4 +1,4 @@
-import { getBackblazePath } from "../../appSecrets";
+import { getBackblazePath } from "../misc/appPaths";
 import { getGitURLLive, getGitRefLive } from "../4-deploy/git";
 import { Querysub } from "../4-querysub/Querysub";
 import { runPromise } from "../functional/runCommand";

package/src/diagnostics/trackResources.ts

@@ -3,7 +3,6 @@ import { lazy } from "socket-function/src/caching";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { blue } from "socket-function/src/formatting/logColors";
 import { isNode } from "socket-function/src/misc";
-import { registerPeriodic } from "./periodic";
 import { registerMeasureInfo } from "socket-function/src/profiling/measure";
 import { logNodeStateStats } from "../-0-hooks/hooks";
 
@@ -89,4 +88,7 @@ registerMeasureInfo(() => {
     return `MEM ${formatNumber(getUsedHeapSize())}B+${formatNumber(getBufferUsage())}B/${formatNumber(getHeapSize())}B `;
 });
 
-registerPeriodic(logResourcesNow);
+setImmediate(async () => {
+    let { registerPeriodic } = await import("./periodic");
+    registerPeriodic(logResourcesNow);
+});

package/src/library-components/URLParam.ts

@@ -170,6 +170,12 @@ export function createURLSync<T>(urlKey: string, defaultValue: T, config?: URLPa
     return param;
 }
 
+if (!syncSchema) {
+    debugger;
+    require("debugbreak")(2);
+    debugger;
+}
+
 // TODO: Support pathname keys (ex /name-value)
 const { data } = syncSchema<{
     params: {

package/src/misc/appPaths.ts

@@ -0,0 +1,56 @@
+import fs from "fs";
+import os from "os";
+import { isNode } from "socket-function/src/misc";
+import { lazy } from "socket-function/src/caching";
+import { parseArgsFactory } from "./rawParams";
+
+// Where the machine's own configuration and credentials live on disk. Deliberately dependency-free (beyond rawParams, which has no dependencies of its own): the storage layer needs these paths, and it sits far below the application - routing this through appSecrets.ts made the lowest-level archives code depend on Querysub.
+
+// getSecret always imports appSecrets relative to the process cwd, so the cwd is the repo root and this matches what getStorageDir would resolve
+export const STORAGE_DIR = "./database-storage/";
+
+// querysubConfig and getDomain are copied verbatim from src/config.ts, so this file has no dependencies on our own application
+
+let yargObj = parseArgsFactory()
+    .option("domain", { type: "string", desc: `Sets the domain` })
+    .argv
+    ;
+
+type QuerysubConfig = {
+    domain?: string;
+    emaildomain?: string;
+    notifyemails?: string[];
+};
+let querysubConfig = lazy((): QuerysubConfig => {
+    if (!isNode()) throw new Error("querysubConfig is only available on the server");
+    const path = "./querysub.json";
+    if (!fs.existsSync(path)) {
+        return {};
+    }
+    try {
+        return JSON.parse(fs.readFileSync(path, "utf8"));
+    } catch (e) {
+        console.error("Error parsing querysub.json", e);
+        return {};
+    }
+});
+
+export function getDomain() {
+    if (!isNode()) {
+        return location.hostname.split(".").slice(-2).join(".");
+    }
+    return yargObj.domain || querysubConfig().domain || "querysub.com";
+}
+
+export function getBackblazePath() {
+    let testPaths = [
+        STORAGE_DIR + "backblaze.json",
+        os.homedir() + "/backblaze.json",
+    ];
+    for (let path of testPaths) {
+        if (fs.existsSync(path)) {
+            return path;
+        }
+    }
+    return testPaths[0];
+}