querysub 0.576.0 → 0.578.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.576.0",
+  "version": "0.578.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -71,7 +71,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.45",
+    "sliftutils": "^1.7.49",
     "socket-function": "^1.2.26",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-a-archives/archives2.ts

@@ -7,6 +7,7 @@ import { timeInMinute } from "socket-function/src/misc";
 
 export const STORAGE_ACCOUNT = "root";
 
+
 function createSourceWindows(
     overrides: Partial<RemoteConfigBase>,
     sourceWindows: {
@@ -76,7 +77,7 @@ const archivesBuilderCache = cacheJSONArgsEqual((domain: string, overrides: Part
     domain = domain.toLowerCase().replaceAll(/[^a-z0-9-]/g, "-");
     return archiveBuilder(domain, overrides);
 });
-function nestBucket(folder: string, archives: IArchives): IArchives {
+function nestBucket(folder: string, archives: ReturnType<typeof createArchives>) {
     if (!folder) return archives;
     if (!folder.endsWith("/")) {
         folder = folder + "/";
@@ -94,6 +95,7 @@ function nestBucket(folder: string, archives: IArchives): IArchives {
         hasWriteAccess: () => archives.hasWriteAccess(),
         getConfig: () => archives.getConfig(),
         getURL: (path: string) => archives.getURL(folder + path),
+        getURLs: (path: string) => archives.getURLs(folder + path),
     };
 }
 
@@ -124,4 +126,18 @@ export function getArchives2PublicImmutable(folder: string) {
 export function getArchives2Public(folder: string) {
     let domain = getSafeDomain();
     return nestBucket(folder, archivesBuilderCache(domain + "-public", { public: true, allowedOrigins: getAllowedOrigins(getDomain()) }));
+}
+
+
+export async function initArchives2() {
+    let buckets = [
+        getArchives2(""),
+        getArchives2Buffered(""),
+        getArchives2PrivateImmutable(""),
+        getArchives2PublicImmutable(""),
+        getArchives2Public(""),
+    ];
+    for (let bucket of buckets) {
+        await bucket.hasWriteAccess()
+    }
 }

package/src/3-path-functions/PathFunctionRunner.ts

@@ -764,6 +764,14 @@ export class PathFunctionRunner {
                     throw new Error(`Could not sync stable function spec, aborting call. Retries: ${retries}`);
                 }
 
+                let devModule = getDevelopmentModule(functionSpec.ModuleId);
+                let devFunctionMetadata: FunctionMetadata | undefined;
+                if (devModule) {
+                    let schemaObj = getSchemaObject(devModule);
+                    devFunctionMetadata = schemaObj?.functionMetadata?.[functionSpec.FunctionId];
+                    functionSpec = { ...functionSpec }
+                    functionSpec.FilePath = devModule.filename;
+                }
                 let moduleExports = await this.getExportsFromSpec(functionSpec);
 
                 // ALWAYS get the permissions as well. This isn't perfect, as we could very well access data in other files, but... it helps
@@ -781,13 +789,7 @@ export class PathFunctionRunner {
                     throw new Error(`Export at ${JSON.stringify(exportPath.join("."))} was not a function (it was ${typeof baseFunction}). If in development, you must restart the development function runner server when you add new functions.`);
                 }
 
-                let devFunctionMetadata: FunctionMetadata | undefined;
 
-                let devModule = getDevelopmentModule(functionSpec.ModuleId);
-                if (devModule) {
-                    let schemaObj = getSchemaObject(devModule);
-                    devFunctionMetadata = schemaObj?.functionMetadata?.[functionSpec.FunctionId];
-                }
 
                 console.info(`Running function: ${debugNameColored}`, {
                     callId: callSpec.CallId,

package/src/3-path-functions/pathFunctionLoader.ts

@@ -24,6 +24,7 @@ import { sha256 } from "js-sha256";
 import os from "os";
 import { formatTime } from "socket-function/src/formatting/format";
 import path from "path";
+import pathMod from "path";
 
 export type LoadFunctionSpec = {
     gitURL: string;
@@ -348,9 +349,9 @@ async function getModuleFromSpecBase(
 
         let specFilePath = spec.FilePath;
         if (specFilePath.startsWith("/")) {
-            specFilePath = specFilePath.slice(1);
+            specFilePath = "." + specFilePath;
         }
-        path = packagePath + specFilePath;
+        path = pathMod.resolve(packagePath, specFilePath);
         deployPath = packagePath + "deploy.ts";
     }
 
@@ -372,7 +373,7 @@ async function getModuleFromSpecBase(
             });
         });
     } catch (e: any) {
-        throw new Error(`Error when loading function for ${JSON.stringify(path)}:${spec.FunctionId}\n${e.stack}`);
+        throw new Error(`${!isPublic() && "(IF you added a new module, restart the local function runner so it sees it!) " || ""}Error when loading function for ${JSON.stringify(path)}:${spec.FunctionId}\n${e.stack}`);
     }
 
 

package/src/3-path-functions/syncSchema.ts

@@ -5,7 +5,7 @@ import { Args } from "socket-function/src/types";
 import { appendToPathStr, getPathFromStr, getPathStr, joinPathStres, rootPathStr } from "../path";
 import { writeFunctionCall } from "./PathFunctionHelpers";
 import { CallSpec, functionSchema } from "./PathFunctionRunner";
-import { getDomain, isLocal } from "../config";
+import { getDomain, isLocal, isPublic } from "../config";
 import { isHotReloading } from "socket-function/hot/HotReloadController";
 import { Schema2, Schema2Fncs, Schema2T, SchemaPath } from "../2-proxy/schema2";
 import { PathValueProxyWatcher, atomic, proxyWatcher, registerSchema } from "../2-proxy/PathValueProxyWatcher";
@@ -474,7 +474,9 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
         // NOTE: We clobber the previous value, as we might just be hotreloading, which
         //  could mean the exports value has our previous value.
         module.exports[SCHEMA_EXPORT_KEY] = result;
-        developmentModules.set(moduleId, module);
+        if (!isPublic()) {
+            developmentModules.set(moduleId, module);
+        }
         return result;
     };
 }

package/src/deployManager/components/MachineDetailPage.tsx

@@ -192,7 +192,7 @@ export class MachineDetailPage extends qreact.Component {
                                             Launches: {serviceInfo.totalTimesLaunched}
                                         </div>
                                         <div>
-                                            Guessed Node ID: {serviceInfo.nodeId}
+                                            Guessed Node IDs: {Object.entries(serviceInfo.nodeIds || {}).map(([index, nodeId]) => `#${index} ${nodeId}`).join(", ")}
                                         </div>
                                         {hasError && (
                                             <div className={css.colorhsl(0, 80, 50)}>

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -349,7 +349,7 @@ export class ServiceDetailPage extends qreact.Component {
                                         {screenName}
                                     </div>
                                     <div>
-                                        {serviceInfo?.nodeId || machineId} ({machineInfo.info["getExternalIP"]})
+                                        {serviceInfo?.nodeIds?.[index] || machineId} ({machineInfo.info["getExternalIP"]})
                                     </div>
                                     {Object.keys(variables).length > 0 && <>
                                         {Object.entries(variables).map(([name, value]) =>

package/src/deployManager/components/ServicesListPage.tsx

@@ -4,7 +4,9 @@ import { DEFAULT_OVERLAP_TIME, MachineServiceController, ServiceConfig, getLiveS
 import { css } from "typesafecss";
 import { t } from "../../2-proxy/schema2";
 import { Querysub } from "../../4-querysub/Querysub";
-import { currentViewParam, selectedServiceIdParam } from "../urlParams";
+import { currentViewParam, selectedServiceIdParam, selectedMachineIdParam } from "../urlParams";
+import { managementPageURL } from "../../diagnostics/managementPages";
+import { getServiceCategory } from "../serviceCategories";
 import { formatDateTimeDetailed, formatNiceDateTime, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { cache } from "socket-function/src/caching";
@@ -18,6 +20,26 @@ import { Tools } from "./Tools";
 
 module.hotreload = true;
 
+/** What kind of service this is, when we can tell from its command, linking to wherever that kind is administered. Its own box beside the service's, so clicking the service still opens the service. */
+class ServiceCategoryBadge extends qreact.Component<{ config: ServiceConfig }> {
+    render() {
+        let category = getServiceCategory(getLiveServiceParameters(this.props.config));
+        if (!category) return undefined;
+        let target = category.target;
+        // Leaving the service and machine selections behind, as they mean nothing to the page being opened
+        let values = "view" in target
+            ? [currentViewParam.getOverride(target.view), selectedServiceIdParam.getOverride(""), selectedMachineIdParam.getOverride("")]
+            : [managementPageURL.getOverride(target.managementPage)];
+        return <Anchor noStyles
+            values={values}
+            title={`Administered from ${category.label}`}
+            className={css.pad2(12).button.bord2(0, 0, 20).boldStyle.hbox(0).alignItems("center").alignSelf("stretch").flexShrink0.whiteSpace("nowrap").background(category.color)}
+        >
+            {category.label}
+        </Anchor>;
+    }
+}
+
 export class ServicesListPage extends qreact.Component {
 
     render() {
@@ -117,6 +139,7 @@ export class ServicesListPage extends qreact.Component {
                     let unknown = enabledMachineIds.length - runningMachines.length - failingMachines.length - missingMachines.length;
                     let duplicateKey = (keyCounts.get(config.parameters.key || "") || 0) > 1;
                     return <div className={css.hbox(10)}>
+                        <ServiceCategoryBadge config={config} />
                         <Anchor noStyles key={serviceId}
                             values={[currentViewParam.getOverride("service-detail"), selectedServiceIdParam.getOverride(serviceId)]}
                             className={

package/src/deployManager/components/StoragePage.tsx

@@ -18,10 +18,10 @@ import { STORAGE_ACCOUNT } from "../../-a-archives/archives2";
 import { Table } from "../../5-diagnostics/Table";
 import { RouteConfigView } from "./RouteConfigView";
 import { Tag, SourceTag, getSourceTags } from "./Tag";
+import { STORAGE_COMMAND_PREFIX } from "../serviceCategories";
 
 module.hotreload = true;
 
-const STORAGE_COMMAND_PREFIX = "yarn storageserve";
 const URL_ARG_REGEX = /--url\s+(\S+)/g;
 
 /** The release a storage service is in the middle of. Read straight from the service configs, so it resolves without contacting any storage server - which is exactly when it matters most, since a deploy is the usual reason the servers don't answer. */

package/src/deployManager/components/deployButtons.tsx

@@ -85,9 +85,8 @@ export class UpdateButtons extends qreact.Component<{
             let updated = deepCloneJSON(service);
             updated.parameters.gitRef = latestRef;
             updated.parameters.releaseTime = Date.now();
-            if (noOverlap) {
-                updated.parameters.overlapTime = 0;
-            }
+            // Always set, never inherited: the overlap belongs to the deploy being made, and leaving the previous deploy's value in place means one no-overlap deploy silently makes every later deploy a no-overlap one
+            updated.parameters.overlapTime = noOverlap && 0 || DEFAULT_OVERLAP_TIME;
             return updated;
         });
         let controller = MachineServiceController(SocketFunction.browserNodeId());
@@ -197,34 +196,41 @@ export class UpdateServiceButtons extends qreact.Component<{
     state = t.state({
         isDeploying: t.type(false),
     });
+    // Deploys to latestRef, always immediately. The overlap is what keeps the service up - the old instances keep running for it, so connections have somewhere to go while clients move across. noOverlap kills them at once instead.
+    private deploy(latestRef: string, noOverlap: boolean) {
+        this.state.isDeploying = true;
+        // Props are synchronized state, so everything the async work needs is cloned into plain locals HERE, in the synced part
+        let service = deepCloneJSON(this.props.service);
+        service.parameters.gitRef = latestRef;
+        service.parameters.releaseTime = Date.now();
+        // Always set, never inherited - see deployAll
+        service.parameters.overlapTime = noOverlap && 0 || DEFAULT_OVERLAP_TIME;
+        let controller = MachineServiceController(SocketFunction.browserNodeId());
+        Querysub.onCommitFinished(async () => {
+            try {
+                await controller.setServiceConfigs.promise([service]);
+            } finally {
+                Querysub.commit(() => {
+                    this.state.isDeploying = false;
+                });
+            }
+        });
+    }
     render() {
         let controller = MachineServiceController(SocketFunction.browserNodeId());
         const gitInfo = controller.getGitInfo();
         let gitRef = this.props.service.parameters.gitRef;
         let repoUrl = this.props.service.parameters.repoUrl;
         if (gitInfo?.repoUrl !== repoUrl) return undefined;
+        if (gitRef === gitInfo.latestRef) return undefined;
 
         return <>
-            {gitRef !== gitInfo.latestRef && <button
+            <button
                 className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
                 disabled={this.state.isDeploying}
+                title={`Deploys now; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
                 onClick={() => {
-                    this.state.isDeploying = true;
-                    // Props are synchronized state, so everything the async work needs is cloned into plain locals HERE, in the synced part
-                    let service = deepCloneJSON(this.props.service);
-                    service.parameters.gitRef = gitInfo.latestRef;
-                    // Delay the release by the configured overlap (an unrelated value, but a reasonable delay)
-                    let overlap = service.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME;
-                    service.parameters.releaseTime = Date.now() + overlap;
-                    Querysub.onCommitFinished(async () => {
-                        try {
-                            await controller.setServiceConfigs.promise([service]);
-                        } finally {
-                            Querysub.commit(() => {
-                                this.state.isDeploying = false;
-                            });
-                        }
-                    });
+                    this.deploy(gitInfo.latestRef, false);
                 }}
             >
                 <div>
@@ -238,7 +244,19 @@ export class UpdateServiceButtons extends qreact.Component<{
                     <b>Current</b>
                     <RenderGitRefInfo gitRef={gitRef} />
                 </div>
-            </button>}
+            </button>
+            <button
+                className={buttonStyle.hsl(0, 85, 70).alignSelf("stretch")}
+                disabled={this.state.isDeploying}
+                title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
+                onClick={() => {
+                    this.deploy(gitInfo.latestRef, true);
+                }}
+            >
+                <div>
+                    {bigEmoji("⚡", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || "Update to New, No Overlap"}</span>
+                </div>
+            </button>
         </>;
     }
 }

package/src/deployManager/machineApplyMainCode.ts

@@ -267,8 +267,8 @@ async function ensureScreen(config: {
     let serviceId = config.config.serviceId;
     let serviceConfig = config.config;
 
-    // A version being replaced is renamed aside and given this long before it is taken down, which is the only thing an overlap is
-    let killTime = Date.now() + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
+    // A version being replaced is renamed aside and given until this before it is taken down, which is the only thing an overlap is. Anchored to the release, not to whenever this resync happened to run, so every machine's old process goes at the same moment however staggered the resyncs are.
+    let killTime = (serviceConfig.parameters.releaseTime || Date.now()) + (serviceConfig.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME);
 
     // The screen name means one thing: the version that should be running. There is no second name to reason about - anything it replaced is already renamed aside and carries its own deadline.
     let parameters: ServiceParameters | undefined = getLiveServiceParameters(serviceConfig);
@@ -284,7 +284,8 @@ async function ensureScreen(config: {
         lastLaunchedTime,
         errorFromLastRun: "",
         totalTimesLaunched: launchCount,
-        nodeId: "",
+        // Every instance of this service on this machine shares the entry, so a later index must not wipe out what an earlier one already found
+        nodeIds: machineInfo.services[serviceId]?.nodeIds || {},
     };
 
     try {
@@ -320,11 +321,14 @@ async function ensureScreen(config: {
 
             let nodePathId = folder + SERVICE_NODE_FILE_NAME;
             if (await fsExistsAsync(nodePathId)) {
-                machineInfo.services[serviceId].nodeId = await fs.promises.readFile(nodePathId, "utf8");
+                machineInfo.services[serviceId].nodeIds[index] = await fs.promises.readFile(nodePathId, "utf8");
             }
 
+            // Whether the version in the folder is the one that should be running. Also what tells a crash apart from a deploy further down: unchanged parameters that are not running means it died on its own.
+            let sameParameters = sameRestartParameters(prevParameters, parameters);
+
             // SMOOTH PATH: what is running is not what should be, so it is renamed aside with its own deadline and keeps serving out its overlap there. The name is free from here, which is what makes a replacement and a first start the same thing.
-            if (isUp && panePid && !sameRestartParameters(prevParameters, parameters)) {
+            if (isUp && panePid && !sameParameters) {
                 console.log(green(`Replacing what is running as ${magenta(screenName)}: ${newParametersString}`));
                 // Recorded against the parameters it was actually started with, not the ones replacing it, so its history says what it really ran
                 let outgoingParameters: ServiceParameters | undefined;
@@ -354,6 +358,11 @@ async function ensureScreen(config: {
             if (!isUp && panePid) {
                 await killScreenNow(screenName, "the screen is there but nothing is running in it");
                 panePid = undefined;
+                // The process that registered is already gone, so nothing is going to deregister it - and the replacement started below writes its own registration over the file
+                let deadNodeId = await readServiceNodeId(folder);
+                if (deadNodeId) {
+                    void removeServiceNode({ folder, nodeId: deadNodeId, ownsFile: false });
+                }
             }
 
             // AFTER: EXISTS correct   VERSION correct   RUNNING correct - nothing to start
@@ -395,7 +404,8 @@ async function ensureScreen(config: {
                 serviceKey: parameters.key,
                 screenName,
                 machineId,
-                reason: prevParameters && "update" || "crashed",
+                // Reaching a start with the parameters unchanged means nothing replaced it - it died on its own
+                reason: sameParameters && "crashed" || "update",
                 time: Date.now(),
             });
 

package/src/deployManager/machineSchema.ts

@@ -58,10 +58,8 @@ export type MachineInfo = {
         errorFromLastRun: string;
         // Only times launched for the current applyNodeId, but... still very useful.
         totalTimesLaunched: number;
-        // Might take a while to set (15 minutes or more). It's better to look in the nodes list and find the one that seems to match (maybe looking at the startup path to find it?)
-        // TODO: Make this work correctly when we deploy multiple instances (the node needs to know which index it is so it can write to a unique file).
-        //  - We could probably set environment variables in the screen?
-        nodeId: string;
+        // Keyed by the instance's index on this machine, as one machine can run several instances of a service and they each register as their own node. Might take a while to set (15 minutes or more). It's better to look in the nodes list and find the one that seems to match (maybe looking at the startup path to find it?)
+        nodeIds: Record<number, string>;
     }>;
 };
 
@@ -111,7 +109,7 @@ export type ServiceConfig = {
     };
 };
 
-export const DEFAULT_OVERLAP_TIME = timeInMinute * 10;
+export const DEFAULT_OVERLAP_TIME = timeInSecond * 60;
 
 const TEMPLATE_VARIABLE_REGEX = /\$\{([a-zA-Z0-9_-]+)\}/g;
 /** The `${name}` placeholder names in a command template, in order of first appearance. */

package/src/deployManager/serviceCategories.ts

@@ -0,0 +1,28 @@
+import { ServiceParameters } from "./machineSchema";
+import { ViewType } from "./urlParams";
+import { DISK_PATHVALUE_COLOR, PATHVALUE_COLOR, FUNCTION_RUNNER_COLOR } from "../misc/nodeCategoryColors";
+
+// What a service IS, worked out from the command it runs. The commands are the only signal we have - nothing in the config says "this is a storage server" - so the prefixes live here rather than in whichever page happened to need one first.
+export const STORAGE_COMMAND_PREFIX = "yarn storageserve";
+const PATHVALUE_COMMAND_PREFIX = "yarn server-public";
+const FUNCTION_RUNNER_COMMAND_PREFIX = "yarn function-public";
+
+export type ServiceCategory = {
+    label: string;
+    color: string;
+    /** Where this kind of service is administered: a tab of the deploy manager, or a management page of its own */
+    target: { view: ViewType } | { managementPage: string };
+};
+
+// Ordered, and matched by prefix, so a more specific command would go above a more general one
+const CATEGORIES: { prefix: string; category: ServiceCategory }[] = [
+    { prefix: STORAGE_COMMAND_PREFIX, category: { label: "Storage", color: DISK_PATHVALUE_COLOR, target: { view: "storage" } } },
+    { prefix: PATHVALUE_COMMAND_PREFIX, category: { label: "Path Values", color: PATHVALUE_COLOR, target: { managementPage: "RoutingTablePage" } } },
+    { prefix: FUNCTION_RUNNER_COMMAND_PREFIX, category: { label: "Functions", color: FUNCTION_RUNNER_COLOR, target: { view: "deploy" } } },
+];
+
+/** What kind of service this is, or undefined for one we can't categorize (most services). */
+export function getServiceCategory(parameters: ServiceParameters): ServiceCategory | undefined {
+    let command = parameters.command.trimStart();
+    return CATEGORIES.find(x => command.startsWith(x.prefix))?.category;
+}

package/src/deployManager/setupMachineMain.ts

@@ -7,11 +7,113 @@ import os from "os";
 import readline from "readline";
 import open from "open";
 import { fsExistsAsync } from "../fs";
+import { delay } from "socket-function/src/batching";
 // Import querysub, to fix missing dependencies
 Querysub;
 
 const pinnedNodeVersion = 22;
 
+const SERVICE_NAME = "machine-alwaysup";
+const SERVICE_UNIT_NAME = `${SERVICE_NAME}.service`;
+const DAEMON_SCRIPT_NAME = "machine-daemon.sh";
+// How long after the process dies before it is started again. Short, because until it is back nothing on the machine is being deployed or repaired.
+const SERVICE_RESTART_DELAY_SECONDS = 5;
+// The supervisor holds almost no memory, and it is the only thing that can bring the services back. The kernel should take the services it runs long before it takes it.
+const SERVICE_OOM_SCORE_ADJUST = -500;
+
+// How often the screen is checked, and how long a newly created one gets before it is checked - a fresh pane is a bare shell until the command comes up, which is indistinguishable from a screen whose process died
+const SCREEN_CHECK_INTERVAL_SECONDS = 10;
+const SCREEN_START_GRACE_SECONDS = 15;
+// How long setup waits for the daemon to bring the screen up before treating the install as failed
+const SCREEN_VERIFY_TIMEOUT = 60 * 1000;
+const SCREEN_VERIFY_POLL_INTERVAL = 2000;
+
+/** Keeps the machine process's tmux screen alive: creates it when it is gone, and re-creates it when nothing is running in it. The process stays in the screen where it can be watched, killed and changed by hand - what it can NOT be is its own supervisor, which is how the OOM killer taking the screen left the machine dead with nothing to bring it back.
+ *
+ * Written by setup rather than run out of the repo, so it exists from the moment the unit does - it cannot wait on a querysub publish, and a machine whose checkout is broken is exactly when it has to keep working. */
+function getDaemonScript(home: string): string {
+    return `#! /bin/bash
+
+SESSION=${SERVICE_NAME}
+
+start_session() {
+    echo "$(date -Is) starting the $SESSION screen"
+    tmux kill-session -t "$SESSION" 2>/dev/null || true
+    tmux new-session -d -s "$SESSION"
+    sleep 3
+    tmux send-keys -t "$SESSION" "cd ${home}/machine-alwaysup && yarn machine-alwaysup" Enter
+}
+
+# The same test the deploy code uses on the service screens: a pane whose command exited still has its shell, so the session existing proves nothing - only a child process does.
+is_running() {
+    local pane_pid
+    pane_pid=$(tmux list-panes -t "$SESSION" -F "#{pane_pid}" 2>/dev/null | head -n 1)
+    if [ -z "$pane_pid" ]; then
+        echo "$(date -Is) $SESSION does not exist"
+        return 1
+    fi
+    if ! pgrep -P "$pane_pid" > /dev/null; then
+        echo "$(date -Is) $SESSION exists, but nothing is running in it (pane pid $pane_pid)"
+        return 1
+    fi
+    return 0
+}
+
+while true; do
+    if ! is_running; then
+        start_session
+        sleep ${SCREEN_START_GRACE_SECONDS}
+    fi
+    sleep ${SCREEN_CHECK_INTERVAL_SECONDS}
+done
+`;
+}
+
+/** The daemon that keeps the machine process's screen alive. It exists because a tmux screen is not a supervisor: when the OOM killer took the screen, nothing underneath was left to start it again. */
+function getServiceUnit(user: string, home: string): string {
+    return `[Unit]
+Description=Querysub machine apply (always up)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=${user}
+WorkingDirectory=${home}/machine-alwaysup
+Environment=HOME=${home}
+Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ExecStart=/bin/bash ${home}/${DAEMON_SCRIPT_NAME}
+Restart=always
+RestartSec=${SERVICE_RESTART_DELAY_SECONDS}
+# Everything real runs in tmux screens this daemon starts but does not own - the machine process's own screen, and every service screen under it - and the whole overlap design assumes those outlive a restart of the daemon. Killing the control group would take every one of them down each time, so only the daemon's own process is ever signalled.
+KillMode=process
+# The screens are where processes are expected to die and be restarted; one of them being OOM killed is what this daemon is FOR, and must not stop the unit
+OOMPolicy=continue
+OOMScoreAdjust=${SERVICE_OOM_SCORE_ADJUST}
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target
+`;
+}
+
+function getServiceSudoers(user: string): string {
+    // Both paths, as which one systemctl lives at differs by distro and a sudoers rule only matches the exact path
+    let commands = ["restart", "start", "stop"].flatMap(verb => [
+        `/usr/bin/systemctl ${verb} ${SERVICE_UNIT_NAME}`,
+        `/bin/systemctl ${verb} ${SERVICE_UNIT_NAME}`,
+    ]);
+    return `${user} ALL=(ALL) NOPASSWD: ${commands.join(", ")}\n`;
+}
+
+/** Writes a file on the remote verbatim. Sent base64 encoded, as the content is multi-line and full of the characters that a command line going through ssh, a local shell and a remote shell would each try to interpret. */
+async function writeRemoteFile(sshRemote: string, remotePath: string, content: string, sudo: boolean): Promise<void> {
+    let encoded = Buffer.from(content, "utf8").toString("base64");
+    let write = sudo && `sudo tee ${remotePath} > /dev/null` || `cat > ${remotePath}`;
+    await runPromise(`ssh ${sshRemote} "echo ${encoded} | base64 -d | ${write}"`);
+}
+
 function getGitHubKeyCachePath(repoUrl: string): string {
     let repoOwner = "";
     let repoName = "";
@@ -410,38 +512,69 @@ async function main() {
     await runPromise(`ssh ${sshRemote} "cd ~/machine-alwaysup && yarn install"`);
     console.log("✅ Dependencies installed");
 
-    // 6. Create ~/machine-startup.sh which runs ~/machine-alwaysup/src/deployManager/machine.sh, and setup crontab
-    console.log("Setting up cron job...");
+    // 6. Install the daemon that keeps the machine process running
+    let remoteUser = (await runPromise(`ssh ${sshRemote} "whoami"`)).trim();
+    let remoteHome = (await runPromise(`ssh ${sshRemote} "echo $HOME"`)).trim();
+    console.log(`Installing the ${SERVICE_UNIT_NAME} daemon (user ${remoteUser}, home ${remoteHome})...`);
+    await writeRemoteFile(sshRemote, `~/${DAEMON_SCRIPT_NAME}`, getDaemonScript(remoteHome), false);
+    await runPromise(`ssh ${sshRemote} "chmod +x ~/${DAEMON_SCRIPT_NAME}"`);
+    await writeRemoteFile(sshRemote, `/etc/systemd/system/${SERVICE_UNIT_NAME}`, getServiceUnit(remoteUser, remoteHome), true);
+
+    // Restarting the daemon is how a machine deploy takes effect, and that runs as the service user rather than a login shell, so it can never answer a password prompt
+    let sudoersPath = `/etc/sudoers.d/${SERVICE_NAME}`;
+    await writeRemoteFile(sshRemote, sudoersPath, getServiceSudoers(remoteUser), true);
+    await runPromise(`ssh ${sshRemote} "sudo chmod 0440 ${sudoersPath}"`);
+    // A malformed file here breaks sudo for everything, so it does not get to stay if it does not parse
+    await runPromise(`ssh ${sshRemote} "sudo visudo -cf ${sudoersPath} || sudo rm -f ${sudoersPath}"`);
+
+    // The daemon's journal is where "the screen was not running, starting it" ends up, and a non-root user can only read it from these groups
+    await runPromise(`ssh ${sshRemote} "sudo usermod -aG systemd-journal,adm ${remoteUser}"`, { nothrow: true });
+
+    await runPromise(`ssh ${sshRemote} "sudo systemctl daemon-reload"`);
+    await runPromise(`ssh ${sshRemote} "sudo systemctl enable ${SERVICE_UNIT_NAME}"`);
+    console.log("✅ Daemon installed and enabled at boot");
+
+    // 7. The daemon replaces the @reboot cron entry, which would otherwise start a second copy at every boot
+    let existingCron = await runPromise(`ssh ${sshRemote} "crontab -l"`, { nothrow: true });
+    if (existingCron.includes("machine-startup.sh")) {
+        let remainingCron = existingCron.split("\n").filter(line => !line.includes("machine-startup.sh")).join("\n");
+        await writeRemoteFile(sshRemote, "~/crontab-without-machine-startup.txt", remainingCron, false);
+        await runPromise(`ssh ${sshRemote} "crontab ~/crontab-without-machine-startup.txt && rm ~/crontab-without-machine-startup.txt"`);
+        console.log("✅ Removed the old @reboot cron entry, which the daemon replaces");
+    }
 
-    // Create startup script directly on remote machine
-    await runPromise(`ssh ${sshRemote} "echo '#!/bin/bash' > ~/machine-startup.sh"`);
-    await runPromise(`ssh ${sshRemote} "echo 'cd ~/machine-alwaysup/node_modules/querysub/src/deployManager' >> ~/machine-startup.sh"`);
-    await runPromise(`ssh ${sshRemote} "echo 'bash machine.sh' >> ~/machine-startup.sh"`);
+    // ~/machine-startup.sh keeps its meaning - "(re)start the machine service" - as that is what a machine deploy runs. It is only how it does it that changed.
+    // The screen deliberately survives a restart of the daemon, so restarting the unit alone would leave the old process running - this is the one place that means "take it down and bring it back on the new code", so it kills the screen itself.
+    await writeRemoteFile(sshRemote, "~/machine-startup.sh", `#!/bin/bash
+sudo systemctl stop ${SERVICE_UNIT_NAME}
+tmux kill-session -t ${SERVICE_NAME} 2>/dev/null || true
+sudo systemctl start ${SERVICE_UNIT_NAME}
+`, false);
     await runPromise(`ssh ${sshRemote} "chmod +x ~/machine-startup.sh"`);
 
-    // 7. Setup crontab to run ~/machine-startup.sh on startup
-    const cronEntry = "@reboot ~/machine-startup.sh";
+    // Start the machine service immediately
+    console.log("Starting machine service...");
+    await runPromise(`ssh ${sshRemote} "~/machine-startup.sh"`);
 
-    // Get existing crontab, add our entry if not already present
-    let existingCron = "";
-    try {
-        existingCron = await runPromise(`ssh ${sshRemote} "crontab -l"`, { nothrow: true });
-    } catch {
-        // No existing crontab is fine
+    // The daemon creating the screen is the entire point, so setup does not get to claim success on the unit merely having been started
+    let screenDeadline = Date.now() + SCREEN_VERIFY_TIMEOUT;
+    let screenUp = false;
+    while (Date.now() < screenDeadline) {
+        let panePid = (await runPromise(`ssh ${sshRemote} "tmux list-panes -t ${SERVICE_NAME} -F '#{pane_pid}' 2>/dev/null || true"`, { nothrow: true })).trim();
+        if (panePid) {
+            screenUp = true;
+            break;
+        }
+        await delay(SCREEN_VERIFY_POLL_INTERVAL);
     }
-
-    if (!existingCron.includes("machine-startup.sh")) {
-        // Add cron entry directly
-        await runPromise(`ssh ${sshRemote} "(crontab -l 2>/dev/null || true; echo '${cronEntry}') | crontab -"`);
-        console.log("✅ Cron job added");
-    } else {
-        console.log("✅ Cron job already exists");
+    if (!screenUp) {
+        let status = await runPromise(`ssh ${sshRemote} "systemctl status ${SERVICE_UNIT_NAME} --no-pager"`, { nothrow: true });
+        throw new Error(`The ${SERVICE_UNIT_NAME} daemon did not create the ${SERVICE_NAME} screen within ${SCREEN_VERIFY_TIMEOUT / 1000}s:\n${status}`);
     }
 
-    // Start the machine service immediately
-    console.log("Starting machine service...");
-    await runPromise(`ssh ${sshRemote} "~/machine-startup.sh"`);
     console.log("✅ Machine service started!");
+    console.log(`   Screen:  ssh ${sshRemote} -t "tmux attach -t ${SERVICE_NAME}"`);
+    console.log(`   Status:  ssh ${sshRemote} "systemctl status ${SERVICE_UNIT_NAME}"`);
 
     console.log("\n🎉 Machine setup complete!");
 }

package/src/diagnostics/managementPages.tsx

@@ -397,6 +397,10 @@ class ManagementRoot extends qreact.Component {
                             managementPageURL.getOverride("MachinesPage"),
                             currentViewParam.getOverride("deploy"),
                         ]}>Deploy Application</ATag>
+                        <ATag values={[
+                            managementPageURL.getOverride("MachinesPage"),
+                            currentViewParam.getOverride("storage"),
+                        ]}>Storage</ATag>
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
                         )}

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -21,11 +21,9 @@ import { formatTime, formatNumber } from "socket-function/src/formatting/format"
 import { LatencyGraph, LatencyGraphNode, LatencyGraphLink, LatencyGraphLabelLine } from "../../library-components/LatencyGraph";
 import { URLParam } from "../../library-components/URLParam";
 import { mainResets } from "../../library-components/urlResetGroups";
+import { FUNCTION_RUNNER_COLOR, PATHVALUE_COLOR } from "../../misc/nodeCategoryColors";
 
 const ID_CHARS = 8;
-// The single place these are defined: purple means function runner (networks, function calls, querysub addCalls), blue means path value.
-const FUNCTION_RUNNER_COLOR = "hsl(280, 65%, 72%)";
-const PATHVALUE_COLOR = "hsl(210, 75%, 68%)";
 
 const PROBE_TIMEOUT_MS = 5000;
 const RANGE_BAR_WIDTH_PX = 360;

package/src/misc/nodeCategoryColors.ts

@@ -0,0 +1,5 @@
+// The single place these are defined, so the routing table's graph and the service list's badges can never drift apart on what a colour means.
+export const FUNCTION_RUNNER_COLOR = "hsl(280, 65%, 72%)";
+export const PATHVALUE_COLOR = "hsl(210, 75%, 68%)";
+/** A path value server that keeps its values on disk, rather than only serving them. */
+export const DISK_PATHVALUE_COLOR = "hsl(140, 60%, 65%)";

package/src/deployManager/machine.sh

@@ -1,9 +0,0 @@
-#! /bin/bash
-tmux new -s machine-alwaysup -d
-tmux send-keys -t machine-alwaysup "C-c" Enter
-sleep 3
-# ctrl+c wasn't working, which causes huge issues, so... just kill the screen so it's definitely dead
-tmux kill-session -t machine-alwaysup 2>/dev/null || true
-tmux new -s machine-alwaysup -d
-sleep 3
-tmux send-keys -t machine-alwaysup "cd ~/machine-alwaysup && yarn machine-alwaysup" Enter