querysub 0.195.0 → 0.196.0

package/.cursorrules

@@ -0,0 +1,96 @@
+The code automatically updates on save, so you do not need to ever run commands to run the site.
+
+Always directly set the state (this.state.x = 1), and never use setState.
+
+Unless schema values use atomic or type they will always be returned as a value (as a proxy). Use "in" to check if a value is in a t.lookup.
+
+NEVER EVER pass state to qreact.Component as a template parameter. It should ALSO be declared like so (inside the class):
+    state = t.state({
+        num: t.number,
+        nested: {
+            atomicObj: t.atomic<{ nestedAtomicField: number }>;
+            k: t.string
+        },
+        // All lookups are keyed by string
+        //  (There is no t.array, arrays are not allowed, unless they are inside t.atomic)
+        lookup: t.lookup({
+            userName: t.string
+        })
+    });
+
+    t.lookup objects cannot be set with `this.state.someLookup = {}`. You can add keys or remove keys, and keys are added implicitly (`this.state.someLookup["someKey"] = { x: 1, y: 1 }`, always works, the key is automatically added if it doesn't exist). Removal uses delete `delete this.state.someLookup["someKey"]`.
+
+
+Try not to use "null", and instead always use "undefined".
+
+Never use the ternary operator. Instead, do this: "x ? y : z" => "x && y || z".
+
+If are inside an async Event Handlers, you need to use... Querysub.onCommitFinished(() => ...). and put the async code inside the callback. Then when you set state, you need to put the state setting code inside of Querysub.commit(() => ...).
+
+The site automatically builds and hotreloads. Just save the files. DO NOT try to run any build scripts, because it automatically builds on save. DO NOT RUN "npm run build". DO NOT TRY TO RUN UNIT TESTS. DO NOT TRY TO RUN TESTS.
+
+If you want to make a NON-Button feel like a button, you can use "css.button", which makes the background color change on hover, and make the cursor a pointer. Only use this if the background color is set, otherwise you need to message it's a button in another way. And never use it for <Button> which already does this.
+
+General use hbox/vbox to set the spacing between elements, instead of using margins.
+
+When checking for if a value lies within a range, compare like so: `start <= pos && pos < end`. Specifically, making the pos in the middle.
+
+Never use non-null assertion operator, instead check the value, and if necessary (because it is accessed in nested functions), use a const variable to preserve the type.
+
+Use <InputLabel /> (from "querysub/src/library-components/InputLabel"), instead of <input />. Use <InputLabel textarea /> instead of <textarea />. The input props of this allow you to set a label with "label", to act as a number with "number", as an integer with "integer", as a checkbox with "checkbox", and allow clicking to edit with "edit". There is a callback called "onChangeValue" which returns new values (as opposed to the entire event), which is useful.
+
+Never use em or rem. Only use px or vw/vh/%.
+
+Errors should almost always use a template string to include the expected value, and the actual value, as in: "throw new Error(`Expected X, was ${Y}`);"
+
+We use a dark theme.
+
+Don't use switch statements. Use if statements instead.
+
+Don't use ! when accessing a value from a map. Use the get / if undefined initialize and set, and then use style. It's faster, and more type safe.
+
+Sort with this function, which takes a single function to map each object to a sortable value
+    import { sort } from "socket-function/src/misc";
+    export function sort<T>(arr: T[], sortKey: (obj: T) => unknown);
+
+css should be set using className={css.cssPropertyName(cssPropertyValue).anotherPropertyName...}
+There are also some special aliases, some of which take parameters, some of which don't (which allows them to be chained like so: css.vbox0.wrap):
+let nonCallAliases = {
+    relative: (c: CSSHelperTypeBase) => c.position("relative"),
+    absolute: (c: CSSHelperTypeBase) => c.position("absolute"),
+    fixed: (c: CSSHelperTypeBase) => c.position("fixed"),
+    wrap: (c: CSSHelperTypeBase) => c.flexWrap("wrap").display("flex", "soft").alignItems("center", "soft"),
+    marginAuto: (c: CSSHelperTypeBase) => c.margin("auto"),
+    fillBoth: (c: CSSHelperTypeBase) => c.width("100%").height("100%"),
+    fillWidth: (c: CSSHelperTypeBase) => c.width("100%"),
+    fillHeight: (c: CSSHelperTypeBase) => c.height("100%"),
+    flexShrink0: (c: CSSHelperTypeBase) => c.flexShrink(0),
+    ellipsis: (c: CSSHelperTypeBase) => c.overflow("hidden").textOverflow("ellipsis").whiteSpace("nowrap").display("inline-block"),
+    overflowAuto: (c: CSSHelperTypeBase) => c.overflow("auto"),
+    overflowHidden: (c: CSSHelperTypeBase) => c.overflow("hidden"),
+};
+let callAliases = {
+    hbox: (c: CSSHelperTypeBase, gap: number, rowGap?: number) => c.display("flex").flexDirection("row").rowGap(rowGap ?? gap).columnGap(gap).alignItems("center", "soft"),
+    vbox: (c: CSSHelperTypeBase, gap: number, columnGap?: number) => c.display("flex").flexDirection("column").rowGap(gap).columnGap(columnGap ?? gap).alignItems("start", "soft"),
+    pad2: (c: CSSHelperTypeBase, value: number, verticalValue?: number): CSSHelperTypeBase => {
+        if (verticalValue !== undefined) return c.padding(`${verticalValue}px ${value}px` as any);
+        return c.padding(value);
+    },
+    hsl: (c: CSSHelperTypeBase, h: number, s: number, l: number): CSSHelperTypeBase => c.background(`hsl(${h}, ${s}%, ${l}%)`),
+    hslhover: (c: CSSHelperTypeBase, h: number, s: number, l: number): CSSHelperTypeBase => c.background(`hsl(${h}, ${s}%, ${l}%)`, "hover"),
+    hsla: (c: CSSHelperTypeBase, h: number, s: number, l: number, a: number): CSSHelperTypeBase => c.background(`hsla(${h}, ${s}%, ${l}%, ${a})`),
+    hslahover: (c: CSSHelperTypeBase, h: number, s: number, l: number, a: number): CSSHelperTypeBase => c.background(`hsla(${h}, ${s}%, ${l}%, ${a})`, "hover"),
+    bord: (c: CSSHelperTypeBase, width: number, color: string | { h: number; s: number; l: number; a?: number; }, style = "solid"): CSSHelperTypeBase => {
+        let colorStr = typeof color === "string" ? color : `hsla(${color.h}, ${color.s}%, ${color.l}%, ${color.a ?? 1})`;
+        return c.border(`${width}px ${style} ${colorStr}`);
+    },
+    bord2: (c: CSSHelperTypeBase, h: number, s: number, l: number, width = 1, style = "solid"): CSSHelperTypeBase => {
+        return c.border(`${width}px ${style} hsla(${h}, ${s}%, ${l}%, 1)`);
+    },
+    hslcolor: (c: CSSHelperTypeBase, h: number, s: number, l: number): CSSHelperTypeBase => c.color(`hsl(${h}, ${s}%, ${l}%)`),
+    colorhsl: (c: CSSHelperTypeBase, h: number, s: number, l: number): CSSHelperTypeBase => c.color(`hsl(${h}, ${s}%, ${l}%)`),
+    hslacolor: (c: CSSHelperTypeBase, h: number, s: number, l: number, a: number): CSSHelperTypeBase => c.color(`hsla(${h}, ${s}%, ${l}%, ${a})`),
+    colorhsla: (c: CSSHelperTypeBase, h: number, s: number, l: number, a: number): CSSHelperTypeBase => c.color(`hsla(${h}, ${s}%, ${l}%, ${a})`),
+    size: (c: CSSHelperTypeBase, width: LengthOrPercentage, height: LengthOrPercentage) => c.width(width).height(height),
+    pos: (c: CSSHelperTypeBase, x: LengthOrPercentage, y: LengthOrPercentage) => c.left(x).top(y),
+};

package/bin/machine-alwaysup.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/applyAlwaysUpMain");

package/bin/setup-machine.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/setupMachineMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.195.0",
+  "version": "0.196.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -48,7 +48,9 @@
     "server-public": "./bin/server-public.js",
     "function": "./bin/function.js",
     "function-public": "./bin/function-public.js",
-    "machine": "./bin/machine.js",
+    "machine": "./bin/machine-once.js",
+    "machine-alwaysup": "./bin/machine-alwaysup.js",
+    "setup-machine": "./bin/setup-machine.js",
     "gc": "./bin/gc.js",
     "gc-watch": "./bin/gc-watch.js",
     "gc-watch-public": "./bin/gc-watch-public.js",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -361,8 +361,8 @@ async function runMemoryAuditLoop() {
 async function writeHeartbeat() {
     if (shutdown) return;
     let now = Date.now();
-    console.log(green(`Writing heartbeat ${formatDateTime(now)}`));
     let nodeId = getMountNodeId();
+    console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
     if (!nodeId) return;
     await archives().set(nodeId, Buffer.from(now + ""));
 }

package/src/4-deploy/git.ts

@@ -1,26 +1,14 @@
 import * as child_process from "child_process";
 import { cache } from "socket-function/src/caching";
+import { runPromise } from "../functional/runCommand";
+import { measureWrap } from "socket-function/src/profiling/measure";
+import fs from "fs";
+import os from "os";
 export async function getGitURLLive(gitDir = ".") {
-    return new Promise<string>((resolve, reject) => {
-        child_process.exec(`git remote get-url origin`, { cwd: gitDir }, (error, stdout, stderr) => {
-            if (error) {
-                reject(error);
-            } else {
-                resolve(stdout.toString().trim());
-            }
-        });
-    });
+    return (await runPromise(`git remote get-url origin`, { cwd: gitDir })).trim();
 }
 export async function getGitRefLive(gitDir = ".") {
-    return new Promise<string>((resolve, reject) => {
-        child_process.exec(`git rev-parse HEAD`, { cwd: gitDir }, (error, stdout, stderr) => {
-            if (error) {
-                reject(error);
-            } else {
-                resolve(stdout.toString().trim());
-            }
-        });
-    });
+    return (await runPromise(`git rev-parse HEAD`, { cwd: gitDir })).trim();
 }
 
 export const getGitURLSync = cache(function getGitURLSync(gitDir?: string) {
@@ -28,4 +16,25 @@ export const getGitURLSync = cache(function getGitURLSync(gitDir?: string) {
 });
 export const getGitRefSync = cache(function getGitRefSync(gitDir?: string) {
     return child_process.execSync(`git rev-parse HEAD`, { cwd: gitDir || "." }).toString().trim();
+});
+
+export const setGitRef = measureWrap(async function setGitRef(config: {
+    gitFolder: string;
+    repoUrl: string;
+    gitRef: string;
+}) {
+    let hostKey = await runPromise(`ssh-keyscan -t rsa bitbucket.org`);
+    hostKey = hostKey.split("\n").filter(x => !x.startsWith("#")).join("\n");
+    let knownHostsPath = os.homedir() + "/.ssh/known_hosts";
+    if (!fs.existsSync(knownHostsPath) || !fs.readFileSync(knownHostsPath).toString().includes(hostKey)) {
+        fs.appendFileSync(knownHostsPath, "\n" + hostKey + "\n");
+    }
+
+    await runPromise(`git remote update`, { cwd: config.gitFolder });
+    await runPromise(`git add --all`, { cwd: config.gitFolder });
+    await runPromise(`git stash`, { cwd: config.gitFolder });
+    await runPromise(`git fetch --all`, { cwd: config.gitFolder });
+    await runPromise(`git reset --hard ${config.gitRef}`, { cwd: config.gitFolder });
+    // Allows us to remove deleted objects from storage, ex, if we accidentally commit a 1GB, this deletes it, so we don't have to fix each server individually. Also I think the repo breaks if we go too long without pruning it?
+    await runPromise(`git prune`, { cwd: config.gitFolder });
 });

package/src/4-querysub/Querysub.ts

@@ -1249,13 +1249,6 @@ setImmediate(() => {
     });
 });
 
-if (isNode()) {
-    setImmediate(async () => {
-        const { hookErrors } = await import("../diagnostics/errorLogs/hookErrors");
-        hookErrors();
-    });
-}
-
 setImmediate(async () => {
     // Import, so it registers addStatPeriodic
     await import("../5-diagnostics/nodeMetadata");

package/src/deployManager/MachinesPage.tsx

@@ -0,0 +1,53 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../4-dom/qreact";
+import { DeployController } from "./machineSchema";
+import { css } from "typesafecss";
+import { currentViewParam, selectedServiceIdParam, selectedMachineIdParam } from "./urlParams";
+import { ServicesListPage } from "./components/ServicesListPage";
+import { MachinesListPage } from "./components/MachinesListPage";
+import { ServiceDetailPage } from "./components/ServiceDetailPage";
+import { MachineDetailPage } from "./components/MachineDetailPage";
+
+export class MachinesPage extends qreact.Component {
+    private renderTabs() {
+        return <div className={
+            css.hbox(4)
+            //.bord2(0, 0, 20)
+        }>
+            {[
+                { key: "services", label: "Services", otherKeys: ["service-detail"] },
+                { key: "machines", label: "Machines", otherKeys: ["machine-detail"] }
+            ].map(tab => {
+                let isActive = currentViewParam.value === tab.key || tab.otherKeys.includes(currentViewParam.value);
+                return <div key={tab.key}
+                    className={css.pad2(12, 8).button
+                        + (isActive && css.hsl(0, 0, 100).bord2(210, 50, 50))
+                        + (!isActive && css.hsl(0, 0, 70).colorhsl(0, 0, 50).bord2(0, 0, 50))
+                    }
+                    onClick={() => {
+                        currentViewParam.value = tab.key as any;
+                        selectedServiceIdParam.value = "";
+                        selectedMachineIdParam.value = "";
+                    }}>
+                    {tab.label}
+                </div>;
+            })}
+        </div>;
+    }
+
+    render() {
+        let controller = DeployController(SocketFunction.browserNodeId());
+        let serviceConfigType = controller.getServiceConfigType();
+        if (!serviceConfigType) return <div>Loading...</div>;
+
+        return <div className={css.vbox(0).fillBoth.pad2(20)}>
+            {this.renderTabs()}
+            <div className={css.flexGrow(1).overflowAuto.pad2(20, 10)}>
+                {currentViewParam.value === "services" && <ServicesListPage />}
+                {currentViewParam.value === "machines" && <MachinesListPage />}
+                {currentViewParam.value === "service-detail" && <ServiceDetailPage />}
+                {currentViewParam.value === "machine-detail" && <MachineDetailPage />}
+            </div>
+        </div>;
+    }
+}

package/src/deployManager/applyAlwaysUpMain.js

@@ -0,0 +1,75 @@
+const child_process = require("child_process");
+const fs = require("fs");
+
+export function shellEscape(array) {
+    return array
+        .map((value) => {
+            if (/[^A-Za-z0-9_\/:=-]/.test(value)) {
+                value = "'" + value.replace(/'/g, "'\\''") + "'";
+                value = value
+                    .replace(/^(?:'')+/g, "") // unduplicate single-quote at the beginning
+                    .replace(/\\'''/g, "\\'"); // remove non-escaped single-quote if there are enclosed between 2 escaped
+            }
+            return value;
+        })
+        .join(" ");
+}
+
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function runCommandWithLogging(command) {
+    return new Promise((resolve, reject) => {
+        const childProc = child_process.spawn(command, {
+            shell: true,
+            stdio: ["inherit", "pipe", "pipe"], // stdin: inherit, stdout: pipe, stderr: pipe
+        });
+
+        let combinedOutput = "";
+        function limitOutputSize(output) {
+            if (output.length > 100 * 1000) {
+                output = output.slice(-50 * 1000);
+            }
+        }
+
+        // Capture and log stdout
+        childProc.stdout?.on("data", (data) => {
+            const chunk = data.toString();
+            combinedOutput += chunk;
+            limitOutputSize(chunk);
+            process.stdout.write(chunk);
+        });
+
+        // Capture and log stderr
+        childProc.stderr?.on("data", (data) => {
+            const chunk = data.toString();
+            combinedOutput += chunk;
+            limitOutputSize(chunk);
+            process.stderr.write(chunk);
+        });
+
+        childProc.on("error", (err) => {
+            reject(combinedOutput.slice(-1000));
+        });
+
+        childProc.on("close", (code) => {
+            reject(`${combinedOutput.slice(-1000)}\nProcess exited with code ${code}, recent output:`);
+        });
+    });
+}
+
+async function main() {
+    let args = process.argv.slice(2);
+    while (true) {
+        try {
+            await runCommandWithLogging(shellEscape(args));
+        } catch (e) {
+            console.error(`Process finished, restarting in 5 seconds`);
+            fs.writeFileSync("lastAlwaysUpError.txt", e.stack);
+        }
+        await delay(5000);
+    }
+}
+
+main().catch(console.error);

package/src/deployManager/components/MachineDetailPage.tsx

@@ -0,0 +1,159 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../4-dom/qreact";
+import { MACHINE_RESYNC_INTERVAL, DeployController, ServiceConfig } from "../machineSchema";
+import { css } from "typesafecss";
+import { currentViewParam, selectedMachineIdParam, selectedServiceIdParam } from "../urlParams";
+import { formatVeryNiceDateTime } from "socket-function/src/formatting/format";
+import { sort } from "socket-function/src/misc";
+
+export class MachineDetailPage extends qreact.Component {
+    render() {
+        const selectedMachineId = selectedMachineIdParam.value;
+        if (!selectedMachineId) return <div>No machine selected</div>;
+
+        let controller = DeployController(SocketFunction.browserNodeId());
+        let machineInfo = controller.getMachineInfo(selectedMachineId);
+        let serviceList = controller.getServiceList();
+
+        if (controller.isAnyLoading()) return <div>Loading machine info...</div>;
+        if (!machineInfo) return <div>Machine not found</div>;
+        if (!serviceList) return <div>Service list not found</div>;
+
+        const machine = machineInfo; // Create const reference for type safety
+
+        // Get all service configs that target this machine
+        let relevantServiceConfigs = new Map<string, ServiceConfig>();
+        for (let serviceId of serviceList) {
+            let serviceConfig = controller.getServiceConfig(serviceId);
+            if (serviceConfig && serviceConfig.machineIds.includes(selectedMachineId)) {
+                relevantServiceConfigs.set(serviceId, serviceConfig);
+            }
+        }
+
+        // Find configured services that aren't running
+        let configuredButNotRunning = Array.from(relevantServiceConfigs.entries()).filter(([serviceId]) =>
+            !(serviceId in machine.services)
+        );
+        sort(configuredButNotRunning, x => x[0]); // Sort by service ID
+
+        // Sort running services
+        let runningServices = Object.entries(machine.services);
+        sort(runningServices, x => -x[1].lastLaunchedTime); // Sort by last launched time, most recent first
+
+        const isMachineDead = Date.now() - machine.heartbeat > (MACHINE_RESYNC_INTERVAL * 4);
+
+        return <div className={css.vbox(16)}>
+            <div className={css.hbox(12)}>
+                <h2 className={css.flexGrow(1)}>{selectedMachineId}</h2>
+                {isMachineDead && <div className={css.colorhsl(0, 80, 60)}>
+                    ⚠️ Machine is likely dead
+                </div>}
+            </div>
+
+            <div className={css.vbox(12)}>
+                <div>
+                    <h3>Machine Info</h3>
+                    <div>
+                        Last heartbeat: {formatVeryNiceDateTime(machine.heartbeat)}
+                    </div>
+                    <div>
+                        Apply Node ID: {machine.applyNodeId}
+                    </div>
+                </div>
+
+                <div>
+                    <h3>System Information</h3>
+                    <div className={css.vbox(4)}>
+                        {Object.entries(machine.info).map(([key, value]) => (
+                            <div key={key}>
+                                <span>{key}:</span> {value}
+                            </div>
+                        ))}
+                    </div>
+                </div>
+
+                {configuredButNotRunning.length > 0 && (
+                    <div>
+                        <h3>⚠️ Configured Services Not Running ({configuredButNotRunning.length})</h3>
+                        <div className={css.vbox(8)}>
+                            {configuredButNotRunning.map(([serviceId, serviceConfig]) => (
+                                <div key={serviceId}
+                                    className={
+                                        css.pad2(12).button.bord2(0, 0, 20).hsl(0, 70, 90)
+                                    }
+                                    onClick={() => {
+                                        currentViewParam.value = "service-detail";
+                                        selectedServiceIdParam.value = serviceId;
+                                    }}>
+                                    <div className={css.hbox(12)}>
+                                        <div className={css.flexGrow(1)}>
+                                            {serviceConfig.info.title} ({serviceConfig.parameters.key} / {serviceId})
+                                        </div>
+                                        <div className={css.colorhsl(0, 80, 60)}>
+                                            ⚠️ Not Running
+                                        </div>
+                                        <div>
+                                            Deploy: {serviceConfig.parameters.deploy ? "Yes" : "No"}
+                                        </div>
+                                    </div>
+                                </div>
+                            ))}
+                        </div>
+                    </div>
+                )}
+
+                <div>
+                    <h3>Services ({Object.keys(machine.services).length} running)</h3>
+                    <div className={css.vbox(8)}>
+                        {runningServices.map(([serviceId, serviceInfo]) => {
+                            let serviceConfig = relevantServiceConfigs.get(serviceId);
+                            let isConfigured = relevantServiceConfigs.has(serviceId);
+                            let hasError = serviceInfo.errorFromLastRun;
+
+                            let backgroundColor = css.hsl(0, 0, 100); // Default: white
+                            if (!isConfigured) {
+                                backgroundColor = css.hsl(30, 70, 90); // Orange-ish for unconfigured
+                            } else if (hasError) {
+                                backgroundColor = css.hsl(0, 70, 90); // Red-ish for errors
+                            }
+
+                            return (
+                                <div key={serviceId}
+                                    className={
+                                        css.pad2(12).button.bord2(0, 0, 20)
+                                        + backgroundColor
+                                    }
+                                    onClick={() => {
+                                        currentViewParam.value = "service-detail";
+                                        selectedServiceIdParam.value = serviceId;
+                                    }}>
+                                    <div className={css.hbox(12)}>
+                                        <div className={css.flexGrow(1)}>
+                                            {serviceConfig && <>{serviceConfig.info.title} ({serviceConfig.parameters.key})</> || <>serviceId = {serviceId}</>}
+                                        </div>
+                                        {!isConfigured && (
+                                            <div className={css.colorhsl(30, 80, 50)}>
+                                                ⚠️ Orphaned screen with no service config
+                                            </div>
+                                        )}
+                                        <div>
+                                            Last deployed: {formatVeryNiceDateTime(serviceInfo.lastLaunchedTime)}
+                                        </div>
+                                        <div>
+                                            Launches: {serviceInfo.totalTimesLaunched}
+                                        </div>
+                                        {hasError && (
+                                            <div className={css.colorhsl(0, 80, 50)}>
+                                                ⚠️ Has Error
+                                            </div>
+                                        )}
+                                    </div>
+                                </div>
+                            );
+                        })}
+                    </div>
+                </div>
+            </div>
+        </div>;
+    }
+}