querysub 0.252.0 → 0.254.0

package/src/deployManager/components/deployButtons.tsx

@@ -0,0 +1,217 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { qreact } from "../../4-dom/qreact";
+import { MachineServiceController, MachineInfo, ServiceConfig } from "../machineSchema";
+import { showFullscreenModal } from "../../5-diagnostics/FullscreenModal";
+import { InputLabel } from "../../library-components/InputLabel";
+import { css } from "../../4-dom/css";
+import { formatTime } from "socket-function/src/formatting/format";
+import { formatTimeJSX } from "../../misc/formatJSX";
+import { MachineController } from "../machineController";
+import { closeAllModals } from "../../5-diagnostics/Modal";
+
+module.hotreload = true;
+
+function bigEmoji(emoji: string, topOffset = 0) {
+    return <span className={css.fontSize(26).marginTop(topOffset)}>{emoji}</span>;
+}
+
+export class RenderGitRefInfo extends qreact.Component<{
+    gitRef: string;
+    isQuerysub?: boolean;
+}> {
+    render() {
+        let controller = MachineServiceController(SocketFunction.browserNodeId());
+        let gitRefInfo = controller.getGitRefInfo({
+            ref: this.props.gitRef,
+            useQuerysub: this.props.isQuerysub,
+        });
+        if (!gitRefInfo) return undefined;
+        return <div className={css.fontWeight("normal")}>{formatTimeJSX(gitRefInfo.time)} AGO {gitRefInfo.description}</div>;
+    }
+}
+
+const buttonStyle = css.pad2(12, 8).button.bord2(0, 0, 20).fontWeight("bold").vbox(0).alignItems("center");
+
+export class UpdateButtons extends qreact.Component<{
+    services: ServiceConfig[];
+}> {
+    render() {
+        let controller = MachineServiceController(SocketFunction.browserNodeId());
+        const gitInfo = controller.getGitInfo();
+
+        let outdatedServices = this.props.services.filter(service => {
+            let { gitRef, repoUrl } = service.parameters;
+            return repoUrl === gitInfo?.repoUrl && gitRef !== gitInfo?.latestRef;
+        });
+        let outdatedRefs = outdatedServices.map(service => service.parameters.gitRef);
+
+        return <>
+            {!!gitInfo?.querysubUncommitted.length && <button
+                className={buttonStyle.hsl(260, 60, 55)}
+                title={gitInfo?.querysubUncommitted.join("\n")}
+                onClick={() => {
+                    showFullscreenModal({
+                        content: <div>
+                            <InputLabel
+                                label="Commit message (shift+enter to commit, escape to cancel)"
+                                value={""}
+                                textarea
+                                fillWidth
+                                focusOnMount
+                                onChangeValue={async value => {
+                                    if (!value) return;
+                                    await controller.commitPushAndPublishQuerysub.promise(value);
+                                    closeAllModals();
+                                }}
+                            />
+                            <div className={css.vbox(5)}>
+                                {gitInfo?.querysubUncommitted.map(change => {
+                                    return <div>
+                                        {change}
+                                    </div>;
+                                })}
+                            </div>
+                        </div>
+                    });
+                }}
+            >
+                <div>
+                    {bigEmoji("📚", -5)}
+                    Publish & Commit & Push Querysub & Update Package.json ({gitInfo?.querysubUncommitted.length} Files Changed)
+                </div>
+                <RenderGitRefInfo gitRef={gitInfo.querysubRef} isQuerysub />
+            </button>}
+            {!!gitInfo?.uncommitted.length && <button
+                className={buttonStyle.hsl(210, 40, 50)}
+                title={gitInfo?.uncommitted.join("\n")}
+                onClick={() => {
+                    showFullscreenModal({
+                        content: <div>
+                            <InputLabel
+                                label="Commit message (shift+enter to commit, escape to cancel)"
+                                value={""}
+                                textarea
+                                fillWidth
+                                focusOnMount
+                                onChangeValue={async value => {
+                                    if (!value) return;
+                                    await controller.commitPushService.promise(value);
+                                    closeAllModals();
+                                }}
+                            />
+                            <div className={css.vbox(5)}>
+                                {gitInfo?.uncommitted.map(change => {
+                                    return <div>
+                                        {change}
+                                    </div>;
+                                })}
+                            </div>
+                        </div>
+                    });
+                }}
+            >
+                {bigEmoji("⬆️")} <span>Commit & Push ({gitInfo?.uncommitted.length} Files Changed)</span>
+            </button>}
+
+            {outdatedServices.length > 0 && gitInfo &&
+                <button
+                    className={buttonStyle.hsl(120, 70, 50)}
+                    onClick={async () => {
+                        for (let service of outdatedServices) {
+                            service.parameters.gitRef = gitInfo.latestRef;
+                        }
+                        await controller.setServiceConfigs.promise(outdatedServices);
+                    }}
+                >
+                    <div>
+                        {bigEmoji("🎯")} <span>Deploy & Restart All ({outdatedServices.length}) Services</span>
+                    </div>
+                    <div className={css.hbox(5)}>
+                        <b>New</b>
+                        <RenderGitRefInfo gitRef={gitInfo.latestRef} />
+                    </div>
+                    {outdatedRefs.map(ref => <div className={css.hbox(5)}>
+                        <b>Current</b>
+                        <RenderGitRefInfo gitRef={ref} />
+                    </div>)}
+                </button>
+            }
+        </>;
+    }
+}
+
+export class UpdateServiceButtons extends qreact.Component<{
+    service: ServiceConfig;
+}> {
+    render() {
+        let controller = MachineServiceController(SocketFunction.browserNodeId());
+        const gitInfo = controller.getGitInfo();
+        let gitRef = this.props.service.parameters.gitRef;
+        let repoUrl = this.props.service.parameters.repoUrl;
+        if (gitInfo?.repoUrl !== repoUrl) return undefined;
+
+        return <>
+            {gitRef !== gitInfo.latestRef && <button
+                className={buttonStyle.hsl(120, 70, 50).alignSelf("stretch")}
+                onClick={async () => {
+                    this.props.service.parameters.gitRef = gitInfo.latestRef;
+                    await controller.setServiceConfigs.promise([this.props.service]);
+                }}
+            >
+                <div>
+                    {bigEmoji("🎯", -4)} <span>Deploy & Restart</span>
+                </div>
+                <div className={css.hbox(5)}>
+                    <b>New</b>
+                    <RenderGitRefInfo gitRef={gitInfo.latestRef} />
+                </div>
+                <div className={css.hbox(5)}>
+                    <b>Current</b>
+                    <RenderGitRefInfo gitRef={gitRef} />
+                </div>
+            </button>}
+        </>;
+    }
+}
+
+export class DeployMachineButtons extends qreact.Component<{
+    machines: MachineInfo[];
+}> {
+    render() {
+        let controller = MachineServiceController(SocketFunction.browserNodeId());
+        const gitInfo = controller.getGitInfo();
+        if (!gitInfo) return undefined;
+
+        let outdatedMachines = this.props.machines.filter(machine => {
+            return machine.repoUrl === gitInfo.repoUrl && machine.gitRef !== gitInfo.latestRef;
+        });
+        let oudatedRefs = outdatedMachines.map(machine => machine.gitRef);
+
+        return <>
+            {outdatedMachines.length > 0 && <div
+                className={buttonStyle.alignSelf("stretch").hsl(120, 70, 50)}
+                onClick={async () => {
+                    let machineController = MachineController(SocketFunction.browserNodeId());
+                    await Promise.all(outdatedMachines.map(machine => {
+                        return machineController.deployMachineFromBrowser.promise({
+                            machineNodeId: machine.applyNodeId,
+                            gitRef: gitInfo.latestRef,
+                        });
+                    }));
+                }}
+            >
+                <div>
+                    {bigEmoji("🎯", -4)} <span>Deploy {outdatedMachines.length} Machines</span>
+                </div>
+                <div className={css.hbox(5)}>
+                    <b>New</b>
+                    <RenderGitRefInfo gitRef={gitInfo.latestRef} />
+                </div>
+                {oudatedRefs.map(ref => <div className={css.hbox(5)}>
+                    <b>Current</b>
+                    <RenderGitRefInfo gitRef={ref} />
+                </div>)}
+            </div>}
+        </>;
+    }
+}

package/src/deployManager/machineApplyMainCode.ts

@@ -4,7 +4,7 @@ import { measureWrap } from "socket-function/src/profiling/measure";
 import { getOwnMachineId } from "../-a-auth/certs";
 import { getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { Querysub } from "../4-querysub/QuerysubController";
-import { MACHINE_RESYNC_INTERVAL, DeployControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos } from "./machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos } from "./machineSchema";
 import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
@@ -18,7 +18,7 @@ import os from "os";
 import fs from "fs";
 import { spawn, ChildProcess } from "child_process";
 import { lazy } from "socket-function/src/caching";
-import { getGitRefLive, setGitRef } from "../4-deploy/git";
+import { getGitRefLive, getGitURLLive, setGitRef } from "../4-deploy/git";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { shutdown } from "../diagnostics/periodic";
 import { onServiceConfigChange } from "./machineController";
@@ -34,6 +34,8 @@ const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
         heartbeat: Date.now(),
         info: {},
         services: {},
+        repoUrl: "",
+        gitRef: "",
     };
 
     machineInfo.info.hostnamectl = await errorToUndefinedSilent(runPromise("hostnamectl")) || "";
@@ -41,6 +43,9 @@ const getLiveMachineInfo = measureWrap(async function getLiveMachineInfo() {
     machineInfo.info.lscpu = await errorToUndefinedSilent(runPromise("lscpu")) || "";
     machineInfo.info.id = await errorToUndefinedSilent(runPromise("id")) || await errorToUndefinedSilent(runPromise("whoami")) || "";
 
+    machineInfo.repoUrl = await getGitURLLive(".");
+    machineInfo.gitRef = await getGitRefLive(".");
+
     return machineInfo;
 });
 
@@ -78,7 +83,7 @@ export async function streamScreenOutput(config: {
         try {
             await serialOnData(data);
         } catch (e: any) {
-            console.log(`Callback for stream output ${screenName} failed. It probably just disconnected, almost certainly not an error: ${e.stack}`);
+            console.log(`Callback for stream output ${screenName} failed. It probably just disconnected, almost certainly not an error: ${e.message}`);
             await stop();
         } finally {
             pendingDataCalls--;
@@ -425,7 +430,7 @@ async function quickIsOutdated() {
                 index: i,
             });
             let screen = screens.find(x => x.screenName === screenName);
-            if (!screen) return true;
+            if (!screen?.isProcessRunning) return true;
 
             let folder = root + screenName + "/";
             await fs.promises.mkdir(folder, { recursive: true });
@@ -433,6 +438,7 @@ async function quickIsOutdated() {
             if (!fs.existsSync(parameterPath)) return true;
             let prevParameters = await fs.promises.readFile(parameterPath, "utf8");
             if (prevParameters !== newParametersString) return true;
+
         }
     }
     return false;

package/src/deployManager/machineController.ts

@@ -8,6 +8,10 @@ import { isNode } from "typesafecss";
 import { streamScreenOutput } from "./machineApplyMainCode";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { getPathStr2 } from "../path";
+import { getGitURLLive, setGitRef } from "../4-deploy/git";
+import os from "os";
+import { runPromise } from "../functional/runCommand";
+import { getSyncedController } from "../library-components/SyncedController";
 
 const POLL_INTERVAL = timeInMinute * 15;
 
@@ -78,27 +82,55 @@ class MachineControllerBase {
     }) {
         let caller = SocketFunction.getCaller();
         forwardedCallbacks.set(config.callbackId, caller.nodeId);
-        await MachineController.nodes[config.nodeId].streamScreenOutput({
+        await MachineController(config.nodeId).streamScreenOutput.promise({
             key: config.key,
             index: config.index,
             callbackId: config.callbackId,
         });
     }
+
+    public async deployMachineFromBrowser(config: {
+        machineNodeId: string;
+        gitRef: string;
+    }) {
+        await MachineController(config.machineNodeId).deployMachine.promise({
+            gitRef: config.gitRef,
+        });
+    }
+
+    public async deployMachine(config: {
+        gitRef: string;
+    }) {
+        let gitFolder = os.homedir() + "/machine-alwaysup";
+        await setGitRef({
+            gitFolder,
+            gitRef: config.gitRef,
+        });
+        await runPromise("bash machine-startup.sh", { cwd: os.homedir() });
+    }
+
 }
 
 let forwardedCallbacks = new Map<string, string>();
 
-const MachineController = SocketFunction.register(
+export const MachineController = getSyncedController(SocketFunction.register(
     "machine-controller-c3157d4a-580c-4e76-9dc9-072dd92e70af",
     () => new MachineControllerBase(),
     () => ({
         streamScreenOutput: {},
         watchOtherScreenOutput: {},
+        deployMachineFromBrowser: {},
+        deployMachine: {},
     }),
     () => ({
         hooks: [assertIsManagementUser],
     }),
-);
+), {
+    writes: {
+        deployMachineFromBrowser: ["MachineInfo"],
+        deployMachine: ["MachineInfo"],
+    }
+});
 
 let callbacks = new Map<string, (data: string) => Promise<void>>();
 export async function watchScreenOutput(config: {
@@ -110,7 +142,7 @@ export async function watchScreenOutput(config: {
 }) {
     let callbackId = config.callbackId;
     callbacks.set(callbackId, config.onData);
-    await MachineController.nodes[SocketFunction.browserNodeId()].watchOtherScreenOutput({
+    await MachineController(SocketFunction.browserNodeId()).watchOtherScreenOutput.promise({
         nodeId: config.nodeId,
         key: config.key,
         index: config.index,

package/src/deployManager/machineSchema.ts

@@ -13,12 +13,42 @@ import { getOwnMachineId } from "../-a-auth/certs";
 import { runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { lazy } from "socket-function/src/caching";
 import { errorToUndefinedSilent } from "../errors";
-import { getGitRefLive, getGitURLLive } from "../4-deploy/git";
+import { commitAndPush, getGitRefInfo, getGitRefLive, getGitURLLive, getGitUncommitted, getLatestRefOnUpstreamBranch, setGitRef } from "../4-deploy/git";
 import { OnServiceChange } from "./machineController";
+import path from "path";
+import fs from "fs";
+import { runPromise } from "../functional/runCommand";
 
 export const SERVICE_FOLDER = "machine-services/";
 export const MACHINE_RESYNC_INTERVAL = timeInMinute * 15;
 
+export type MachineInfo = {
+    machineId: string;
+
+    // Used to tell the apply tool to update it's configs now
+    applyNodeId: string;
+
+    heartbeat: number;
+    /* 
+        // TODO: ShowMore on each of the infos, so large ones are fine.
+        hostnamectl (fallback to hostname)
+        getExternalIP()
+        lscpu
+        id (fallback to whoami)
+    */
+    info: Record<string, string>;
+
+    repoUrl: string;
+    gitRef: string;
+
+    services: Record<string, {
+        lastLaunchedTime: number;
+        errorFromLastRun: string;
+        // Only times launched for the current applyNodeId, but... still very useful.
+        totalTimesLaunched: number;
+    }>;
+};
+
 
 export type ServiceConfig = {
     /** Just a random id to manage the service */
@@ -50,38 +80,13 @@ export type ServiceConfig = {
         lastUpdatedTime: number;
     };
 };
-
-export type MachineInfo = {
-    machineId: string;
-
-    // Used to tell the apply tool to update it's configs now
-    applyNodeId: string;
-
-    heartbeat: number;
-    /* 
-        // TODO: ShowMore on each of the infos, so large ones are fine.
-        hostnamectl (fallback to hostname)
-        getExternalIP()
-        lscpu
-        id (fallback to whoami)
-    */
-    info: Record<string, string>;
-
-    services: Record<string, {
-        lastLaunchedTime: number;
-        errorFromLastRun: string;
-        // Only times launched for the current applyNodeId, but... still very useful.
-        totalTimesLaunched: number;
-    }>;
-};
-
-export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
 export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machines/machine-heartbeats/", getArchivesBackblaze(getDomain())));
+export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
 
 
-export class DeployControllerBase {
+export class MachineServiceControllerBase {
 
-    public async getMachineList(): Promise<string[]> {
+    public async getMachineList() {
         return [...new Set([
             ...(await machineInfos.keys()),
         ])];
@@ -151,17 +156,28 @@ export class DeployControllerBase {
         await this.notifyMachines(config.machineIds, []);
     }
 
-    public async setServiceConfig(serviceId: string, config: ServiceConfig) {
-        config.info.lastUpdatedTime = Date.now();
-        let serviceConfig = await serviceConfigs.get(serviceId);
+    public async setServiceConfigs(configs: ServiceConfig[]) {
+        let newMachines = new Set<string>();
+        let oldMachines = new Set<string>();
+        for (let config of configs) {
+            let serviceId = config.serviceId;
+            config.info.lastUpdatedTime = Date.now();
+            let serviceConfig = await serviceConfigs.get(serviceId);
 
-        if (!serviceConfig) {
-            throw new Error(`Service ${serviceId} does not exist`);
-        }
+            if (!serviceConfig) {
+                throw new Error(`Service ${serviceId} does not exist`);
+            }
 
-        // Set the new service config
-        await serviceConfigs.set(serviceId, config);
-        await this.notifyMachines(config.machineIds, serviceConfig.machineIds);
+            // Set the new service config
+            await serviceConfigs.set(serviceId, config);
+            for (let machineId of config.machineIds) {
+                newMachines.add(machineId);
+            }
+            for (let machineId of serviceConfig.machineIds) {
+                oldMachines.add(machineId);
+            }
+        }
+        await this.notifyMachines([...newMachines], [...oldMachines]);
     }
 
     public async deleteServiceConfig(serviceId: string) {
@@ -180,19 +196,86 @@ export class DeployControllerBase {
     }
     public async getGitInfo() {
         let repoUrl = await getGitURLLive();
-        let gitRef = await getGitRefLive();
+        let uncommitted = await getGitUncommitted(".");
+        let latestRef = await getLatestRefOnUpstreamBranch(".");
+        let querysubFolder = path.resolve("../querysub");
+        let querysubRef = "";
+        let querysubUncommitted: string[] = [];
+        if (fs.existsSync(querysubFolder)) {
+            querysubRef = await getGitRefLive(querysubFolder);
+            querysubUncommitted = await getGitUncommitted(querysubFolder);
+        }
         return {
             repoUrl,
-            gitRef,
+            latestRef,
+            uncommitted,
+            querysubRef,
+            querysubUncommitted,
         };
     }
+
+    public async getGitRefInfo(config: {
+        ref: string;
+        useQuerysub?: boolean;
+    }) {
+        if (!config.ref) return undefined;
+        try {
+            let gitDir = ".";
+            if (config.useQuerysub) {
+                gitDir = path.resolve("../querysub");
+            }
+            return await getGitRefInfo({
+                gitDir,
+                ref: config.ref,
+            });
+        } catch (error) {
+            console.error(error);
+            return undefined;
+        }
+    }
+
+    public async commitPushService(commitMessage: string) {
+        await commitAndPush({
+            gitDir: ".",
+            message: commitMessage,
+        });
+    }
+    public async commitPushAndPublishQuerysub(commitMessage: string) {
+        let querysubFolder = path.resolve("../querysub");
+        if (!fs.existsSync(querysubFolder)) {
+            throw new Error(`Querysub folder does not exist at ${querysubFolder}`);
+        }
+
+        // Increment minor version and publish to npm with yarn
+        await runPromise("yarn version --minor --no-git-tag-version", { cwd: querysubFolder });
+        await runPromise("yarn publish --non-interactive", { cwd: querysubFolder });
+
+        // Update package.json reference in our actual repo (version only, no install)
+        let querysubPackageJson = JSON.parse(fs.readFileSync(path.join(querysubFolder, "package.json"), "utf8"));
+        let newVersion = querysubPackageJson.version;
+
+        let currentPackageJsonPath = path.join(".", "package.json");
+        let currentPackageJson = JSON.parse(fs.readFileSync(currentPackageJsonPath, "utf8"));
+
+        if (!currentPackageJson.dependencies?.querysub) {
+            throw new Error(`Querysub is not a dependency in package.json?`);
+        }
+        currentPackageJson.dependencies.querysub = `^${newVersion}`;
+
+        fs.writeFileSync(currentPackageJsonPath, JSON.stringify(currentPackageJson, null, 2));
+
+        await commitAndPush({
+            gitDir: querysubFolder,
+            message: commitMessage,
+        });
+    }
 }
 
 
-export const DeployController = getSyncedController(
+export const MachineServiceController = getSyncedController(
     SocketFunction.register(
-        "deploy-eda94f05-5e4d-4f5a-b1c1-98613fba60b8",
-        () => new DeployControllerBase(),
+        "machine-controller-eda94f05-5e4d-4f5a-b1c1-98613fba60b8",
+        () => new MachineServiceControllerBase(),
         () => ({
             getMachineList: {},
             deleteMachineIds: {},
@@ -202,10 +285,13 @@ export const DeployController = getSyncedController(
             getServiceList: {},
             getServiceConfig: {},
             addServiceConfig: {},
-            setServiceConfig: {},
+            setServiceConfigs: {},
             getServiceConfigType: {},
             deleteServiceConfig: {},
             getGitInfo: {},
+            getGitRefInfo: {},
+            commitPushService: {},
+            commitPushAndPublishQuerysub: {},
         }),
         () => ({
             hooks: [assertIsManagementUser],
@@ -217,14 +303,17 @@ export const DeployController = getSyncedController(
             addMachineInfo: ["MachineInfo", "MachineInfoList"],
             setMachineInfo: ["MachineInfo"],
             addServiceConfig: ["ServiceConfig", "ServiceConfigList"],
-            setServiceConfig: ["ServiceConfig"],
+            setServiceConfigs: ["ServiceConfig"],
             deleteServiceConfig: ["ServiceConfig", "ServiceConfigList"],
+            commitPushService: ["gitInfo", "ServiceConfig"],
+            commitPushAndPublishQuerysub: ["gitInfo", "ServiceConfig"],
         },
         reads: {
             getMachineList: ["MachineInfoList"],
             getMachineInfo: ["MachineInfo"],
             getServiceList: ["ServiceConfigList"],
             getServiceConfig: ["ServiceConfig"],
+            getGitInfo: ["gitInfo"],
         }
     }
 );