querysub 0.256.0 → 0.258.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.256.0",
+  "version": "0.258.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-b-authorities/dnsAuthority.ts

@@ -105,7 +105,10 @@ export async function setRecord(type: string, key: string, value: string, proxie
     });
     // NOTE: Apparently... even if the record didn't exist, we still have to wait...
     console.log(`Waiting ${ttl} seconds for DNS to propagate...`);
-    await delay(ttl * 1000);
+    for (let ttlLeft = ttl; ttlLeft > 0; ttlLeft--) {
+        await delay(1000);
+        console.log(`${ttlLeft} seconds left...`);
+    }
     console.log(`Done waiting for DNS to update.`);
 
 }
@@ -127,6 +130,9 @@ export async function addRecord(type: string, key: string, value: string, proxie
         proxied: proxied === "proxied",
     });
     console.log(`Waiting ${ttl} seconds for DNS to propagate...`);
-    await delay(ttl * 1000);
+    for (let ttlLeft = ttl; ttlLeft > 0; ttlLeft--) {
+        await delay(1000);
+        console.log(`${ttlLeft} seconds left...`);
+    }
     console.log(`Done waiting for DNS to update.`);
 }

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -472,6 +472,11 @@ if (isServer()) {
     }
 }
 
+export async function forceRemoveNode(nodeId: string) {
+    await archives().del(nodeId);
+    void tellEveryoneNodesChanges();
+}
+
 
 /** Called on shutdown, to completely remove this node from discovery. */
 export async function nodeDiscoveryShutdown() {

package/src/4-querysub/Querysub.ts

@@ -48,6 +48,8 @@ import yargs, { check } from "yargs";
 import { parseArgsFactory } from "../misc/rawParams";
 
 import * as typesafecss from "typesafecss";
+
+
 typesafecss.setMeasureBlock(measureBlock);
 typesafecss.setDelayFnc(async fnc => {
     await (clientWatcher.waitForBeforeTriggerFinished() || Promise.resolve());
@@ -1254,6 +1256,13 @@ setImmediate(async () => {
     await import("../5-diagnostics/nodeMetadata");
 });
 
+setImmediate(async () => {
+    if (isNode()) {
+        const { doRegisterNodeForMachineCleanup } = await import("../deployManager/machineSchema");
+        doRegisterNodeForMachineCleanup();
+    }
+});
+
 registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
 registerGetCompressDisk(() => Querysub.COMPRESS_DISK);
 
@@ -1269,4 +1278,5 @@ import { css } from "../4-dom/css";
 import { getCountPerPaint } from "../functional/onNextPaint";
 import { addEpsilons } from "../bits";
 import { blue } from "socket-function/src/formatting/logColors";
+import { MachineController } from "../deployManager/machineController";
 

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -18,6 +18,7 @@ import { ScrollOnMount } from "../../library-components/ScrollOnMount";
 import { StickyBottomScroll } from "../../library-components/StickyBottomScroll";
 import { PrimitiveDisplay } from "../../diagnostics/logs/ObjectDisplay";
 import { parseAnsiColors, rgbToHsl } from "../../diagnostics/logs/ansiFormat";
+import { RenderGitRefInfo, UpdateServiceButtons, bigEmoji, buttonStyle } from "./deployButtons";
 
 // Type declarations for Monaco editor
 declare global {
@@ -199,6 +200,8 @@ type ServiceConfig = ${serviceConfigType}
         const selectedServiceId = selectedServiceIdParam.value;
         if (!selectedServiceId) return <div>No service selected</div>;
 
+        const gitInfo = MachineServiceController(SocketFunction.browserNodeId()).getGitInfo();
+
         let controller = MachineServiceController(SocketFunction.browserNodeId());
         let originalConfig = controller.getServiceConfig(selectedServiceId);
         let serviceConfigType = controller.getServiceConfigType();
@@ -511,6 +514,25 @@ type ServiceConfig = ${serviceConfigType}
                     </button>
                 </>}
 
+                {gitInfo?.repoUrl === config.parameters.repoUrl && gitInfo.latestRef !== config.parameters.gitRef &&
+                    <button
+                        className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
+                        onClick={async () => {
+                            configT.parameters.gitRef = gitInfo.latestRef;
+                            this.updateUnsavedChanges(configT);
+                        }}
+                    >
+                        <div className={css.hbox(5)}>
+                            <b>New</b>
+                            <RenderGitRefInfo gitRef={gitInfo.latestRef} />
+                        </div>
+                        <div className={css.hbox(5)}>
+                            <b>Current</b>
+                            <RenderGitRefInfo gitRef={config.parameters.gitRef} />
+                        </div>
+                    </button>
+                }
+
                 {this.state.isSaving && <div className={css.pad2(12, 8).bord2(0, 0, 20) + " rainbow-flash"}>
                     Deploying changes
                 </div>}

package/src/deployManager/components/deployButtons.tsx

@@ -12,7 +12,7 @@ import { unique } from "../../misc";
 
 module.hotreload = true;
 
-function bigEmoji(emoji: string, topOffset = 0) {
+export function bigEmoji(emoji: string, topOffset = 0) {
     return <span className={css.fontSize(26).marginTop(topOffset)}>{emoji}</span>;
 }
 
@@ -33,7 +33,7 @@ export class RenderGitRefInfo extends qreact.Component<{
     }
 }
 
-const buttonStyle = css.pad2(12, 8).button.bord2(0, 0, 20).fontWeight("bold").vbox(0).alignItems("center");
+export const buttonStyle = css.pad2(12, 8).button.bord2(0, 0, 20).fontWeight("bold").vbox(0).alignItems("center");
 
 export class UpdateButtons extends qreact.Component<{
     services: ServiceConfig[];
@@ -50,7 +50,7 @@ export class UpdateButtons extends qreact.Component<{
 
         return <>
             {!!gitInfo?.querysubUncommitted.length && <button
-                className={buttonStyle.hsl(260, 60, 55)}
+                className={buttonStyle.hsl(260, 60, 90)}
                 title={gitInfo?.querysubUncommitted.join("\n")}
                 onClick={() => {
                     showFullscreenModal({
@@ -61,10 +61,14 @@ export class UpdateButtons extends qreact.Component<{
                                 textarea
                                 fillWidth
                                 focusOnMount
-                                onChangeValue={async value => {
+                                noEnterKeyCommit
+                                onKeyDown={async e => {
+                                    let value = e.currentTarget.value;
                                     if (!value) return;
-                                    await controller.commitPushAndPublishQuerysub.promise(value);
-                                    closeAllModals();
+                                    if (e.key === "Enter" && e.shiftKey) {
+                                        await controller.commitPushAndPublishQuerysub.promise(value);
+                                        closeAllModals();
+                                    }
                                 }}
                             />
                             <div className={css.vbox(5)}>
@@ -88,7 +92,7 @@ export class UpdateButtons extends qreact.Component<{
                 </div>
             </button>}
             {!!gitInfo?.uncommitted.length && <button
-                className={buttonStyle.hsl(210, 40, 50)}
+                className={buttonStyle.hsl(210, 40, 90)}
                 title={gitInfo?.uncommitted.join("\n")}
                 onClick={() => {
                     showFullscreenModal({
@@ -99,10 +103,14 @@ export class UpdateButtons extends qreact.Component<{
                                 textarea
                                 fillWidth
                                 focusOnMount
-                                onChangeValue={async value => {
+                                noEnterKeyCommit
+                                onKeyDown={async e => {
+                                    let value = e.currentTarget.value;
                                     if (!value) return;
-                                    await controller.commitPushService.promise(value);
-                                    closeAllModals();
+                                    if (e.key === "Enter" && e.shiftKey) {
+                                        await controller.commitPushService.promise(value);
+                                        closeAllModals();
+                                    }
                                 }}
                             />
                             <div className={css.vbox(5)}>
@@ -121,7 +129,7 @@ export class UpdateButtons extends qreact.Component<{
 
             {outdatedServices.length > 0 && gitInfo &&
                 <button
-                    className={buttonStyle.hsl(120, 70, 50)}
+                    className={buttonStyle.hsl(120, 70, 90)}
                     onClick={async () => {
                         for (let service of outdatedServices) {
                             service.parameters.gitRef = gitInfo.latestRef;
@@ -158,14 +166,14 @@ export class UpdateServiceButtons extends qreact.Component<{
 
         return <>
             {gitRef !== gitInfo.latestRef && <button
-                className={buttonStyle.hsl(120, 70, 50).alignSelf("stretch")}
+                className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
                 onClick={async () => {
                     this.props.service.parameters.gitRef = gitInfo.latestRef;
                     await controller.setServiceConfigs.promise([this.props.service]);
                 }}
             >
                 <div>
-                    {bigEmoji("🎯", -4)} <span>Deploy & Restart</span>
+                    {bigEmoji("🎯", -4)} <span>Update to New & Restart</span>
                 </div>
                 <div className={css.hbox(5)}>
                     <b>New</b>

package/src/deployManager/machineApplyMainCode.ts

@@ -2,9 +2,9 @@ import "../inject";
 
 import { measureWrap } from "socket-function/src/profiling/measure";
 import { getOwnMachineId } from "../-a-auth/certs";
-import { getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
+import { forceRemoveNode, getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
 import { Querysub } from "../4-querysub/QuerysubController";
-import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos } from "./machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME } from "./machineSchema";
 import { runPromise } from "../functional/runCommand";
 import { getExternalIP } from "socket-function/src/networking";
 import { errorToUndefined, errorToUndefinedSilent } from "../errors";
@@ -305,10 +305,17 @@ const getScreenState = measureWrap(async function getScreenState(populateIsProce
 
     return screenList;
 });
+async function removeOldNodeId(screenName: string) {
+    let screenNameFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
+    if (fs.existsSync(screenNameFile)) {
+        let nodeId = await fs.promises.readFile(screenNameFile, "utf8");
+        await fs.promises.unlink(screenNameFile);
+        await forceRemoveNode(nodeId);
+    }
+}
 const runScreenCommand = measureWrap(async function runScreenCommand(config: {
     screenName: string;
     command: string;
-    folder: string;
 }) {
     let screenName = config.screenName;
     // C:/cygwin64/bin/tmux has-session -t myproj 2>/dev/null || C:/cygwin64/bin/tmux new-session -d -s myproj
@@ -321,6 +328,7 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'C-c' Enter`);
     await delay(1000);
 
+
     let screens = await getScreenState();
     let screen = screens.find(x => x.screenName === screenName);
     let pid = screen?.pid;
@@ -343,23 +351,24 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
                 await runScreenCommand({
                     screenName,
                     command: config.command,
-                    folder: config.folder,
                 });
                 return;
             }
         }
     }
-    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${config.folder}' Enter`);
+    await removeOldNodeId(screenName);
+    let folder = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/";
+    await runPromise(`${prefix}tmux send-keys -t ${screenName} 'cd ${folder}git' Enter`);
     let command = `#!/bin/bash
 ${config.command}
 `;
-    await fs.promises.writeFile(config.folder + "../command.sh", command);
+    await fs.promises.writeFile(folder + "command.sh", command);
     await runPromise(`${prefix}tmux send-keys -t ${screenName} 'bash ../command.sh' Enter`);
 
     // Setup pipe-pane as well
 
-    let pipeFile = path.resolve(config.folder + "../pipe.txt");
-    let pipeScript = path.resolve(config.folder + "../pipe.sh");
+    let pipeFile = path.resolve(folder + "pipe.txt");
+    let pipeScript = path.resolve(folder + "pipe.sh");
     await fs.promises.writeFile(pipeScript, `#!/bin/bash
 line_count=0
 while IFS= read -r line; do
@@ -390,6 +399,7 @@ const killScreen = measureWrap(async function killScreen(config: {
 }) {
     let prefix = getTmuxPrefix();
     await runPromise(`${prefix}tmux kill-session -t ${config.screenName}`);
+    await removeOldNodeId(config.screenName);
 });
 const ensureGitSynced = measureWrap(async function ensureGitSynced(config: {
     // Ensures existingFolder is the repo, as in, cwd to it, and git clone ., if needed. Reset the hash, in whatever way we need to, to force it to gitRef (git add --all and stash, etc, etc).
@@ -515,7 +525,6 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
                 await runScreenCommand({
                     screenName,
                     command: config.parameters.command,
-                    folder: gitFolder,
                 });
                 await delay(2000);
                 let newScreens = await getScreenState(false);

package/src/deployManager/machineSchema.ts

@@ -18,9 +18,12 @@ import { OnServiceChange } from "./machineController";
 import path from "path";
 import fs from "fs";
 import { runPromise } from "../functional/runCommand";
+import { isNode } from "typesafecss";
 
-export const SERVICE_FOLDER = "machine-services/";
+const SERVICE_FOLDER_NAME = "machine-services";
+export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
 export const MACHINE_RESYNC_INTERVAL = timeInMinute * 15;
+export const SERVICE_NODE_FILE_NAME = "serviceNodeId.txt";
 
 export type MachineInfo = {
     machineId: string;
@@ -83,6 +86,20 @@ export type ServiceConfig = {
 export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machines/machine-heartbeats/", getArchivesBackblaze(getDomain())));
 export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
 
+export function doRegisterNodeForMachineCleanup() {
+    if (isNode()) {
+        void SocketFunction.mountPromise.finally(() => {
+            void registerNodeForMachineCleanup(SocketFunction.mountedNodeId);
+        });
+    }
+}
+async function registerNodeForMachineCleanup(nodeId: string) {
+    let currentPath = path.resolve(".").replaceAll("\\", "/");
+    let array = currentPath.split("/");
+    if (array.at(-1) === "git" && array.at(-2) === SERVICE_FOLDER_NAME) {
+        await fs.promises.writeFile(array.slice(0, -2).join("/") + "/" + SERVICE_NODE_FILE_NAME, nodeId);
+    }
+}
 
 export class MachineServiceControllerBase {
 
@@ -168,13 +185,15 @@ export class MachineServiceControllerBase {
                 throw new Error(`Service ${serviceId} does not exist`);
             }
 
-            // Set the new service config
             await serviceConfigs.set(serviceId, config);
-            for (let machineId of config.machineIds) {
-                newMachines.add(machineId);
-            }
-            for (let machineId of serviceConfig.machineIds) {
-                oldMachines.add(machineId);
+            // Only notify we were or are deployed. If it's not deployed, this will be ignored anyways.
+            if (config.parameters.deploy || serviceConfig.parameters.deploy) {
+                for (let machineId of config.machineIds) {
+                    newMachines.add(machineId);
+                }
+                for (let machineId of serviceConfig.machineIds) {
+                    oldMachines.add(machineId);
+                }
             }
         }
         await this.notifyMachines([...newMachines], [...oldMachines]);

package/src/deployManager/spec.txt

@@ -1,23 +1,10 @@
-OH, okay, so... we DO need machine update support, because... we will need to update them
+OH! Each service has a different machine id?
+    Hmm... well... it's the WORST thing every. It is a bit annoying...
+    - That explains the slow startup time...
+    
 
 
-7) Quick node removal on process crash OR removal
-    - In the service, check our parent folder to see if we are in a screen (/machine-services/git/), and then write our nodeId to /machine-services/nodeId
-    - If we see a nodeId when we are removing a screen, or killing the service, then delete that nodeId from the nodeId directory (and call tellEveryoneNodesChanges)
-7.1) Verify this by killing a lot of services (the function runner?), by just poking it over and over, verifying the nodes are quickly deleted
-
-
-
-In general, hit our servers hard, to make sure we can launch them, they can talk to each other, etc. Even with entirely new folders (to somewhat emulate another machine), and they can all startup smoothly.
-
-AH, I think I fixed that bug. I think it was an issue with using old hosts?
-6) BUG! On startup, new nodes might take a poll interval to be fully ready? Hmm... we should restart our servers a lot again to make sure they come back up QUICKLY
-    - OH! It might be that the identity takes some time to propagate (the DNS), which causes us to require a second poll to be alive? Maybe... our local machine deleted it, because it couldn't see the DNS? No... none of these are reasonable...
-    - Try changing the key, so it syncs again, so we start fresh with the server
-        - Reduce it to just 1 instance as well?
-6.1) HTTP required a reboot when it was running without PathValueServer, as it didn't clone the right hashes, and then never did, even with PathValueServer was up?
-    - But maybe it was a node polling issue? But we are supposed to send notifications when nodes are launched? Hmm...
-FORTUNATELY, looking at logs is now very simple, as well as updating nodes, so... this shouldn't be too hard to debug
+7.1) Verify quick node removal work by having a repeatedly crashing service, verifying the node exists, and then goes away on crash.
 
 
 8) Fix deploy user notification issue, where the refresh button doesn't work?

package/src/persistentLocalStore.ts

@@ -5,12 +5,17 @@ import fs from "fs";
 import { getStorageDir, getSubFolder } from "./fs";
 import { isClient } from "./config2";
 import { getDomain } from "./config";
+import os from "os";
 
 export function createKeyStore<T>(key: string): {
     value: T | null;
 } {
     if (isNode()) {
-        let path = getStorageDir() + key + ".json";
+        let keyStoreDir = os.homedir() + "/querysub/";
+        if (!fs.existsSync(keyStoreDir)) {
+            fs.mkdirSync(keyStoreDir);
+        }
+        let path = keyStoreDir + key + ".json";
         return {
             get value() {
                 let contents: string | undefined = undefined;