querysub 0.290.0 → 0.292.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.290.0",
+  "version": "0.292.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/4-deploy/edgeNodes.ts

@@ -6,11 +6,11 @@ import { getAllNodeIds, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery"
 import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
 import { nestArchives } from "../-a-archives/archives";
 import { SocketFunction } from "socket-function/SocketFunction";
-import { runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { compare, compareArray, isNodeTrue, sort, timeInMinute } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
-import { shutdown } from "../diagnostics/periodic";
+import { registerShutdownHandler, shutdown } from "../diagnostics/periodic";
 import { hostArchives } from "../-b-authorities/cdnAuthority";
 import { getModuleFromConfig } from "../3-path-functions/pathFunctionLoader";
 import path from "path";
@@ -26,6 +26,7 @@ import { onEdgeNodesChanged } from "./edgeBootstrap";
 import { startEdgeNotifier } from "./edgeClientWatcher";
 import { getGitRefLive, getGitURLLive } from "./git";
 import { DeployProgress } from "./deployFunctions";
+import { watchOnRollingUpdate } from "../deployManager/machineController";
 
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
@@ -41,13 +42,17 @@ const getEdgeNodeConfig = cacheLimited(10000, async (fileName: string): Promise<
     return obj;
 });
 let nextNodeNum = 1;
+let registeredNodePaths = new Set<string>();
 function getNextNodePath() {
-    return "node" + (nextNodeNum++) + "_" + getOwnNodeId();
+    let path = "node" + (nextNodeNum++) + "_" + getOwnNodeId();
+    registeredNodePaths.add(path);
+    return path;
 }
 function getNodeIdFromPath(path: string) {
     return path.split("_").slice(1).join("_");
 }
 
+let edgeShutdown = false;
 
 export type EdgeNodesIndex = {
     edgeNodes: EdgeNodeConfig[];
@@ -85,6 +90,29 @@ export async function registerEdgeNode(config: {
     registeredEdgeNode = true;
     let host = config.host;
 
+
+    registerShutdownHandler(async () => {
+        edgeShutdown = true;
+        console.log(magenta(`Removing node from edge node list due to shutdown`));
+        await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
+            await edgeNodeStorage.del(nodePath);
+        }));
+        await updateEdgeNodesFile();
+    });
+    watchOnRollingUpdate({
+        callback: async (time) => {
+            let duration = time - Date.now();
+            // Wait a bit to remove ourself from the edge node list
+            void delay(duration * 0.2).finally(async () => {
+                console.log(magenta(`Removing node from edge node list due to rolling update at ${new Date().toLocaleString()}`));
+                await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
+                    await edgeNodeStorage.del(nodePath);
+                }));
+                await updateEdgeNodesFile();
+            });
+        },
+    });
+
     await waitForFirstTimeSync();
 
     if (!isLocal() && !noSyncing()) {
@@ -193,6 +221,7 @@ const startUpdateLoop = lazy(async () => {
 });
 
 async function updateLoop() {
+    if (edgeShutdown) return;
     await runEdgeNodesAliveCheck();
     await updateEdgeNodesFile();
 }

package/src/deployManager/machineApplyMainCode.ts

@@ -306,11 +306,11 @@ const getScreenState = measureWrap(async function getScreenState(populateIsProce
     return screenList;
 });
 async function removeOldNodeId(screenName: string) {
-    let screenNameFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
-    if (fs.existsSync(screenNameFile)) {
-        let nodeId = await fs.promises.readFile(screenNameFile, "utf8");
-        console.log(green(`Removing node if for dead service on ${screenNameFile}, node id ${nodeId}`));
-        await fs.promises.unlink(screenNameFile);
+    let nodeIdFile = os.homedir() + "/" + SERVICE_FOLDER + screenName + "/" + SERVICE_NODE_FILE_NAME;
+    if (fs.existsSync(nodeIdFile)) {
+        let nodeId = await fs.promises.readFile(nodeIdFile, "utf8");
+        console.log(green(`Removing node if for dead service on ${nodeIdFile}, node id ${nodeId}`));
+        await fs.promises.unlink(nodeIdFile);
         await forceRemoveNode(nodeId);
     }
 }
@@ -346,10 +346,16 @@ const runScreenCommand = measureWrap(async function runScreenCommand(config: {
             let rollingFinalTime = Date.now() + config.rollingWindow;
             if (fs.existsSync(nodeIdPath)) {
                 let nodeId = await fs.promises.readFile(nodeIdPath, "utf8");
-                await triggerRollingUpdate({
-                    nodeId,
-                    time: rollingFinalTime,
-                });
+                // REMOVE the nodeId file, so we the node isn't terminated!
+                await fs.promises.unlink(nodeIdPath);
+                try {
+                    await triggerRollingUpdate({
+                        nodeId,
+                        time: rollingFinalTime,
+                    });
+                } catch (e: any) {
+                    console.warn(`Error triggering rolling update notification to ${nodeId} for ${screenName}: ${e.stack}`);
+                }
             }
             let rollingScreenName = screenName + "-rolling" + SCREEN_SUFFIX;
             console.log(green(`Renaming screen ${screenName} to ${rollingScreenName} for rolling interval ${config.rollingWindow} at ${new Date().toLocaleString()}`));

package/src/deployManager/machineController.ts

@@ -44,6 +44,8 @@ export function watchOnRollingUpdate(config: {
 }) {
     rollingUpdateWatchers.add(config.callback);
 }
+// TODO: PathValueServer should use this to remove itself from the node pool, and then after 50% of the time until time tell it's clients to find a new server to get values from
+// TODO: FunctionRunner needs to reshard when it sees this (somehow?)
 export async function triggerRollingUpdate(config: {
     nodeId: string;
     time: number;

package/src/deployManager/spec.txt

@@ -1,8 +1,10 @@
+Hmm... our cyoa servers are dying very fast... why?
+
 9) Rolling service updates
     - Add rollingWindow to the definition
     - Set for public facing services (but not for scripts)
         - PathValueServer - 10 minutes
-        - FunctionRunner - 10 minutes
+        - FunctionRunner - CANNOT use rolling
         - HTTP - 4 hours
         - Boostrapper - NO rolling
         - gc/join - NO rolling