querysub 0.610.0 → 0.612.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.610.0",
+  "version": "0.612.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -75,7 +75,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "sliftutils": "^1.7.84",
+    "sliftutils": "^1.7.85",
     "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -24,7 +24,7 @@ import { logDisk } from "../diagnostics/logs/diskLogger";
 import { MaybePromise } from "socket-function/src/types";
 import { getPathStr2 } from "../path";
 
-let HEARTBEAT_INTERVAL = timeInMinute * 5;
+let HEARTBEAT_INTERVAL = timeInMinute * 1;
 // Interval which we check other heartbeats
 let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
 // If the heartbeat is older than thing, it fails the dead check
@@ -34,7 +34,7 @@ let DEAD_THRESHOLD = HEARTBEAT_INTERVAL * 2;
 let DEAD_CHECK_COUNT = 4;
 // If we find we are unable to write our heartbeat, we have to kill our own process. Otherwise it may
 //  be too out of sync, and might commit unverified data to the disk.
-let SUICIDE_HEARTBEAT_THRESHOLD = HEARTBEAT_INTERVAL * 4;
+let SUICIDE_HEARTBEAT_THRESHOLD = Math.max(timeInMinute * 10, HEARTBEAT_INTERVAL * 4);
 
 let CLIENTSIDE_POLL_RATE = timeInMinute * 30;
 
@@ -56,6 +56,7 @@ let DEAD_NODE_POLL_COOLDOWN = timeInMinute * 5;
 
 let shutdown = false;
 
+// NOTE: It's not really required to use buffered archives for this. Previously, our heartbeat interval was five minutes and we could raise it again to five minutes. However, keeping it fast means that if our buffered archives has bugs, we'll notice it pretty quickly because if files disappear, for example, if our node discovery file disappears, processes will terminate themselves, which makes it really obvious that there's a problem, and files should never disappear. So it's kind of a canary to say, hey, there's something wrong with our storage, we need to fix it. Even if one node goes away, its writes should have already been propagated. 
 const archives = lazy(() => getArchives2Buffered("nodes/"));
 
 let logging = true;

package/src/4-deploy/edgeNodes.ts

@@ -26,6 +26,8 @@ import { getGitRefLive, getGitURLLive } from "./git";
 import { DeployProgress } from "./deployFunctions";
 import { PromiseObj } from "../promise";
 import type { FunctionRunnerIndex } from "../4-querysub/FunctionRunnerTracking";
+import { isDefined } from "sliftutils/misc/types";
+
 const UPDATE_POLL_INTERVAL = timeInMinute * 15;
 const HEARTBEAT_INTERVAL = timeInMinute;
 const EDGE_NODE_DELETE_THRESHOLD = timeInMinute * 15;
@@ -34,11 +36,8 @@ const EDGE_NODE_READ_PARALLEL = 64;
 export const edgeNodeStorage = isNodeTrue() && getArchives2BufferedPublic("edgenodes2/");
 export const edgeNodeIndexFile = "edge-nodes-index.json";
 
-let registeredNodePaths = new Set<string>();
 function getNextNodePath() {
-    let path = "node0_" + getOwnNodeId();
-    registeredNodePaths.add(path);
-    return path;
+    return "node0_" + getOwnNodeId();
 }
 
 let edgeShutdown = false;
@@ -94,6 +93,7 @@ declare global {
     var EDGE_NODE_STATS: EdgeNodeStats | undefined;
 }
 let registeredEdgeNode: { host: string; entryPaths: string[] } | boolean | undefined;
+let lastEdgeNodeConfig: EdgeNodeConfig | undefined;
 export async function registerEdgeNode(config: {
     host: string;
     entryPaths: string[];
@@ -114,9 +114,7 @@ export async function registerEdgeNode(config: {
     registerShutdownHandler(async () => {
         edgeShutdown = true;
         console.log(magenta(`Removing node from edge node list due to shutdown`));
-        await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
-            await edgeNodeStorage.del(nodePath);
-        }));
+        await edgeNodeStorage.del(getNextNodePath());
         await updateEdgeNodesFile();
     });
     const { watchScheduledShutdown } = await import("../-g-core-values/scheduledShutdown");
@@ -128,9 +126,7 @@ export async function registerEdgeNode(config: {
             // Wait a bit to fully remove ourself from the edge node list
             await delay(Math.max(0, duration * 0.2));
             console.log(magenta(`Removing node from edge node list due to scheduled shutdown at ${new Date(time).toLocaleString()}`));
-            await Promise.all(Array.from(registeredNodePaths).map(async nodePath => {
-                await edgeNodeStorage.del(nodePath);
-            }));
+            await edgeNodeStorage.del(getNextNodePath());
             await updateEdgeNodesFile();
         })();
     });
@@ -177,7 +173,7 @@ export async function registerEdgeNode(config: {
             public: isPublic(),
             functionRunnerIndex: await getInitialFunctionRunnerIndex(),
         };
-
+        lastEdgeNodeConfig = edgeNodeConfig;
         await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
     }
     registeredEdgeNode = {
@@ -229,6 +225,7 @@ const loadEntryPointsByHash = runInSerial(async function loadEntryPointsByHash(c
         public: isPublic(),
         functionRunnerIndex: await getInitialFunctionRunnerIndex(),
     };
+    lastEdgeNodeConfig = edgeNodeConfig;
 
     await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(edgeNodeConfig)));
     console.log(magenta(`Registered edge node`), edgeNodeConfig);
@@ -245,14 +242,9 @@ async function getInitialFunctionRunnerIndex(): Promise<FunctionRunnerIndex | un
 }
 
 async function publishOwnScheduledShutdown(time: number) {
-    for (let nodePath of Array.from(registeredNodePaths)) {
-        let buffer = await edgeNodeStorage.get(nodePath);
-        if (!buffer) continue;
-        let config: EdgeNodeConfig = JSON.parse(buffer.toString());
-        if (config.scheduledShutdownTime === time) continue;
-        config.scheduledShutdownTime = time;
-        await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(config)));
-    }
+    if (!lastEdgeNodeConfig) return;
+    lastEdgeNodeConfig.scheduledShutdownTime = time;
+    await edgeNodeStorage.set(getNextNodePath(), Buffer.from(JSON.stringify(lastEdgeNodeConfig)));
     await updateEdgeNodesFile();
 }
 
@@ -271,12 +263,10 @@ const startUpdateLoop = lazy(async () => {
 
 async function heartbeatLoop() {
     if (edgeShutdown) return;
+    if (!lastEdgeNodeConfig) return;
     let nodePath = getNextNodePath();
-    let buffer = await edgeNodeStorage.get(nodePath);
-    if (!buffer) return;
-    let config: EdgeNodeConfig = JSON.parse(buffer.toString());
-    config.heartbeatTime = Date.now();
-    await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(config)));
+    lastEdgeNodeConfig.heartbeatTime = Date.now();
+    await edgeNodeStorage.set(nodePath, Buffer.from(JSON.stringify(lastEdgeNodeConfig)));
     // Push the fresh heartbeat into the index so clients see it (they read only the index, not the per-node files).
     await updateEdgeNodesFile();
 }
@@ -287,7 +277,6 @@ async function updateLoop() {
 }
 
 async function updateEdgeNodesFile() {
-    // Snapshot the time BEFORE reading anything below. Staleness is judged against when we STARTED, so if the reads block we can never mass-evict live nodes just because wall-clock advanced while we were stuck.
     let now = Date.now();
 
     let prevEdgeNodeFile = await edgeNodeStorage.get(edgeNodeIndexFile);
@@ -318,7 +307,6 @@ async function updateEdgeNodesFile() {
     let edgeNodeFiles = await edgeNodeStorage.find("node", { type: "files" });
     // Reads are independent, so run them in parallel (bounded): with many files a serial pass would take files × read-latency, which could be minutes.
     let readEdgeNodeFile = runInParallel({ parallelCount: EDGE_NODE_READ_PARALLEL }, async (nodeFile: string) => {
-        // Read directly, not through a cache: heartbeatTime mutates every minute, so a cached config would report a stale heartbeat and wrongly evict a live peer.
         let buffer = await edgeNodeStorage.get(nodeFile);
         if (!buffer) return undefined;
         let config: EdgeNodeConfig = JSON.parse(buffer.toString());
@@ -330,19 +318,9 @@ async function updateEdgeNodesFile() {
         }
         return config;
     });
-    let configs = await Promise.all(edgeNodeFiles.map(readEdgeNodeFile));
-
-    let bestByNodeId = new Map<string, EdgeNodeConfig>();
-    for (let config of configs) {
-        if (!config) continue;
-        // One entry per nodeId (re-registrations can leave more than one), keeping the freshest heartbeat.
-        let existing = bestByNodeId.get(config.nodeId);
-        if (!existing || (config.heartbeatTime || config.bootTime) > (existing.heartbeatTime || existing.bootTime)) {
-            bestByNodeId.set(config.nodeId, config);
-        }
-    }
+    let configs = (await Promise.all(edgeNodeFiles.map(readEdgeNodeFile))).filter(isDefined);
 
-    let newEdgeNodes = sort(Array.from(bestByNodeId.values()), x => -(x.heartbeatTime || x.bootTime));
+    let newEdgeNodes = sort(configs, x => -(x.heartbeatTime || x.bootTime));
     let newEdgeNodeIndex: EdgeNodesIndex = {
         edgeNodes: newEdgeNodes,
         liveHash,