querysub 0.704.0 → 0.706.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.704.0",
+  "version": "0.706.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -35,6 +35,7 @@ let DEAD_CHECK_COUNT = 4;
 // If we find we are unable to write our heartbeat, we have to kill our own process. Otherwise it may
 //  be too out of sync, and might commit unverified data to the disk.
 let SUICIDE_HEARTBEAT_THRESHOLD = Math.max(timeInMinute * 10, HEARTBEAT_INTERVAL * 4);
+let SELF_HEARTBEAT_CHECK_STARTUP_DELAY = HEARTBEAT_INTERVAL * 5;
 
 let CLIENTSIDE_POLL_RATE = timeInMinute * 30;
 
@@ -492,13 +493,30 @@ async function runMemoryAuditLoop() {
     runInfinitePoll(API_AUDIT_RATE, fastMemorySync);
 }
 
+let firstHeartbeatWriteTime: number | undefined;
+
 async function writeHeartbeat() {
     if (shutdown) return;
-    let now = Date.now();
     let nodeId = getMountNodeId();
-    console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
     if (!nodeId) return;
+    if (firstHeartbeatWriteTime !== undefined && Date.now() - firstHeartbeatWriteTime >= SELF_HEARTBEAT_CHECK_STARTUP_DELAY && !isNoNetwork()) {
+        let lastTime = Number((await archives().get(nodeId))?.toString());
+        let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
+        if (!lastTime) {
+            console.error(red(`Self node was removed due to heartbeat file not existing (${nodeId}). Terminating self process, as it likely has very stale data.`));
+            process.exit();
+        }
+        if (lastTime < suicideThreshold) {
+            console.error(red(`Self node has very old heartbeat. Terminating self process, as it likely has very stale data. ${formatDateTime(lastTime)} < ${formatDateTime(suicideThreshold)}, now is ${formatDateTime(Date.now())}`));
+            process.exit();
+        }
+    }
+    let now = Date.now();
+    console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
     await archives().set(nodeId, Buffer.from(now + ""), { fallbacks: true });
+    if (firstHeartbeatWriteTime === undefined) {
+        firstHeartbeatWriteTime = now;
+    }
 }
 
 async function runServerSyncLoops() {
@@ -533,34 +551,15 @@ async function runServerSyncLoops() {
     console.log(magenta(`Node discovery is loaded`));
 
     await runInfinitePollCallAtStart(HEARTBEAT_INTERVAL, writeHeartbeat);
-    // Delayed by a bit as it might take a little bit for our heartbeat to show up on startup, especially if we are the storage node that is supposed to store our own heartbeat.
-    delay(HEARTBEAT_INTERVAL * 5).then(() => runInfinitePoll(HEARTBEAT_INTERVAL, async function nodeDiscoverHeartbeat() {
-        // If we waited too long, other nodes might think we are dead. In which case, we SHOULD terminate.
-        if (!isNoNetwork()) {
-            // FIRST, verify we didn't delay too long (to make sure we kill any nodes that were disconnected
-            //  from the internet for too long)
-            // NOTE: If get can't contact a server, it will throw. It won't return undefined. So if this does return undefined, It means we actually talk to the server and it says the file doesn't exist, which means we are fairly safe in self-terminating.
-            //let lastTime = Number((await archives().get(selfNodeId, { noFallbacks: true }))?.toString());
-            // NOTE: We're temporary not using no fallbacks, So, this is a good canary to tell us if something's wrong with storage. We're writing so many heartbeats, at least one of the writes should have gotten to Backblaze.
-            let lastTime = Number((await archives().get(selfNodeId))?.toString());
-            let suicideThreshold = Date.now() - SUICIDE_HEARTBEAT_THRESHOLD;
-            if (!lastTime || lastTime < suicideThreshold) {
-                if (!lastTime) {
-                    console.error(red(`Self node was removed due to heartbeat file not existing (${selfNodeId}). Terminating self process, as it likely has very stale data.`));
-                } else {
-                    console.error(red(`Self node was has very old heartbeat. Terminating self process, as it likely has very stale data. ${formatDateTime(lastTime)} < ${formatDateTime(suicideThreshold)}, now is ${formatDateTime(Date.now())}`));
-                }
-                process.exit();
-            }
-        } else {
-            // We DO have to re-broadcast, otherwise no one will know we exist
-            for (let nodeId of allNodeIds2) {
-                if (isOwnNodeId(nodeId)) continue;
-                // Ignore errors, as nodes die often, and we only want to log once - when we actually remove the node id.
-                ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId()));
-            }
+    runInfinitePoll(HEARTBEAT_INTERVAL, async function nodeDiscoverRebroadcast() {
+        if (!isNoNetwork()) return;
+        // We DO have to re-broadcast, otherwise no one will know we exist
+        for (let nodeId of allNodeIds2) {
+            if (isOwnNodeId(nodeId)) continue;
+            // Ignore errors, as nodes die often, and we only want to log once - when we actually remove the node id.
+            ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId()));
         }
-    }));
+    });
 }
 
 let discoveryReady = new PromiseObj<void>();