querysub 0.17.0 → 0.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.17.0",
+    "version": "0.18.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -22,6 +22,7 @@ import { getPublicIP } from "../misc/networking";
 import dns from "dns/promises";
 import { isDefined } from "../misc";
 import { diskLog, noDiskLogPrefix } from "../diagnostics/logs/diskLogger";
+import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 2;
 // Interval which we check other heartbeats
@@ -37,10 +38,16 @@ let SUICIDE_HEARTBEAT_THRESHOLD = timeInMinute * 15;
 
 let CLIENTSIDE_POLL_RATE = timeInMinute * 5;
 
+// We can't poll backblaze too often. One a minute starts to cost around 10 cents per month.
+//  So once a second would cost 6 USD per month, per service... which starts to get expensive.
+let DISK_AUDIT_RATE = timeInMinute * 5;
+// We CAN poll our API frequently. The overhead for calls should be far less than 1ms, and
+//  the bandwidth should also be effectively nothing. Maybe 2.5GB per month if we send a
+//  request per second, and each request is 1000 bytes (which as we use websockets, it
+//  probably is less than that). Which is around 2.5 cents on backblaze IF we go over
+//  our 1TB/month allowance.
 let API_AUDIT_RATE = timeInSecond;
-
-let MEMORY_AUDIT_RATE = timeInMinute * 5;
-let MEMORY_AUDIT_COUNT = 3;
+let API_AUDIT_COUNT = 12;
 
 let shutdown = false;
 
@@ -161,6 +168,7 @@ function addNodeIdBase(nodeId: string) {
 }
 function setNodeIds(nodeIds: string[]) {
     nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
+
     diskLog("setNodeIds", { nodeIds });
     // Also try all localhost ports, if we are in dev mode
     if (isNode() && isDevDebugbreak()) {
@@ -285,14 +293,15 @@ async function runHeartbeatAuditLoop() {
 }
 
 async function fastMemorySync() {
-    let checkNodes = shuffle(Array.from(allNodeIds2).filter(x => x !== getOurNodeId()), Date.now()).slice(0, MEMORY_AUDIT_COUNT).slice(0, 10);
+    let checkNodes = shuffle(Array.from(allNodeIds2).filter(x => x !== getOurNodeId()), Date.now()).slice(0, API_AUDIT_COUNT);
     let otherNodesAll = await Promise.all(
         checkNodes.map(async nodeId =>
             await timeoutToUndefinedSilent(200, NodeDiscoveryController.nodes[nodeId].getAllNodeIds()) || []
         )
     );
     let otherNodes = Array.from(new Set(otherNodesAll.flat()));
-    //console.log(magenta(`Fast memory sync at ${formatVeryNiceDateTime(Date.now())}, nodes found ${otherNodes.length}`));
+    // This would log WAY too much, because we poll a lot, because we want to minimize downtime
+    //console.log(magenta(`Fast memory sync at ${formatVeryNiceDateTime(Date.now())}, nodes found ${otherNodes.length}`), getDebuggerUrl());
 
     // If they are missing nodes that's fine. We constantly have extra nodes, and have to function correctly
     //  with extra nodes. However, if we are missing nodes, we'd prefer to have them quickly, so we should
@@ -306,7 +315,7 @@ async function fastMemorySync() {
 
 async function runMemoryAuditLoop() {
     await onNodeDiscoveryReady();
-    runInfinitePoll(MEMORY_AUDIT_RATE, syncArchives);
+    runInfinitePoll(DISK_AUDIT_RATE, syncArchives);
     runInfinitePoll(API_AUDIT_RATE, fastMemorySync);
 }