querysub 0.16.0 → 0.18.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.16.0",
+    "version": "0.18.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -13,7 +13,7 @@ import { blue, green, magenta, red, yellow } from "socket-function/src/formattin
 import debugbreak from "debugbreak";
 import { getArchivesBackblaze, getBackblazePath } from "../-a-archives/archivesBackBlaze";
 import { PromiseObj } from "../promise";
-import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { formatDateTime, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
 import { isClient, isServer } from "../config2";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { decodeNodeId, decodeNodeIdAssert, getMachineId } from "../-a-auth/certs";
@@ -22,6 +22,7 @@ import { getPublicIP } from "../misc/networking";
 import dns from "dns/promises";
 import { isDefined } from "../misc";
 import { diskLog, noDiskLogPrefix } from "../diagnostics/logs/diskLogger";
+import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 2;
 // Interval which we check other heartbeats
@@ -37,10 +38,16 @@ let SUICIDE_HEARTBEAT_THRESHOLD = timeInMinute * 15;
 
 let CLIENTSIDE_POLL_RATE = timeInMinute * 5;
 
-let API_AUDIT_RATE = timeInSecond * 5;
-
-let MEMORY_AUDIT_RATE = timeInMinute * 5;
-let MEMORY_AUDIT_COUNT = 3;
+// We can't poll backblaze too often. One a minute starts to cost around 10 cents per month.
+//  So once a second would cost 6 USD per month, per service... which starts to get expensive.
+let DISK_AUDIT_RATE = timeInMinute * 5;
+// We CAN poll our API frequently. The overhead for calls should be far less than 1ms, and
+//  the bandwidth should also be effectively nothing. Maybe 2.5GB per month if we send a
+//  request per second, and each request is 1000 bytes (which as we use websockets, it
+//  probably is less than that). Which is around 2.5 cents on backblaze IF we go over
+//  our 1TB/month allowance.
+let API_AUDIT_RATE = timeInSecond;
+let API_AUDIT_COUNT = 12;
 
 let shutdown = false;
 
@@ -161,6 +168,7 @@ function addNodeIdBase(nodeId: string) {
 }
 function setNodeIds(nodeIds: string[]) {
     nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
+
     diskLog("setNodeIds", { nodeIds });
     // Also try all localhost ports, if we are in dev mode
     if (isNode() && isDevDebugbreak()) {
@@ -284,29 +292,30 @@ async function runHeartbeatAuditLoop() {
     });
 }
 
-async function fastMemorySync(retries = 3) {
-    let checkNode = shuffle(Array.from(allNodeIds2).filter(x => x !== getOurNodeId()), Date.now()).slice(0, MEMORY_AUDIT_COUNT).at(0);
-    if (!checkNode) return;
-    let otherNodes = await errorToUndefinedSilent(NodeDiscoveryController.nodes[checkNode].getAllNodeIds());
-    if (!otherNodes) {
-        if (retries > 0) {
-            await fastMemorySync(retries - 1);
-        }
-        return;
-    }
+async function fastMemorySync() {
+    let checkNodes = shuffle(Array.from(allNodeIds2).filter(x => x !== getOurNodeId()), Date.now()).slice(0, API_AUDIT_COUNT);
+    let otherNodesAll = await Promise.all(
+        checkNodes.map(async nodeId =>
+            await timeoutToUndefinedSilent(200, NodeDiscoveryController.nodes[nodeId].getAllNodeIds()) || []
+        )
+    );
+    let otherNodes = Array.from(new Set(otherNodesAll.flat()));
+    // This would log WAY too much, because we poll a lot, because we want to minimize downtime
+    //console.log(magenta(`Fast memory sync at ${formatVeryNiceDateTime(Date.now())}, nodes found ${otherNodes.length}`), getDebuggerUrl());
+
     // If they are missing nodes that's fine. We constantly have extra nodes, and have to function correctly
     //  with extra nodes. However, if we are missing nodes, we'd prefer to have them quickly, so we should
     //  sync now.
     let missingNodes = otherNodes.filter(nodeId => !allNodeIds2.has(nodeId));
     if (missingNodes.length > 0) {
-        console.log(yellow(`Node list is missing nodes, resyncing node`), { checkNode, missingNodes, otherNodes });
+        console.log(yellow(`Node list is missing nodes, resyncing node`), { missingNodes, otherNodes });
         await syncArchives();
     }
 }
 
 async function runMemoryAuditLoop() {
     await onNodeDiscoveryReady();
-    runInfinitePoll(MEMORY_AUDIT_RATE, syncArchives);
+    runInfinitePoll(DISK_AUDIT_RATE, syncArchives);
     runInfinitePoll(API_AUDIT_RATE, fastMemorySync);
 }
 
@@ -473,6 +482,7 @@ const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNod
 
 class NodeDiscoveryControllerBase {
     public async addNode(nodeId: string) {
+        console.log(magenta(`Received addNode`), { nodeId });
         addNodeId(nodeId);
     }
     public async resyncNodes() {
@@ -483,7 +493,7 @@ class NodeDiscoveryControllerBase {
     }
 
     public async getAllNodeIds(): Promise<string[]> {
-        return Array.from(allNodeIds2);
+        return Array.from(allNodeIds2).filter(x => !x.startsWith("127-0-0-1."));
     }
     public async getNodeId() {
         console.log(`Returning node id ${SocketFunction.mountedNodeId}`);

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -949,7 +949,9 @@ class PathControllerBase {
     }
 
     public async broadcastReadReady(time: number) {
-        onReadyReady(IdentityController_getCurrentReconnectNodeIdAssert(), time);
+        let nodeIdCaller = IdentityController_getCurrentReconnectNodeIdAssert();
+        console.log(magenta(`Received ready broadcast`), { nodeIdCaller });
+        onReadyReady(nodeIdCaller, time);
     }
 }
 const PathController = SocketFunction.register(

package/src/1-path-client/RemoteWatcher.ts

@@ -170,7 +170,10 @@ export class RemoteWatcher {
                 //  us to circumvent earlier watchers, causing us to watch out of order, causing reads
                 //  to come back out of order, causing watching functions to trigger out of order!
                 let remoteAuthorityId = this.remoteWatchPaths.get(path);
-                if (remoteAuthorityId !== undefined) continue;
+                if (remoteAuthorityId !== undefined) {
+                    this.disconnectedPaths.delete(path);
+                    continue;
+                }
 
                 let authorityId = pathValueAuthority2.getSingleReadNodeSync(path);
                 if (!authorityId) {