querysub 0.646.0 → 0.647.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.646.0",
+  "version": "0.647.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-c-identity/IdentityController.ts

@@ -12,22 +12,26 @@ import { getClientNodeId, getNodeId, getNodeIdDomain, getNodeIdIP, getNodeIdLoca
 import { decodeNodeId, getCommonName, getIdentityCA, getMachineId, getOwnMachineId, getPublicIdentifier, getThreadKeyCert, parseCert, sign, validateCertificate, verify } from "sliftutils/misc/https/certs";
 import { getShortNumber } from "socket-function/src/bits";
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { timeoutToError } from "../errors";
+import { logErrors, timeoutToError } from "../errors";
 import { delay } from "socket-function/src/batching";
 import { formatTime } from "socket-function/src/formatting/format";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { red } from "socket-function/src/formatting/logColors";
 import { isNode } from "typesafecss";
 import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
-import { timeInMinute, isIpDomain } from "socket-function/src/misc";
+import { timeInMinute, isIpDomain, PromiseObj } from "socket-function/src/misc";
 import { isClient, isServer } from "../config2";
 import { getDomain } from "../config";
 
 // NOTE: This used to be small, but we cache this, so it would mean a node on startup would time out, and then we would refuse to talk to it ever again. So... this can't be small
 const MAX_CHANGE_IDENTITY_TIMEOUT = timeInMinute * 5;
+// Such a large timeout because sometimes startup can be really slow, such as if our storage system has to fall back (as the storage system does retries, etc).
+const MOUNT_WAIT_TIMEOUT = timeInMinute * 3;
 
 let callerInfo = new Map<CallerContext, {
-    reconnectNodeId: string | undefined;
+    reconnectNodeId: string | Promise<string | Error> | undefined;
+    resolveReconnectNodeId?: (value: string | Error) => void;
+    certCommonName: string;
     machineId: string;
     cert: CertInfo;
     pubKey: Buffer;
@@ -49,25 +53,40 @@ export function IdentityController_getReconnectNodeId(callerContext: CallerConte
         if (allowUndefined) return undefined;
         throw new Error(callerInfoErrorString);
     }
-    return info.reconnectNodeId;
+    let value = info.reconnectNodeId;
+    if (typeof value !== "string") return undefined;
+    return value;
 }
-export function IdentityController_getReconnectNodeIdAssert(callerContext: CallerContext): string {
+export async function IdentityController_getReconnectNodeIdAssert(callerContext: CallerContext): Promise<string> {
     if (!isClientNodeId(callerContext.nodeId)) {
         return callerContext.nodeId;
     }
-    let reconnectId = IdentityController_getReconnectNodeId(callerContext);
-    if (!reconnectId) {
+    let info = callerInfo.get(callerContext);
+    if (!info) {
+        throw new Error(callerInfoErrorString);
+    }
+    let valueBase = info.reconnectNodeId;
+    let value: string | Error | undefined;
+    if (valueBase instanceof Promise) {
+        value = await valueBase;
+    } else {
+        value = valueBase;
+    }
+    if (value instanceof Error) {
+        throw value;
+    }
+    if (!value) {
         console.error(`Caller did not mount before connecting. This call requires the caller to be listening.`, {
             callerNodeId: callerContext.nodeId,
         });
         throw new Error(`Caller did not mount before connecting. This call requires the caller to be listening.`);
     }
-    return reconnectId;
+    return value;
 }
 export function IdentityController_getSecureIP(callerContext: CallerContext): string {
     return getNodeIdIP(callerContext.nodeId);
 }
-export function IdentityController_getCurrentReconnectNodeIdAssert(): string {
+export function IdentityController_getCurrentReconnectNodeIdAssert(): Promise<string> {
     return IdentityController_getReconnectNodeIdAssert(SocketFunction.getCaller());
 }
 export function IdentityController_getCurrentReconnectNodeId(): string | undefined {
@@ -76,7 +95,9 @@ export function IdentityController_getCurrentReconnectNodeId(): string | undefin
 
 export function debugNodeId(nodeId: string) {
     let info = Array.from(callerInfo.entries()).find(x => x[0].nodeId === nodeId);
-    return info?.[1].reconnectNodeId || nodeId;
+    let value = info?.[1].reconnectNodeId;
+    if (typeof value !== "string") return nodeId;
+    return value;
 };
 (globalThis as any).debugNodeId = debugNodeId;
 
@@ -177,12 +198,21 @@ class IdentityControllerBase {
         // Verify we even trust the issuer/cert pair
         validateCertificate(getDomain(), payload.cert, payload.certIssuer);
 
-        let reconnectNodeId: string | undefined;
+        // NOTE: We use the common name, and not getNodeIdIP... because anything connecting will need
+        //  to use HTTPS, so connecting over the IP won't work! (unless they turn off certificate validation,
+        //  which we shouldn't do...)
+        let certCommonName = getCommonName(payload.cert);
+        let reconnectNodeId: string | Promise<string | Error> | undefined;
+        let resolveReconnectNodeId: ((value: string | Error) => void) | undefined;
         if (payload.mountedPort) {
-            // NOTE: We use the common name, and not getNodeIdIP... because anything connecting will need
-            //  to use HTTPS, so connecting over the IP won't work! (unless they turn off certificate validation,
-            //  which we shouldn't do...)
-            reconnectNodeId = getNodeId(getCommonName(payload.cert), payload.mountedPort);
+            reconnectNodeId = getNodeId(certCommonName, payload.mountedPort);
+        } else {
+            let pending = new PromiseObj<string | Error>();
+            reconnectNodeId = pending.promise;
+            resolveReconnectNodeId = pending.resolve;
+            setTimeout(() => {
+                pending.resolve(new Error(`Caller did not mount within ${formatTime(MOUNT_WAIT_TIMEOUT)} of connecting. This call requires the caller to be listening.`));
+            }, MOUNT_WAIT_TIMEOUT);
         }
 
         let pubKey = getPublicIdentifier(payload.cert);
@@ -191,7 +221,9 @@ class IdentityControllerBase {
         callerInfo.set(caller, {
             cert: { certPEM: payload.cert, issuerPEM: payload.certIssuer },
             machineId,
+            certCommonName,
             reconnectNodeId,
+            resolveReconnectNodeId,
             pubKey,
             pubKeyShort: getShortNumber(pubKey),
         });
@@ -206,13 +238,28 @@ class IdentityControllerBase {
         });
 
         SocketFunction.onNextDisconnect(caller.nodeId, () => {
-            // NOTE: I don't really see any purpose of deleting from caller info. I don't think we're going to run out of memory because of too many callers authenticating. 
+            // NOTE: I don't really see any purpose of deleting from caller info. I don't think we're going to run out of memory because of too many callers authenticating.
             // However, logging here is useful as it allows us to complete the life cycle so we know how long a client was connected for.
             console.info(`Disconnected client`, {
                 clientId: caller.nodeId,
             });
         });
     }
+
+    @measureFnc
+    public async setMountedPort(mountedPort: number) {
+        const caller = SocketFunction.getCaller();
+        let info = callerInfo.get(caller);
+        if (!info) throw new Error(callerInfoErrorString);
+        let reconnectNodeId = getNodeId(info.certCommonName, mountedPort);
+        info.reconnectNodeId = reconnectNodeId;
+        info.resolveReconnectNodeId?.(reconnectNodeId);
+        console.info(`Caller mounted after connecting, set reconnect node id`, {
+            clientId: caller.nodeId,
+            reconnectNodeId,
+            mountedPort,
+        });
+    }
 }
 
 const IdentityController = SocketFunction.register(
@@ -227,6 +274,7 @@ const IdentityController = SocketFunction.register(
             // noClientHooks: true,
             // noDefaultHooks: true,
         },
+        setMountedPort: {},
     })
 );
 
@@ -250,6 +298,19 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         IdentityController.nodes[nodeId].changeIdentity(signature, payload),
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
     );
+    if (isServer() && !payload.mountedPort) {
+        logErrors((async () => {
+            await SocketFunction.mountPromise;
+            let mountedPort = getNodeIdLocation(SocketFunction.mountedNodeId)?.port;
+            if (!mountedPort) return;
+            console.info(`Mounted after connecting to ${nodeId}, sending mounted port ${mountedPort}`);
+            await timeoutToError(
+                MAX_CHANGE_IDENTITY_TIMEOUT,
+                IdentityController.nodes[nodeId].setMountedPort(mountedPort),
+                () => new Error(`Timeout calling setMountedPort for ${nodeId}`)
+            );
+        })());
+    }
 });
 let startupTime = Date.now();
 async function identityHook(context: ClientHookContext) {

package/src/0-path-value-core/AuthorityLookup.ts

@@ -180,7 +180,7 @@ class AuthorityLookup {
         isReady: boolean;
         scheduledShutdownTime?: number;
     }> {
-        let nodeId = IdentityController_getCurrentReconnectNodeIdAssert();
+        let nodeId = await IdentityController_getCurrentReconnectNodeIdAssert();
         this.updatePaths(nodeId, config.spec, config.isReady, config.scheduledShutdownTime);
         console.info(`Received network sync, returning our data`, {
             nodeId,
@@ -211,7 +211,7 @@ class AuthorityLookup {
         }, stopObj);
 
         const onNodeRemoved = () => {
-            this.topology.nodes.delete(nodeId);
+            this.removeNode(nodeId, "node removed from node discovery");
             stopObj.stop = true;
             this.connectTo.clear(nodeId);
             this.disconnectWatch.clear(nodeId);
@@ -230,15 +230,20 @@ class AuthorityLookup {
         let onDisconnect = () => {
             SocketFunction.onNextDisconnect(nodeId, onDisconnect, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
             // NOTE: Will be re-added once we receive an updatePaths call (or the node will be removed by NodeDiscovery, in which case we will be entirely cleaned up).
-            this.topology.nodes.delete(nodeId);
+            this.removeNode(nodeId, "socket disconnected");
         };
         SocketFunction.onNextDisconnect(nodeId, onDisconnect, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
     });
+    private removeNode(nodeId: string, reason: string) {
+        if (!this.topology.nodes.has(nodeId)) return;
+        this.topology.nodes.delete(nodeId);
+        console.log(`Removed authority topology entry for ${nodeId} (${this.topology.nodes.size} entries remaining). Reason: ${reason}`);
+    }
     private async syncNodeNow(nodeId: string) {
         try {
             await this.syncNodeNowBase(nodeId);
-        } catch {
-            this.topology.nodes.delete(nodeId);
+        } catch (e: any) {
+            this.removeNode(nodeId, `sync failed: ${String(e?.stack || e)}`);
         }
     }
 

package/src/0-path-value-core/PathValueCommitter.ts

@@ -210,6 +210,7 @@ class PathValueCommitter {
                         timeId: pathValue.time.time,
                         source: pathValue.source,
                         otherAuthorities,
+                        rawTopology: authorityLookup.getTopologySync(),
                     });
                     continue;
                 }

package/src/deployManager/components/storage/storageLogsController.ts

@@ -66,7 +66,8 @@ class StorageLogsControllerBase {
             let stores: ServerPathValue["stores"] = [];
             try {
                 let routing = await controller.get2({ account, bucketName, path: ROUTING_FILE, sourceConfig: fabricatedSource, internal: true });
-                let sources = routing && parseRoutingData(routing.data).sources.map(normalizeSource) || [];
+                let routingParsed = routing && parseRoutingData(routing.data);
+                let sources = routingParsed && routingParsed.sources.map(normalizeSource) || [];
                 let seenNames = new Set<string>();
                 for (let source of sources) {
                     if (source.type !== "remote") continue;