querysub 0.646.0 → 0.648.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.646.0",
+  "version": "0.648.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-c-identity/IdentityController.ts

@@ -12,28 +12,39 @@ import { getClientNodeId, getNodeId, getNodeIdDomain, getNodeIdIP, getNodeIdLoca
 import { decodeNodeId, getCommonName, getIdentityCA, getMachineId, getOwnMachineId, getPublicIdentifier, getThreadKeyCert, parseCert, sign, validateCertificate, verify } from "sliftutils/misc/https/certs";
 import { getShortNumber } from "socket-function/src/bits";
 import { measureBlock, measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { timeoutToError } from "../errors";
+import { logErrors, timeoutToError } from "../errors";
 import { delay } from "socket-function/src/batching";
 import { formatTime } from "socket-function/src/formatting/format";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { red } from "socket-function/src/formatting/logColors";
 import { isNode } from "typesafecss";
 import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
-import { timeInMinute, isIpDomain } from "socket-function/src/misc";
+import { timeInMinute, isIpDomain, PromiseObj } from "socket-function/src/misc";
 import { isClient, isServer } from "../config2";
 import { getDomain } from "../config";
 
 // NOTE: This used to be small, but we cache this, so it would mean a node on startup would time out, and then we would refuse to talk to it ever again. So... this can't be small
 const MAX_CHANGE_IDENTITY_TIMEOUT = timeInMinute * 5;
+// Such a large timeout because sometimes startup can be really slow, such as if our storage system has to fall back (as the storage system does retries, etc).
+const MOUNT_WAIT_TIMEOUT = timeInMinute * 3;
 
 let callerInfo = new Map<CallerContext, {
-    reconnectNodeId: string | undefined;
+    reconnectNodeId: string | Promise<string>;
+    reconnectNodeIdPromiseObj: PromiseObj<string> | undefined;
+    certCommonName: string;
     machineId: string;
     cert: CertInfo;
     pubKey: Buffer;
     pubKeyShort: number;
 }>();
-const callerInfoErrorString = `Internal error, caller did not updated their identity. Is this an HTTPS call (instead of a websocket call)? The caller should import the server controller, which imports this function, which will add a client hook to always update the identity`;
+
+function getCallerInfoAssert(callerContext: CallerContext) {
+    let info = callerInfo.get(callerContext);
+    if (!info) {
+        throw new Error(`Internal error, caller did not updated their identity. Is this an HTTPS call (instead of a websocket call)? The caller should import the server controller, which imports this function, which will add a client hook to always update the identity`);
+    }
+    return info;
+}
 
 /** Gets the nodeId of the caller suitable for reconnecting (to the same process).
  *  - Also useful for logs, to identify a node on the network.
@@ -47,27 +58,24 @@ export function IdentityController_getReconnectNodeId(callerContext: CallerConte
     let info = callerInfo.get(callerContext);
     if (!info) {
         if (allowUndefined) return undefined;
-        throw new Error(callerInfoErrorString);
+        info = getCallerInfoAssert(callerContext);
+    }
+    if (typeof info.reconnectNodeId === "string") {
+        return info.reconnectNodeId;
     }
-    return info.reconnectNodeId;
+    return undefined;
 }
-export function IdentityController_getReconnectNodeIdAssert(callerContext: CallerContext): string {
+export async function IdentityController_getReconnectNodeIdAssert(callerContext: CallerContext): Promise<string> {
     if (!isClientNodeId(callerContext.nodeId)) {
         return callerContext.nodeId;
     }
-    let reconnectId = IdentityController_getReconnectNodeId(callerContext);
-    if (!reconnectId) {
-        console.error(`Caller did not mount before connecting. This call requires the caller to be listening.`, {
-            callerNodeId: callerContext.nodeId,
-        });
-        throw new Error(`Caller did not mount before connecting. This call requires the caller to be listening.`);
-    }
-    return reconnectId;
+    let info = getCallerInfoAssert(callerContext);
+    return await info.reconnectNodeId;
 }
 export function IdentityController_getSecureIP(callerContext: CallerContext): string {
     return getNodeIdIP(callerContext.nodeId);
 }
-export function IdentityController_getCurrentReconnectNodeIdAssert(): string {
+export function IdentityController_getCurrentReconnectNodeIdAssert(): Promise<string> {
     return IdentityController_getReconnectNodeIdAssert(SocketFunction.getCaller());
 }
 export function IdentityController_getCurrentReconnectNodeId(): string | undefined {
@@ -76,7 +84,11 @@ export function IdentityController_getCurrentReconnectNodeId(): string | undefin
 
 export function debugNodeId(nodeId: string) {
     let info = Array.from(callerInfo.entries()).find(x => x[0].nodeId === nodeId);
-    return info?.[1].reconnectNodeId || nodeId;
+    let reconnectNodeId = info?.[1].reconnectNodeId;
+    if (typeof reconnectNodeId !== "string") {
+        return nodeId;
+    }
+    return reconnectNodeId;
 };
 (globalThis as any).debugNodeId = debugNodeId;
 
@@ -97,21 +109,19 @@ export function IdentityController_getMachineId(callerContext: CallerContext, al
     let info = callerInfo.get(callerContext);
     if (!info) {
         if (allowEmpty) return "";
-        throw new Error(callerInfoErrorString);
+        info = getCallerInfoAssert(callerContext);
     }
     return info.machineId;
 }
 
 export type CertInfo = { certPEM: Buffer | string; issuerPEM: Buffer | string; };
 export function IdentityController_getCertInfo(callerContext: CallerContext): CertInfo {
-    let info = callerInfo.get(callerContext);
-    if (!info) throw new Error(callerInfoErrorString);
+    let info = getCallerInfoAssert(callerContext);
     return info.cert;
 }
 
 export function IdentityController_getPubKeyShort(callerContext: CallerContext): number {
-    let info = callerInfo.get(callerContext);
-    if (!info) throw new Error(callerInfoErrorString);
+    let info = getCallerInfoAssert(callerContext);
     return info.pubKeyShort;
 }
 export const IdentityController_getOwnPubKeyShort = lazy((): number => {
@@ -128,7 +138,7 @@ export interface ChangeIdentityPayload {
     serverId: string;
     mountedPort: number | undefined;
     debugEntryPoint: string | undefined;
-    clientIsNode: boolean;
+    callerIsNode: boolean;
 }
 class IdentityControllerBase {
     // IMPORTANT! We HAVE to call changeIdentity NOT JUST because we can't use peer certificates in the browser, BUT, also
@@ -146,7 +156,7 @@ class IdentityControllerBase {
             throw new Error(`Signed payload too old, ${payload.time} < ${signedThreshold} from ${caller.localNodeId} (${caller.nodeId})`);
         }
 
-        if (payload.clientIsNode && payload.serverId !== getOwnNodeId()) {
+        if (payload.callerIsNode && payload.serverId !== getOwnNodeId()) {
             // This is extremely common when we reuse ports, which we do frequently for the edge nodes. 
             throw new Error(`You tried to contact another server. We are ${getOwnNodeId()}, you tried to contact ${payload.serverId}.`);
         }
@@ -159,7 +169,7 @@ class IdentityControllerBase {
             throw new Error(`Identity is for another server! The connection is calling us ${localNodeId}, but signature is for ${payload.serverId}`);
         }
         // If they're calling from the browser, then they're not going to be able to use our machine ID, etc. However, they should be calling an actual https node, so it should still be secure for them. 
-        if (payload.clientIsNode) {
+        if (payload.callerIsNode) {
             let calledMachineId = getMachineId(payload.serverId, getDomain());
             if (calledMachineId !== "127-0-0-1" && calledMachineId !== getOwnMachineId(getDomain())) {
                 throw new Error(`Tried to call a different machine. We are ${getOwnMachineId(getDomain())}, they called ${calledMachineId}`);
@@ -177,12 +187,24 @@ class IdentityControllerBase {
         // Verify we even trust the issuer/cert pair
         validateCertificate(getDomain(), payload.cert, payload.certIssuer);
 
-        let reconnectNodeId: string | undefined;
+        // NOTE: We use the common name, and not getNodeIdIP... because anything connecting will need
+        //  to use HTTPS, so connecting over the IP won't work! (unless they turn off certificate validation,
+        //  which we shouldn't do...)
+        let certCommonName = getCommonName(payload.cert);
+        let reconnectNodeId: string | Promise<string>;
+        let reconnectNodeIdPromiseObj: PromiseObj<string> | undefined;
         if (payload.mountedPort) {
-            // NOTE: We use the common name, and not getNodeIdIP... because anything connecting will need
-            //  to use HTTPS, so connecting over the IP won't work! (unless they turn off certificate validation,
-            //  which we shouldn't do...)
-            reconnectNodeId = getNodeId(getCommonName(payload.cert), payload.mountedPort);
+            reconnectNodeId = getNodeId(certCommonName, payload.mountedPort);
+        } else {
+            reconnectNodeIdPromiseObj = new PromiseObj<string>();
+            reconnectNodeId = reconnectNodeIdPromiseObj.promise;
+            setTimeout(() => {
+                reconnectNodeIdPromiseObj?.reject(new Error(`Caller did not call setMountedPort within ${formatTime(MOUNT_WAIT_TIMEOUT)} of connecting. This call requires the caller to be listening.`));
+            }, MOUNT_WAIT_TIMEOUT);
+        }
+
+        if (!payload.callerIsNode) {
+            reconnectNodeIdPromiseObj?.reject(new Error(`The caller says they are a client, so they are never going to have a reconnect node id`));
         }
 
         let pubKey = getPublicIdentifier(payload.cert);
@@ -191,7 +213,9 @@ class IdentityControllerBase {
         callerInfo.set(caller, {
             cert: { certPEM: payload.cert, issuerPEM: payload.certIssuer },
             machineId,
+            certCommonName,
             reconnectNodeId,
+            reconnectNodeIdPromiseObj,
             pubKey,
             pubKeyShort: getShortNumber(pubKey),
         });
@@ -206,13 +230,28 @@ class IdentityControllerBase {
         });
 
         SocketFunction.onNextDisconnect(caller.nodeId, () => {
-            // NOTE: I don't really see any purpose of deleting from caller info. I don't think we're going to run out of memory because of too many callers authenticating. 
+            // NOTE: I don't really see any purpose of deleting from caller info. I don't think we're going to run out of memory because of too many callers authenticating.
             // However, logging here is useful as it allows us to complete the life cycle so we know how long a client was connected for.
             console.info(`Disconnected client`, {
                 clientId: caller.nodeId,
             });
         });
     }
+
+    @measureFnc
+    public async setMountedPort(mountedPort: number) {
+        const caller = SocketFunction.getCaller();
+        let info = getCallerInfoAssert(caller);
+        if (typeof info.reconnectNodeId === "string") throw new Error(`Already have reconnectNodeId, not sure why we are being given another, Have ${info.reconnectNodeId}, given port: ${mountedPort}`);
+        let reconnectNodeId = getNodeId(info.certCommonName, mountedPort);
+        info.reconnectNodeId = reconnectNodeId;
+        info.reconnectNodeIdPromiseObj?.resolve(reconnectNodeId);
+        console.info(`Caller mounted after connecting, set reconnect node id`, {
+            clientId: caller.nodeId,
+            reconnectNodeId,
+            mountedPort,
+        });
+    }
 }
 
 const IdentityController = SocketFunction.register(
@@ -227,6 +266,7 @@ const IdentityController = SocketFunction.register(
             // noClientHooks: true,
             // noDefaultHooks: true,
         },
+        setMountedPort: {},
     })
 );
 
@@ -242,7 +282,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         certIssuer: issuer.cert.toString(),
         mountedPort: getNodeIdLocation(SocketFunction.mountedNodeId)?.port,
         debugEntryPoint: isServer() ? process.argv[1] : "browser",
-        clientIsNode: isServer() && !isIpDomain(nodeId),
+        callerIsNode: isServer() && !isIpDomain(nodeId),
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(
@@ -250,6 +290,15 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         IdentityController.nodes[nodeId].changeIdentity(signature, payload),
         () => new Error(`Timeout calling changeIdentity for ${nodeId}`)
     );
+    if (isServer() && !payload.mountedPort) {
+        logErrors((async () => {
+            await SocketFunction.mountPromise;
+            let mountedPort = getNodeIdLocation(SocketFunction.mountedNodeId)?.port;
+            if (!mountedPort) throw new Error(`Mounted after connecting to ${nodeId}, but no mounted port found? mounted node id: ${SocketFunction.mountedNodeId}`);
+            console.info(`Mounted after connecting to ${nodeId}, sending mounted port ${mountedPort}`);
+            await IdentityController.nodes[nodeId].setMountedPort(mountedPort);
+        })());
+    }
 });
 let startupTime = Date.now();
 async function identityHook(context: ClientHookContext) {

package/src/0-path-value-core/AuthorityLookup.ts

@@ -180,7 +180,7 @@ class AuthorityLookup {
         isReady: boolean;
         scheduledShutdownTime?: number;
     }> {
-        let nodeId = IdentityController_getCurrentReconnectNodeIdAssert();
+        let nodeId = await IdentityController_getCurrentReconnectNodeIdAssert();
         this.updatePaths(nodeId, config.spec, config.isReady, config.scheduledShutdownTime);
         console.info(`Received network sync, returning our data`, {
             nodeId,
@@ -211,7 +211,7 @@ class AuthorityLookup {
         }, stopObj);
 
         const onNodeRemoved = () => {
-            this.topology.nodes.delete(nodeId);
+            this.removeNode(nodeId, "node removed from node discovery");
             stopObj.stop = true;
             this.connectTo.clear(nodeId);
             this.disconnectWatch.clear(nodeId);
@@ -230,15 +230,20 @@ class AuthorityLookup {
         let onDisconnect = () => {
             SocketFunction.onNextDisconnect(nodeId, onDisconnect, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
             // NOTE: Will be re-added once we receive an updatePaths call (or the node will be removed by NodeDiscovery, in which case we will be entirely cleaned up).
-            this.topology.nodes.delete(nodeId);
+            this.removeNode(nodeId, "socket disconnected");
         };
         SocketFunction.onNextDisconnect(nodeId, onDisconnect, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
     });
+    private removeNode(nodeId: string, reason: string) {
+        if (!this.topology.nodes.has(nodeId)) return;
+        this.topology.nodes.delete(nodeId);
+        console.log(`Removed authority topology entry for ${nodeId} (${this.topology.nodes.size} entries remaining). Reason: ${reason}`);
+    }
     private async syncNodeNow(nodeId: string) {
         try {
             await this.syncNodeNowBase(nodeId);
-        } catch {
-            this.topology.nodes.delete(nodeId);
+        } catch (e: any) {
+            this.removeNode(nodeId, `sync failed: ${String(e?.stack || e)}`);
         }
     }
 

package/src/0-path-value-core/PathValueCommitter.ts

@@ -210,6 +210,7 @@ class PathValueCommitter {
                         timeId: pathValue.time.time,
                         source: pathValue.source,
                         otherAuthorities,
+                        rawTopology: authorityLookup.getTopologySync(),
                     });
                     continue;
                 }

package/src/deployManager/components/storage/storageLogsController.ts

@@ -66,7 +66,8 @@ class StorageLogsControllerBase {
             let stores: ServerPathValue["stores"] = [];
             try {
                 let routing = await controller.get2({ account, bucketName, path: ROUTING_FILE, sourceConfig: fabricatedSource, internal: true });
-                let sources = routing && parseRoutingData(routing.data).sources.map(normalizeSource) || [];
+                let routingParsed = routing && parseRoutingData(routing.data);
+                let sources = routingParsed && routingParsed.sources.map(normalizeSource) || [];
                 let seenNames = new Set<string>();
                 for (let source of sources) {
                     if (source.type !== "remote") continue;