querysub 0.415.0 → 0.416.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.415.0",
+  "version": "0.416.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -62,7 +62,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "querysub": "^0.357.0",
-    "socket-function": "^1.1.18",
+    "socket-function": "^1.1.19",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -2,7 +2,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getArchives } from "../-a-archives/archives";
 import { getDomain, isDevDebugbreak, isNoNetwork, isPublic } from "../config";
 import { measureBlock } from "socket-function/src/profiling/measure";
-import { isNode, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { isNode, keyByArray, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToError, timeoutToUndefinedSilent } from "../errors";
 import { ensureWeAreTrusted, requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -22,6 +22,7 @@ import { EdgeNodeConfig } from "../4-deploy/edgeNodes";
 import * as certs from "../-a-auth/certs";
 import { logDisk } from "../diagnostics/logs/diskLogger";
 import { MaybePromise } from "socket-function/src/types";
+import { getPathStr2 } from "../path";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 15;
 // Interval which we check other heartbeats
@@ -196,10 +197,6 @@ function addNodeIdBase(nodeId: string) {
 }
 async function setNodeIds(nodeIds: string[]) {
     nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
-    if (isNode()) {
-        await Promise.allSettled(nodeIds.map(checkWrongServerNodeId));
-        nodeIds = nodeIds.filter(nodeId => !wrongServerNodeIds.has(nodeId));
-    }
 
     console.info("setNodeIds", { nodeIds });
     let newNodeIds = nodeIds.filter(nodeId => !allNodeIds2.has(nodeId));
@@ -297,53 +294,37 @@ export async function triggerNodeChange() {
     }));
 }
 
-// If we can connect on the same port, but it has a different thread ID, it means the old thread ID is gone. We're never going to go back to an old thread ID, and we can't have two threads on the same port. 
-let wrongServerNodeIds = new Set<string>();
-let checkWrongServerNodeId = cache(async (nodeId: string) => {
-    if (wrongServerNodeIds.has(nodeId)) return;
-    let callFactory = await timeoutToUndefinedSilent(timeInSecond * 5, Promise.resolve(getCreateCallFactory(nodeId)));
-    if (!callFactory) {
-        if (SocketFunction.logMessages) {
-            console.log(`Did not find call factory for ${nodeId}`);
-        }
-        // Clear it right away, so we can check for it being alive quickly.
-        checkWrongServerNodeId.clear(nodeId);
-        return;
-    }
-    if (callFactory) {
-        // Not great, but... this should work well enough.
-        for (let i = 0; i < 10; i++) {
-            if (callFactory.receivedInitializeState) break;
-            await delay(500);
-        }
-        if (!callFactory.receivedInitializeState && SocketFunction.logMessages) {
-            console.log(`Did not receive initialize state from ${nodeId}`);
-        }
-    } else {
-        if (SocketFunction.logMessages) {
-            console.log(`Did not find call factory for ${nodeId}`);
-        }
+
+async function clearDeadThreadsFromArchives() {
+    let nodes = await archives().find("");
+
+    function getPortHash(nodeId: string) {
+        let obj = decodeNodeId(nodeId);
+        if (!obj) return undefined;
+        return getPathStr2(obj.machineId, obj.port + "");
     }
-    if (callFactory && callFactory.realNodeId && callFactory.realNodeId !== nodeId) {
-        if (SocketFunction.logMessages) {
-            console.log(red(`Found dead thread, disconnecting node and deleting from archives ${nodeId}`));
-        }
-        wrongServerNodeIds.add(nodeId);
-        callFactory?.disconnect();
-        // Dead threads never come back, so this should be safe to do. 
-        await archives().del(nodeId);
-        // Return, so we don't clear this.
-        return;
-    } else {
-        if (SocketFunction.logMessages) {
-            console.log(green(`Found live thread, node ${nodeId}, real node id ${callFactory?.realNodeId}`));
+    let byPortHash = keyByArray(nodes, getPortHash);
+    for (let [portHash, nodeIds] of Array.from(byPortHash)) {
+        if (!portHash) continue;
+        let aliveNodeId = "";
+        await Promise.all(nodeIds.map(async nodeId => {
+            let alive = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].isAlive());
+            if (alive) {
+                aliveNodeId = nodeId;
+            }
+        }));
+        if (aliveNodeId) {
+            let deadThreads = nodeIds.filter(nodeId => nodeId !== aliveNodeId);
+            await Promise.all(deadThreads.map(async deadNodeId => {
+                console.log(`Removing dead thread. We contacted a node on the same port and same machine (${aliveNodeId}), which means the port has been reused by another thread, which proves that the old thread has died, as otherwise the new thread would not be able to use it.`);
+                await archives().del(deadNodeId);
+            }));
         }
     }
 
-    setTimeout(() => {
-        checkWrongServerNodeId.clear(nodeId);
-    }, timeInMinute * 5);
-});
+    return nodes;
+
+}
 
 async function syncArchives() {
     if (isServer()) {
@@ -381,7 +362,8 @@ async function runHeartbeatAuditLoop() {
         //console.log(magenta(`Auditing node list`));
 
         let deadTime = Date.now() - DEAD_THRESHOLD;
-        let nodeIds = await archives().find("");
+
+        let nodeIds = await clearDeadThreadsFromArchives();
         // We spent the money checking the node list, so we might as well update it
         await setNodeIds(nodeIds);
 
@@ -451,7 +433,6 @@ async function fastMemorySync() {
     //  with extra nodes. However, if we are missing nodes, we'd prefer to have them quickly, so we should
     //  sync now.
     let missingNodes = otherNodes.filter(nodeId => !allNodeIds2.has(nodeId));
-    missingNodes = missingNodes.filter(nodeId => !wrongServerNodeIds.has(nodeId));
     if (missingNodes.length > 0) {
         console.log(yellow(`Node list is missing nodes, resyncing node`), { missingNodes, otherNodes });
         await syncArchives();
@@ -614,6 +595,9 @@ const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNod
 
 
 class NodeDiscoveryControllerBase {
+    public async isAlive() {
+        return true;
+    }
     public async addNode(nodeId: string) {
         console.log(magenta(`Received addNode`), { nodeId });
         addNodeId(nodeId);
@@ -642,7 +626,7 @@ const NodeDiscoveryController = SocketFunction.register(
     "NodeDiscoveryController-7991037e-fd9e-4085-b1db-52035487e72c",
     new NodeDiscoveryControllerBase(),
     () => ({
-        getOwnNodeId: { noClientHooks: true, noDefaultHooks: true },
+        isAlive: {},
         addNode: { hooks: [requiresNetworkTrustHook] },
         resyncNodes: { hooks: [requiresNetworkTrustHook] },
         getAllNodesHash: { hooks: [requiresNetworkTrustHook] },

package/src/3-path-functions/syncSchema.ts

@@ -581,6 +581,11 @@ function createObjectAliveChecker(data: () => any, path: SchemaPath, delay: numb
     });
 }
 
+let querysub: typeof import("../4-querysub/Querysub") | undefined = undefined;
+setImmediate(async () => {
+    querysub = await import("../4-querysub/Querysub");
+});
+
 let callIdOverrideFncs: Map<string, Map<string, {
     prefix: string;
     getKey: (...args: unknown[]) => string;
@@ -591,11 +596,22 @@ export function getCallIdOverride(config: {
     callId: string;
     args: unknown[];
 }): string {
-    let def = callIdOverrideFncs.get(config.moduleId)?.get(config.functionId);
-    if (!def) return config.callId;
-    return createRoutingOverrideKey({
-        remappedPrefix: def.prefix,
-        originalKey: config.callId,
-        routeKey: def.getKey(...config.args),
-    });
+    try {
+        if (querysub) {
+            if (!querysub.Querysub.isInSyncedCall()) {
+                // NOTE: This is wrong and it will result in loading user being used as a key sometimes. However, it should be fine, as if the call id override is wrong, it shouldn't break anything. 
+                return querysub.Querysub.localRead(() => getCallIdOverride(config));
+            }
+        }
+        let def = callIdOverrideFncs.get(config.moduleId)?.get(config.functionId);
+        if (!def) return config.callId;
+        return createRoutingOverrideKey({
+            remappedPrefix: def.prefix,
+            originalKey: config.callId,
+            routeKey: def.getKey(...config.args),
+        });
+    } catch (e: any) {
+        console.warn(`Error getting call id override for ${config.moduleId}.${config.functionId}, falling back to original call id`, { error: e.stack });
+        return config.callId;
+    }
 }

package/src/diagnostics/SyncTestPage.tsx

@@ -150,6 +150,7 @@ export class SyncTestPage extends qreact.Component {
                                             hasError = true;
                                         }
                                         if (hasError) return undefined;
+                                        if (!result?.time) return undefined;
                                         return (
                                             <div className={css.hbox(5).hsl(0, 0, 100).pad2(2)} title={JSON.stringify(thread)}>
                                                 <div>{getUniqueThreadName(thread, allThreads)}</div>

package/tempnotes.txt

@@ -2,60 +2,23 @@
 
 Local CYOA + Local FunctionRunner works
 
+1) Deploy
+2) Verify this works
 
-Wait... Didn't we restart everything? Why does it seem like the function runner is still watching the same things? 
-    "00020e3a3b543382.a794fbcf7b104c68.querysubtest.com:37381"
-
-
-
-Hmm... we ask to watch it, but the watch doesn't show up on the other side? Fuck...
-
-Hmm, it's still failing in the same way. The function runner just has, like, no watches. It tried to watch I assume the same thing as before. Nothing showed up on the other end. 
-
-
-
-
-So we have the remote watcher, so it looks like we were watching it, but we don't think it's synced, so we clearly didn't get a value back. 
-
-WATCHER FAILED TO SYNC findFunctionsToCall DID NOT RECEIVE PATH VALUES. This means PathValueServer is not responding to watches, either to specific paths, or for all paths [
-  '.,querysubtest._com.,',
-  '.,querysubtest._com.,PathFunctionRunner.,'
-] [ '.,querysubtest._com.,PathFunctionRunner.,' ] [Function: findFunctionsToCall]
-Node list is missing nodes, resyncing node {
-  missingNodes: [ '716181d6570f7b55.a794fbcf7b104c68.querysubtest.com:34689' ],
-
-
-UGH... poking fixed it. WHICH MEANS, we have to break it again, and get into the bad state again!
-    - Ugh...
-
--1) We got into a bad state where The query sub nodes weren't really listening or it's it th they had zombie watchers that hadn't finished synchronizing, but it wasn't complaining about being able to un to find any any paths. There were no unwatched paths 
-    - It fixed when we poked them. 
-
-0) If it fails after redeploying, disable the server and run everything locally again, as we made a lot of changes, and so it might not even work locally. 
+1) verify this warning is no longer in our logs "Node list is missing nodes, resyncing node"
+    - We should check the logs. I'm assuming what happened is that at some point some server thought it was the wrong node ID and deleted it?
+    - It's generally the same ones, except new ones get added. 
 
--1) Get a test script that doesn't even use the function runner working. 
 
-0) Get our front end working. I guess we need to run the function runner locally, and then if that works, we need to figure out why the remote function runner isn't working. 
-    - Also, see if we run the local query subserver and local function runner, if those writes can be seen remotely. If it's a remote function runner problem but not a remote path value server problem, we should see the writes. 
+-1) A lot of API calls are getting locked up and never finishing. We need to fix this. What the fuck is happening? It happens a lot with audio calls, as we make a lot of audio calls, but we can probably replicate it with non-audio calls. I think it happened with some embeddings, although we did create a lot of embeddings, and it only happened at the end. If it's only an audio thing, that's different, but we still need some kind of timeout. But I don't think it's an audio thing, I think it's not actually starting. 
 
--1) Fix the remote servers not being able to send values to our local path value server. I don't know why it wouldn't work... They should have changed the name knowing our local name. 
 
-1) Use connection page to verify server can talk to our locally hosted server
-1.1) Verify by breaking into our local server that we are receiving values written on the remote server in the local server. 
-    - This is extremely important. Without this, we can't run a local server. There's all kinds of issues, but our socket function changes should make this so this just works. 
 
 0) SHARD THE FUNCTION RUNNER!
     - And secondary sharding for backup...
     - First shard locally
     - Test on our sync test page (Otherwise all the writes are going to go to the same function runner anyway, so it's not a good test. 
 
--2) Errors are not being automatically grouped. 
-    - We are getting notifications though, which is weird. The notifications are supposed to be automatically grouping it. We should probably look at its logs and see why. 
-
-0) We just are constantly "Node list is missing nodes, resyncing node"
-    - We should check the logs. I'm assuming what happened is that at some point some server thought it was the wrong node ID and deleted it?
-    - It's generally the same ones, except new ones get added. 
-
 I think even if we run into some occasional issues, we should just power through and try to fix them later. Because I'm sick of working on the framework...
 
 MONTHLY SUMMARY!