querysub 0.13.0 → 0.15.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.13.0",
+    "version": "0.15.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
         "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
         "pako": "^2.1.0",
         "preact": "^10.11.3",
-        "socket-function": "^0.34.0",
+        "socket-function": "^0.36.0",
         "terser": "^5.31.0",
         "typesafecss": "^0.6.3",
         "yaml": "^2.5.0",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -2,7 +2,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getArchives } from "../-a-archives/archives";
 import { getDomain, isDevDebugbreak, isNoNetwork } from "../config";
 import { measureWrap } from "socket-function/src/profiling/measure";
-import { isNode, sha256Hash, timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { isNode, sha256Hash, throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { errorToUndefined, errorToUndefinedSilent, ignoreErrors, logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
@@ -21,9 +21,9 @@ import { getPublicIP } from "../misc/networking";
 
 import dns from "dns/promises";
 import { isDefined } from "../misc";
-import { diskLog } from "../diagnostics/logs/diskLogger";
+import { diskLog, noDiskLogPrefix } from "../diagnostics/logs/diskLogger";
 
-let HEARTBEAT_INTERVAL = timeInMinute * 5;
+let HEARTBEAT_INTERVAL = timeInMinute * 2;
 // Interval which we check other heartbeats
 let CHECK_INTERVAL = HEARTBEAT_INTERVAL;
 // If the heartbeat is older than thing, it fails the dead check
@@ -37,6 +37,8 @@ let SUICIDE_HEARTBEAT_THRESHOLD = timeInMinute * 15;
 
 let CLIENTSIDE_POLL_RATE = timeInMinute * 5;
 
+let API_AUDIT_RATE = timeInSecond * 5;
+
 let MEMORY_AUDIT_RATE = timeInMinute * 5;
 let MEMORY_AUDIT_COUNT = 3;
 
@@ -64,6 +66,9 @@ export function getOwnNodeId(): string {
     }
     return nodeId;
 }
+export function getMountNodeId(): string | undefined | "" {
+    return SocketFunction.mountedNodeId;
+}
 export function getOwnNodeIdAssert(): string {
     let nodeId = SocketFunction.mountedNodeId;
     if (!nodeId) {
@@ -154,19 +159,8 @@ function addNodeIdBase(nodeId: string) {
     allNodeIds2.add(nodeId);
     onNodesChanged();
 }
-function removeNodeIds(nodeIds: string[]) {
-    diskLog("removeNodeIds", { nodeIds });
-    nodeIds = nodeIds.filter(nodeId => allNodeIds2.has(nodeId));
-    if (nodeIds.length === 0) return;
-    for (let nodeId of nodeIds) {
-        if (logging) {
-            console.log(red(`Removed node directly ${nodeId}`));
-        }
-        allNodeIds2.delete(nodeId);
-    }
-    onNodesChanged();
-}
 function setNodeIds(nodeIds: string[]) {
+    nodeIds = nodeIds.filter(x => x !== SPECIAL_NODE_ID_FOR_UNMOUNTED_NODE);
     diskLog("setNodeIds", { nodeIds });
     // Also try all localhost ports, if we are in dev mode
     if (isNode() && isDevDebugbreak()) {
@@ -250,25 +244,18 @@ async function syncArchives() {
 async function runHeartbeatAuditLoop() {
     await onNodeDiscoveryReady();
     let deadCount = new Map<string, number>();
-    await runInfinitePollCallAtStart(CHECK_INTERVAL, async () => {
+    // 90% of the normal interval, so we don't run at the same tmie as the other audit
+    await runInfinitePollCallAtStart(CHECK_INTERVAL * 0.9, async () => {
         if (shutdown) return;
         // Wait a bit longer, to try to prevent all nodes from synchronizing their audit times.
-        await delay(HEARTBEAT_INTERVAL * Math.random() * 0.1);
+        await delay(CHECK_INTERVAL * Math.random() * 0.1);
         //console.log(magenta(`Auditing node list`));
 
         let deadTime = Date.now() - DEAD_THRESHOLD;
         let nodeIds = await archives().find("");
-        // Should be unusual, but... if we find new nodes, add them, and let everyone else know?
-        for (let nodeId of nodeIds) {
-            if (!allNodeIds2.has(nodeId)) {
-                console.log(yellow(`Found unexpected node, adding it and broadcasting it, ${nodeId}`));
-                addNodeId(nodeId);
-                for (let nodeId of allNodeIds2) {
-                    if (isOwnNodeId(nodeId)) continue;
-                    ignoreErrors(NodeDiscoveryController.nodes[nodeId].addNode(nodeId));
-                }
-            }
-        }
+        // We spent the money checking the node list, so we might as well update it
+        setNodeIds(nodeIds);
+
         let removedNodeIds: string[] = [];
         for (let nodeId of nodeIds) {
             let lastTime = Number((await archives().get(nodeId))?.toString()) || 0;
@@ -291,62 +278,45 @@ async function runHeartbeatAuditLoop() {
             }
         }
 
-        removeNodeIds(removedNodeIds);
-
-        // Broadcast to all nodes the removal
-        for (let nodeId of allNodeIds2) {
-            if (isOwnNodeId(nodeId)) continue;
-            ignoreErrors(NodeDiscoveryController.nodes[nodeId].removeNodes(removedNodeIds));
+        if (removedNodeIds.length > 0) {
+            void tellEveryoneNodesChanges();
         }
     });
 }
 
-async function isOutOfSync() {
-    let auditNodes = shuffle(Array.from(allNodeIds2), Date.now()).slice(0, MEMORY_AUDIT_COUNT);
-    for (let nodeId of auditNodes) {
-        if (nodeId === getOwnNodeId()) continue;
-
-        let stableWait = timeInSecond * 30;
-        while (true) {
-            let hash = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].getAllNodesHash());
-            if (!hash) break;
-            let ownHash = getAllNodesHash();
-            if (hash === ownHash) break;
-            await delay(stableWait);
-            let hash2 = await errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].getAllNodesHash());
-            if (!hash2) break;
-            if (hash === hash2) {
-                console.log(yellow(`Node discovery is out of sync, syncing with disk. Told by node ${nodeId}, our hash of ${ownHash} is wrong and should be ${hash2}`));
-                return true;
-            }
-            // Otherwise, the hash changed, so... we maybe we just caught it while it was changing.
-            //  Wait less time, to try to catch up with the rate of change
-            stableWait /= 2;
-            if (stableWait < timeInSecond) {
-                // It is changing so fast we better just sync with the disk
-                console.log(yellow(`Node discovery is changing fast and likely out of sync, resyncing with disk. Told by node ${nodeId}`));
-                return true;
-            }
+async function fastMemorySync(retries = 3) {
+    let checkNode = shuffle(Array.from(allNodeIds2).filter(x => x !== getOurNodeId()), Date.now()).slice(0, MEMORY_AUDIT_COUNT).at(0);
+    if (!checkNode) return;
+    let otherNodes = await errorToUndefinedSilent(NodeDiscoveryController.nodes[checkNode].getAllNodeIds());
+    if (!otherNodes) {
+        if (retries > 0) {
+            await fastMemorySync(retries - 1);
         }
+        return;
+    }
+    // If they are missing nodes that's fine. We constantly have extra nodes, and have to function correctly
+    //  with extra nodes. However, if we are missing nodes, we'd prefer to have them quickly, so we should
+    //  sync now.
+    let missingNodes = otherNodes.filter(nodeId => !allNodeIds2.has(nodeId));
+    if (missingNodes.length > 0) {
+        console.log(yellow(`Node list is missing nodes, resyncing node`), { checkNode, missingNodes, otherNodes });
+        await syncArchives();
     }
 }
 
 async function runMemoryAuditLoop() {
     await onNodeDiscoveryReady();
-    runInfinitePoll(MEMORY_AUDIT_RATE, async () => {
-        // In retrospec, reading all nodes every 5 minutes is fine. It costs about 0.035 USD
-        //  per month to do this, which... is a lot less than the server to run this will cost!
-        //if (await isOutOfSync()) {
-        await syncArchives();
-        //}
-    });
+    runInfinitePoll(MEMORY_AUDIT_RATE, syncArchives);
+    runInfinitePoll(API_AUDIT_RATE, fastMemorySync);
 }
 
 async function writeHeartbeat() {
     if (shutdown) return;
     let now = Date.now();
     console.log(green(`Writing heartbeat ${formatDateTime(now)}`));
-    await archives().set(getOwnNodeId(), Buffer.from(now + ""));
+    let nodeId = getMountNodeId();
+    if (!nodeId) return;
+    await archives().set(nodeId, Buffer.from(now + ""));
 }
 
 async function runMainSyncLoops(discoveryReady: PromiseObj<void>) {
@@ -491,23 +461,22 @@ export async function nodeDiscoveryShutdown() {
     if (isServer()) {
         await archives().del(getOwnNodeId());
     }
-    // Don't bother to remove the node from ourself, as we will be shutting down shortly
-    //  (but do bother to tell other nodes!)
-    await Promise.allSettled(Array.from(allNodeIds2).map(async nodeId => {
-        if (isOwnNodeId(nodeId)) return;
-        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].removeNodes([getOwnNodeId()])));
-    }));
+    void tellEveryoneNodesChanges();
 }
+const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNodesChanges() {
+    for (let nodeId of allNodeIds2) {
+        if (isOwnNodeId(nodeId)) continue;
+        ignoreErrors(NodeDiscoveryController.nodes[nodeId].resyncNodes());
+    }
+});
 
 
 class NodeDiscoveryControllerBase {
     public async addNode(nodeId: string) {
         addNodeId(nodeId);
     }
-    public async removeNodes(nodeIds: string[]) {
-        let callerId = SocketFunction.getCaller().nodeId;
-        console.log(`remote removeNode request`, { callerId, nodeIds });
-        removeNodeIds(nodeIds);
+    public async resyncNodes() {
+        await syncArchives();
     }
     public async getAllNodesHash(): Promise<string> {
         return getAllNodesHash();
@@ -529,7 +498,7 @@ const NodeDiscoveryController = SocketFunction.register(
     new NodeDiscoveryControllerBase(),
     () => ({
         addNode: { hooks: [requiresNetworkTrustHook] },
-        removeNodes: { hooks: [requiresNetworkTrustHook] },
+        resyncNodes: { hooks: [requiresNetworkTrustHook] },
         getAllNodesHash: { hooks: [requiresNetworkTrustHook] },
         // Skip client hooks, so we don't block on authentication (IdentityController), as some of these functions
         //  are needed for authentication to finish!

package/src/-g-core-values/NodeCapabilities.ts

@@ -4,7 +4,7 @@
 
 import { SocketFunction } from "socket-function/SocketFunction";
 import { SocketRegistered } from "socket-function/SocketFunctionTypes";
-import { errorToUndefined, errorToUndefinedSilent, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
+import { errorToUndefined, errorToUndefinedSilent, timeoutToUndefinedSilent } from "../errors";
 import { getAllNodeIds } from "../-f-node-discovery/NodeDiscovery";
 import { green, red, yellow } from "socket-function/src/formatting/logColors";
 import { shuffle } from "../misc/random";

package/src/0-path-value-core/PathValueCommitter.ts

@@ -55,7 +55,6 @@ class PathValueCommitter {
     // Returns true if all the writes are accepted by some node (this should almost always be the case)
     public commitValues(values: PathValue[], predictWrites: "predictWrites" | undefined): void {
         if (values.length === 0) return;
-
         ActionsHistory.OnWrite(values);
 
         let now = Date.now();

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -23,7 +23,7 @@ import { LOCAL_DOMAIN_PATH } from "../0-path-value-core/NodePathAuthorities";
 import { registerPeriodic } from "../diagnostics/periodic";
 import { remoteWatcher } from "../1-path-client/RemoteWatcher";
 import { Schema2, Schema2Fncs } from "./schema2";
-import { getDomain } from "../config";
+import { devDebugbreak, getDomain } from "../config";
 
 import type { CallSpec } from "../3-path-functions/PathFunctionRunner";
 import type { FunctionMetadata } from "../3-path-functions/syncSchema";
@@ -597,15 +597,6 @@ export class PathValueProxyWatcher {
             throw new Error(`Tried to write a non-local path in a "noLocks" watcher, ${watcher.debugName}, path ${pathStr}`);
         }
 
-        if (!pathStr.startsWith(LOCAL_DOMAIN_PATH)) {
-            // Copy the value, to ensure any proxy values aren't attempted to be written
-            //      to the database. A bit slower, but... it should be fine. We COULD do this
-            //      later on, but this makes it easier to attribute lag and errors to the original source.
-            //  - If we had a proxy, we WOULD copy it eventually, but we would copy it later,
-            //      which would cause an error, because the reader wouldn't be trackable.
-            value = deepCloneCborx(value);
-        }
-
         if (watcher.permissionsChecker) {
             if (!watcher.permissionsChecker.checkPermissions(pathStr).allowed) {
                 if (value !== specialObjectWriteValue) {
@@ -713,6 +704,14 @@ export class PathValueProxyWatcher {
             };
             addWrites(pathStr, value);
         } else {
+            if (!pathStr.startsWith(LOCAL_DOMAIN_PATH)) {
+                // Copy the value, to ensure any proxy values aren't attempted to be written
+                //      to the database. A bit slower, but... it should be fine. We COULD do this
+                //      later on, but this makes it easier to attribute lag and errors to the original source.
+                //  - If we had a proxy, we WOULD copy it eventually, but we would copy it later,
+                //      which would cause an error, because the reader wouldn't be trackable.
+                value = deepCloneCborx(value);
+            }
             watcher.pendingWrites.set(pathStr, value);
         }
     };

package/src/diagnostics/NodeViewer.tsx

@@ -32,8 +32,11 @@ import { setRecord } from "../-b-authorities/dnsAuthority";
 import tls from "tls";
 import net from "net";
 import { ButtonSelector } from "../library-components/ButtonSelector";
-import { assertIsManagementUser } from "./managementPages";
+import { assertIsManagementUser, managementPageURL } from "./managementPages";
 import { SocketRegistered } from "socket-function/SocketFunctionTypes";
+import { ATag } from "../library-components/ATag";
+import { filterURL, selectedNodeId } from "./logs/DiskLoggerPage";
+import { getSyncedController } from "../library-components/SyncedController";
 
 
 type NodeData = {
@@ -162,7 +165,7 @@ export class NodeViewer extends qreact.Component {
 
         let tables: { [key: string]: NodeData[] } = {};
         tables["PathValueServer"] = [];
-        tables["FunctionRunner"] = [];
+        tables["querysub-function"] = [];
         tables["Querysub"] = [];
         for (let datum of nodeDatas) {
             if (datum.live_authorityPaths?.length) {
@@ -170,7 +173,7 @@ export class NodeViewer extends qreact.Component {
             } else if (datum.live_exposedControllers?.includes("QuerysubController-6db5ef05-7563-4473-a440-2f64f03fe6ef")) {
                 tables["Querysub"].push(datum);
             } else if (datum.live_entryPoint?.endsWith("function.js")) {
-                tables["FunctionRunner"].push(datum);
+                tables["querysub-function"].push(datum);
             } else {
                 let entryName = datum.live_entryPoint?.replaceAll("\\", "/").split("/").pop() || "Unknown";
                 tables[entryName] = tables[entryName] || [];
@@ -179,7 +182,7 @@ export class NodeViewer extends qreact.Component {
         }
 
         let builtinGroups = {
-            "Default": ["buttons", "devToolsURL", "ip", "uptime", "Heap", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
+            "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "Heap", "All Memory", "Blocking Lag", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
         };
         // Column => group
         let builtInGroupsLookup = new Map<string, string>();
@@ -256,6 +259,23 @@ export class NodeViewer extends qreact.Component {
                                         );
                                     },
                                 },
+                                nodeId: {
+                                    formatter: (obj) => {
+                                        let str = String(obj);
+                                        if (str.startsWith("http")) return formatValue(obj);
+                                        https://noproxy.querysub.com:1111/?page=test&showingmanagement&managementpage=DiskLoggerPage&nodeId=b4d19fba5f79d48c3.b68f46ffaffad0636.querysub.com%3A41863&filter=%22%5C%22__nodeId%5C%22%3A%5C%22b654f522bf3a67c49.b68f46ffaffad0636.querysub.com%3A33473%5C%22%22
+
+                                        return (
+                                            <ATag values={[
+                                                managementPageURL.getOverride("DiskLoggerPage"),
+                                                selectedNodeId.getOverride(str),
+                                                filterURL.getOverride(`"__nodeId":"${str}"`),
+                                            ]}>
+                                                Logs
+                                            </ATag>
+                                        );
+                                    },
+                                },
                                 ip: {},
                                 ...x.table?.columns,
                                 //capabilities: null,
@@ -500,4 +520,6 @@ export const NodeViewerController = SocketFunction.register(
         // Auto expose, for getMiscInfo. This is protected via assertIsManagementUser anyways.
         //noAutoExpose: true,
     }
-);
+);
+
+export let nodeViewerController = getSyncedController(NodeViewerController);

package/src/diagnostics/logs/DiskLoggerPage.tsx

@@ -18,7 +18,7 @@ import { LogType } from "../errorLogs/ErrorLogCore";
 import { canHaveChildren } from "socket-function/src/types";
 import { measureBlock } from "socket-function/src/profiling/measure";
 import { binarySearchBasic, list, sort, timeInDay, timeInHour } from "socket-function/src/misc";
-import { NodeViewerController } from "../NodeViewer";
+import { NodeViewerController, nodeViewerController } from "../NodeViewer";
 import { red } from "socket-function/src/formatting/logColors";
 import { errorMessage, formatValue, genericFormat, warnMessage } from "../../5-diagnostics/GenericFormat";
 import { hslToRGB } from "socket-function/src/formatting/colors";
@@ -51,9 +51,9 @@ import { TimeRangeSelector } from "../../library-components/TimeRangeSelector";
 module.hotreload = true;
 module.hotreload = true;
 
-let selectedNodeId = new URLParam("nodeId", "");
+export let selectedNodeId = new URLParam("nodeId", "");
 
-let filterURL = new URLParam("filter", "");
+export let filterURL = new URLParam("filter", "");
 let sortOldestFirst = new URLParam("oldestFirst", false);
 let selectedFields = new URLParam("selectedFields", "");
 
@@ -426,12 +426,7 @@ const processLogs = cacheShallowConfigArgEqual((config: {
                 return true;
             })).filter(x => x.length > 0);
             function logMatches(filter: string, log: LogObj) {
-                for (let [key, value] of Object.entries(log)) {
-                    if (key.toLowerCase().includes(filter)) return true;
-                    if (canHaveChildren(value) && JSON.stringify(value).toLowerCase().includes(filter)) return true;
-                    if (String(value).toLowerCase().includes(filter)) return true;
-                }
-                return false;
+                return JSON.stringify(log).toLowerCase().includes(filter);
             }
             logMatchesFilter = function logMatchesFilter(log: LogObj) {
                 return (
@@ -567,6 +562,4 @@ export const DiskLoggerController = SocketFunction.register(
     }
 );
 
-let diskLoggerController = getSyncedController(DiskLoggerController);
-
-let nodeViewerController = getSyncedController(NodeViewerController);
+let diskLoggerController = getSyncedController(DiskLoggerController);