querysub 0.303.0 → 0.305.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.303.0",
+  "version": "0.305.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -22,7 +22,7 @@
     "js-sha512": "^0.9.0",
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
-    "socket-function": "^0.134.0",
+    "socket-function": "^0.135.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.22.0",
     "yaml": "^2.5.0",

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -19,6 +19,7 @@ import { decodeNodeId, decodeNodeIdAssert } from "../-a-auth/certs";
 import { isDefined } from "../misc";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { getBootedEdgeNode } from "../-0-hooks/hooks";
+import { EdgeNodeConfig } from "../4-deploy/edgeNodes";
 
 let HEARTBEAT_INTERVAL = timeInMinute * 15;
 // Interval which we check other heartbeats
@@ -220,6 +221,47 @@ export function configRootDiscoveryLocation(config: {
 }) {
     rootDiscoveryNodeId = getNodeId(config.domain, config.port);
 }
+
+export function updateRootDiscoveryLocation(edgeNode: EdgeNodeConfig) {
+    let prevRootId = getBrowserUrlNode();
+    if (!isClient()) {
+        throw new Error(`updateRootDiscoveryLocation should only be called in the browser. If not a client, you should be able to discovery new nodes automatically`);
+    }
+    let nodeId = edgeNode.host;
+    let nodeCache = Object.entries(require.cache);
+    for (let [path, module] of nodeCache) {
+        if (!module) continue;
+        if (!path.includes(prevRootId)) continue;
+        let newPath = path.replace(prevRootId, nodeId);
+        // Remove the old module
+        delete require.cache[path];
+        // Add the new module
+        require.cache[newPath] = module;
+        module.filename = module.filename.replace(prevRootId, nodeId);
+        module.id = module.id.replace(prevRootId, nodeId);
+        if (module.original) {
+            module.original.filename = module.original.filename.replace(prevRootId, nodeId);
+            module.original.originalId = module.original.originalId.replace(prevRootId, nodeId);
+            for (let [key, value] of Object.entries(module.original.requests)) {
+                if (key.includes(prevRootId)) {
+                    let newKey = key.replace(prevRootId, nodeId);
+                    delete module.original.requests[key];
+                    module.original.requests[newKey] = value;
+                    key = newKey;
+                }
+                if (value.includes(prevRootId)) {
+                    value = value.replace(prevRootId, nodeId);
+                    module.original.requests[key] = value;
+                }
+            }
+        }
+    }
+    globalThis.BOOTED_EDGE_NODE = edgeNode;
+    rootDiscoveryNodeId = nodeId;
+    addNodeId(nodeId);
+}
+(globalThis as any).updateRootDiscoveryLocation = updateRootDiscoveryLocation;
+
 /** NOTE: Can also be called serverside, if configRootDiscoveryLocation is called (otherwise can always be called clientside). */
 export function getBrowserUrlNode() {
     if (!isClient()) throw new Error(`getBrowserUrlNode can only be called when isClient()`);
@@ -472,6 +514,7 @@ if (isServer()) {
     }
 }
 
+
 export async function forceRemoveNode(nodeId: string) {
     await archives().del(nodeId);
     void tellEveryoneNodesChanges();
@@ -488,6 +531,7 @@ export async function nodeDiscoveryShutdown() {
     void tellEveryoneNodesChanges();
 }
 const tellEveryoneNodesChanges = throttleFunction(1000, function tellEveryoneNodesChanges() {
+    if (isClient()) return;
     for (let nodeId of allNodeIds2) {
         if (isOwnNodeId(nodeId)) continue;
         ignoreErrors(NodeDiscoveryController.nodes[nodeId].resyncNodes());

package/src/-g-core-values/NodeCapabilities.ts

@@ -19,6 +19,8 @@ import { hackDevtoolsWebsocketForward } from "./oneTimeForward";
 
 let loadTime = Date.now();
 
+let controllerNodeIdCache = new Map<string, string | Promise<string | undefined>>();
+
 // NOTE: If this becomes slow (because we are just trying all servers), we could start to store capabilities
 //  in PersistedStorage. However, I don't think it will be a problem for quite some time.
 export async function getControllerNodeId(
@@ -27,26 +29,42 @@ export async function getControllerNodeId(
     retryCount = 1,
     initialTime = Date.now(),
 ): Promise<string | undefined> {
-    let nodeIdsToTest = await getAllNodeIds();
-    // Shuffle, so we aren't always using the same node!
-    nodeIdsToTest = shuffle(nodeIdsToTest, Date.now());
-    for (let nodeId of nodeIdsToTest) {
-        if (await doesNodeExposeController(nodeId, controller)) {
-            if (!quiet) {
-                let duration = Date.now() - initialTime;
-                console.log(green(`Resolved capability ${controller._classGuid} with node ${nodeId}, in ${formatTime(duration)}`));
-            }
-            return nodeId;
-        }
+    let cached = controllerNodeIdCache.get(controller._classGuid);
+    if (cached && typeof cached !== "string") {
+        cached = await cached;
     }
-    if (retryCount > 0) {
-        await delay(2000);
-        return getControllerNodeId(controller, quiet, retryCount - 1, initialTime);
+    if (cached && !SocketFunction.isNodeConnected(cached)) {
+        controllerNodeIdCache.delete(controller._classGuid);
+        cached = undefined;
     }
-    if (!quiet) {
-        console.warn(yellow(`Could not find a node that exposes controller ${controller._classGuid}, tried: ${JSON.stringify(nodeIdsToTest)}`));
+    if (cached) return cached;
+    let promise = getInternal();
+    controllerNodeIdCache.set(controller._classGuid, promise);
+    return promise;
+
+    async function getInternal() {
+
+        let nodeIdsToTest = await getAllNodeIds();
+        // Shuffle, so we aren't always using the same node!
+        nodeIdsToTest = shuffle(nodeIdsToTest, Date.now());
+        for (let nodeId of nodeIdsToTest) {
+            if (await doesNodeExposeController(nodeId, controller)) {
+                if (!quiet) {
+                    let duration = Date.now() - initialTime;
+                    console.log(green(`Resolved capability ${controller._classGuid} with node ${nodeId}, in ${formatTime(duration)}`));
+                }
+                return nodeId;
+            }
+        }
+        if (retryCount > 0) {
+            await delay(2000);
+            return getControllerNodeId(controller, quiet, retryCount - 1, initialTime);
+        }
+        if (!quiet) {
+            console.warn(yellow(`Could not find a node that exposes controller ${controller._classGuid}, tried: ${JSON.stringify(nodeIdsToTest)}`));
+        }
+        return undefined;
     }
-    return undefined;
 }
 
 export async function getControllerNodeIdList(

package/src/0-path-value-core/NodePathAuthorities.ts

@@ -20,7 +20,7 @@ import { formatNumber, formatTime } from "socket-function/src/formatting/format"
 import { cache, cacheLimited } from "socket-function/src/caching";
 import { IdentityController_getCurrentReconnectNodeIdAssert, IdentityController_getReconnectNodeIdAssert } from "../-c-identity/IdentityController";
 import { getBufferFraction, getBufferInt, getShortNumber } from "../bits";
-import { devDebugbreak, isDevDebugbreak } from "../config";
+import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 
@@ -106,17 +106,18 @@ class NodePathAuthorities {
         // NOTE: WHILE calling this function at the module level (because we this is a singleton) is bad,
         //  it SHOULD be fine. "getOwnNodeId" is part of level "-f-node-discovery", and we are level "0-path-value-core".
         //  It should be ready before us, and it should never be allowed to depend on us.
-        let originalOwnId = getOwnNodeId();
+
         let selfOnReady = new PromiseObj();
-        this.authorities.set(getOwnNodeId(), {
-            nodeId: getOwnNodeId(),
-            authorityPaths: [],
-            isReadReady: 0,
-            createTime: this.createTime,
-            self: true,
-            onReady: selfOnReady,
-        });
         if (isServer()) {
+            let originalOwnId = getOwnNodeId();
+            this.authorities.set(getOwnNodeId(), {
+                nodeId: getOwnNodeId(),
+                authorityPaths: [],
+                isReadReady: 0,
+                createTime: this.createTime,
+                self: true,
+                onReady: selfOnReady,
+            });
             void SocketFunction.mountPromise.finally(() => {
                 // Move over to actual nodeId (which only exists after we mount)
                 let newOwnId = getOwnNodeId();
@@ -234,16 +235,17 @@ class NodePathAuthorities {
             nodeIds = nodeIds.filter(nodeId => !isOwnNodeId(nodeId));
             let newNodes = nodeIds.filter(nodeId => !this.authorities.has(nodeId));
 
-            let timeout = first ? POLL_RATE : 0;
+            let doInitialCheck = 0;
+            if (first || this.authorities.size === 0) {
+                doInitialCheck = POLL_RATE;
+            }
             let isCurrentFirst = first;
             first = false;
 
             let promise = Promise.allSettled(newNodes.map(async nodeId => {
                 if (this.authorities.has(nodeId)) return;
 
-                let loc = getNodeIdLocation(nodeId);
-                let ourLoc = getNodeIdLocation(getOwnNodeId());
-                if (loc && loc.address.startsWith("127-0-0-1") && loc.port === ourLoc?.port) {
+                if (isOwnNodeId(nodeId)) {
                     // Ignore it if it's just us, using the special localhost address
                     return;
                 }
@@ -266,7 +268,7 @@ class NodePathAuthorities {
                 console.log(blue(`Identifying ${nodeId} as a path authority. ping latency = ${formatTime(time)}`));
                 this.authorities.set(nodeId, {
                     nodeId,
-                    self: nodeId.startsWith("127-0-0-1."),
+                    self: isOwnNodeId(nodeId),
                     authorityPaths: [],
                     isReadReady: 0,
                     createTime,
@@ -279,14 +281,14 @@ class NodePathAuthorities {
                 });
                 let finished = false;
                 try {
-                    if (timeout) {
+                    if (doInitialCheck) {
                         let timeoutObj = new PromiseObj();
                         setTimeout(() => {
                             timeoutObj.resolve();
-                            if (!finished && timeout) {
-                                console.warn(yellow(`Timeout after ${formatTime(timeout)} while identifying ${nodeId}`));
+                            if (!finished && doInitialCheck) {
+                                console.warn(yellow(`Timeout after ${formatTime(doInitialCheck)} while identifying ${nodeId}`));
                             }
-                        }, timeout);
+                        }, doInitialCheck);
                         await Promise.race([
                             timeoutObj.promise,
                             errorToUndefined(this.forceCheckForReadReady(nodeId)),
@@ -384,7 +386,15 @@ class NodePathAuthorities {
         obj.isReadReady = isReadReady;
         obj.onReady.resolve();
 
-        obj.authorityPaths = await PathController.nodes[nodeId].getAuthorityPaths();
+        if (isClient()) {
+            // If we are in the browser, any node that we can talk to is an edge node, and so presumably owns all paths!
+            //  - Also clients, as... we shouldn't add certs when isClient, even if we are isNode.
+            obj.authorityPaths = [{
+                pathPrefix: getPathStr1(getDomain()),
+            }];
+        } else {
+            obj.authorityPaths = await PathController.nodes[nodeId].getAuthorityPaths();
+        }
         if (obj.authorityPaths.length > 0) {
             console.log(blue(`Identified ${isReadReady ? "(ready)" : yellow("(loading)")} ${nodeId} as a path authority for:`));
             for (let authorityPath of obj.authorityPaths) {
@@ -570,9 +580,6 @@ class NodePathAuthorities {
             );
     }
     public getReadNodes(path: string): string[] {
-        if (isClient()) {
-            return [getBrowserUrlNode()];
-        }
         if (!this.waited) throw new Error(`waitUntilRoutingIsReady must be called before getReadNodes`);
         return this.getReadObjs(path).map(x => x.nodeId);
     }
@@ -631,9 +638,6 @@ class NodePathAuthorities {
         if (this.isSelfAuthority(path)) {
             return getOwnNodeId();
         }
-        if (isClient()) {
-            return getBrowserUrlNode();
-        }
         if (!this.waited) throw new Error(`waitUntilRoutingIsReady must be called before getSingleReadNode`);
         let objs = this.getReadObjs(path);
         return this.pickAuthorityNode(objs)?.nodeId;
@@ -654,9 +658,6 @@ class NodePathAuthorities {
         if (this.isSelfAuthority(path)) {
             return { nodes: [{ nodeId: getOwnNodeId(), range: { start: 0, end: 1 } }] };
         }
-        if (isClient()) {
-            return { nodes: [{ nodeId: getBrowserUrlNode(), range: { start: 0, end: 1 } }] };
-        }
 
         let wildcardPath = appendToPathStr(path, "");
 
@@ -749,9 +750,6 @@ class NodePathAuthorities {
         // IMPORTANT! Even if we are the authority, this doesn't mean other nodes aren't also the authority,
         //  so we have to let it propagate to all nodes.
         //if (this.isSelfAuthority(path)) return [getOwnNodeId()];
-        if (isClient()) {
-            return [getBrowserUrlNode()];
-        }
         await this.waitUntilRoutingIsReady();
 
         let result = Array.from(this.authorities.values())

package/src/1-path-client/RemoteWatcher.ts

@@ -5,7 +5,7 @@ import { cache } from "socket-function/src/caching";
 import { isNode, nextId } from "socket-function/src/misc";
 import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
 import { isOwnNodeId, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { LOCAL_DOMAIN_PATH, nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
 import { PathValueController, pathValueCommitter } from "../0-path-value-core/PathValueController";
 import { WatchConfig, authorityStorage, decodeParentFilter, matchesParentRangeFilter, pathWatcher } from "../0-path-value-core/pathValueCore";
 import { ActionsHistory } from "../diagnostics/ActionsHistory";
@@ -151,9 +151,9 @@ export class RemoteWatcher {
         return watchObj?.nodesId;
     }
 
-    private watchUnwatchSerial = runInSerial((name: string, fnc: () => Promise<unknown>) =>
-        fnc()
-    );
+    private watchUnwatchSerial = runInSerial(async (name: string, fnc: () => Promise<unknown>) => {
+        return await fnc();
+    });
 
     /** NOTE: We dedupe duplicate watches in watchLatest. */
     private async watchLatestBase(config: WatchConfig & { debugName?: string }) {
@@ -183,6 +183,9 @@ export class RemoteWatcher {
         let totalMissingPaths = 0;
         measureBlock(() => {
             for (let path of config.paths) {
+                // NOTE: We weren't ignoring local paths here for a while, but... I'm pretty sure we should, to save time.
+                if (path.startsWith(LOCAL_DOMAIN_PATH)) continue;
+
                 // IMPORTANT! We have to await, otherwise we might see the paths are watched, skip them
                 //  find another path which has an authority, then go to watch that... which causes
                 //  us to circumvent earlier watchers, causing us to watch out of order, causing reads
@@ -229,6 +232,8 @@ export class RemoteWatcher {
             }
 
             for (let path of config.parentPaths) {
+                // NOTE: We weren't ignoring local paths here for a while, but... I'm pretty sure we should, to save time.
+                if (path.startsWith(LOCAL_DOMAIN_PATH)) continue;
                 let watchObj = this.remoteWatchParents.get(path);
                 if (
                     watchObj

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -42,6 +42,8 @@ import { onNextPaint } from "../functional/onNextPaint";
 // With a harness that joins the two parts in a loop (mostly powered by clientWatcher,
 //      which can run the trigger loop).
 
+const DEFAULT_MAX_LOCKS = 1000;
+
 let nextOrderSeqNum = 1;
 
 export interface WatcherOptions<Result> {
@@ -208,6 +210,8 @@ export interface WatcherOptions<Result> {
      *      - Also to tell us how long and how many paths are loading on startup.
     */
     logSyncTimings?: boolean;
+
+    maxLocksOverride?: number;
 }
 
 let harvestableReadyLoopCount = 0;
@@ -1539,6 +1543,11 @@ export class PathValueProxyWatcher {
                     }
                 }
 
+                let maxLocks = watcher.options.maxLocksOverride || DEFAULT_MAX_LOCKS;
+                if (locks.length > maxLocks) {
+                    throw new Error(`Too many locks for ${watcher.debugName} (${locks.length} > ${maxLocks}). You can override max locks with maxLocksOverride (in options / functionMetadata).`);
+                }
+
                 setValues = clientWatcher.setValues({
                     values: watcher.pendingWrites,
                     eventPaths: watcher.pendingEventWrites,
@@ -2229,6 +2238,10 @@ export function noAtomicSchema<T>(code: () => T) {
 
 
 export const proxyWatcher = new PathValueProxyWatcher();
+if ((globalThis as any).proxyWatcher) {
+    debugger;
+    console.error(`Loaded PathValueProxyWatcher twice. This will break this. In ${module.filename}`);
+}
 (globalThis as any).proxyWatcher = proxyWatcher;
 (globalThis as any).ProxyWatcher = PathValueProxyWatcher;
 // Probably the most useful debug function for debugging watch based functions.

package/src/3-path-functions/PathFunctionRunner.ts

@@ -76,6 +76,8 @@ export type FunctionSpec = {
     gitURL: string;
     /** Ex, the hash */
     gitRef: string;
+
+    maxLocksOverride?: number;
 };
 
 export interface CallSpec {
@@ -513,6 +515,7 @@ export class PathFunctionRunner {
                     getPermissionsCheck: PermissionsChecker && (() => new PermissionsChecker(callPath)),
                     nestedCalls: "inline",
                     temporary: true,
+                    maxLocksOverride: functionSpec.maxLocksOverride,
                     watchFunction: function runCallWatcher() {
                         runCount++;
                         if (PathFunctionRunner.DEBUG_CALLS) {
@@ -560,6 +563,7 @@ export class PathFunctionRunner {
                                     || syncedSpec.FilePath !== functionSpec.FilePath
                                     || syncedSpec.gitRef !== functionSpec.gitRef
                                     || syncedSpec.gitURL !== functionSpec.gitURL
+                                    || syncedSpec.maxLocksOverride !== functionSpec.maxLocksOverride
                                 )
                             ) {
                                 isFunctionSpecOutdated = true;

package/src/3-path-functions/syncSchema.ts

@@ -20,6 +20,7 @@ import path from "path";
 import { isEmpty } from "../misc";
 import { getSpecFromModule } from "./pathFunctionLoader";
 import { isNode } from "typesafecss";
+import { Querysub } from "../4-querysub/QuerysubController";
 
 // This is the the function id which should be used when creating the FunctionSpec (in order to load the module),
 //  to access the permissions in the schema.
@@ -45,6 +46,9 @@ export type FunctionMetadata<F = unknown> = {
      *          before writing to B.
      */
     delayCommit?: boolean;
+
+    /** Too many locks can lag the server, and eventually cause crashes. Consider using Querysub.noLocks(() => ...) around code that is accessing too many values, assuming you don't want to lock them. However, if absolutely required, you can override max locks to allow as many locks to be created as you want until the server crashses... */
+    maxLocksOverride?: number;
 };
 
 export interface SchemaObject<Schema = any, Functions = any> {

package/src/4-deploy/deployFunctions.ts

@@ -25,9 +25,6 @@ export async function deployGetFunctions(): Promise<FunctionSpec[]> {
     return functionSpecs;
 }
 
-//todonext;
-// Ugh... some kind of throttling? Nah...
-// So... progress sections, just... start and stop?
 export type DeployProgress = {
     (config: { section: string; progress: number; }): void;
 };

package/src/4-deploy/deployGetFunctionsInner.ts

@@ -76,6 +76,7 @@ async function main() {
         const functionIds = Object.keys(schema.rawFunctions);
         functionIds.push(PERMISSIONS_FUNCTION_ID);
         for (let functionId of functionIds) {
+            let metadata = schema.functionMetadata?.[functionId];
             const exportPathStr = getExportPath(functionId);
             let spec: FunctionSpec = {
                 DomainName: domainName,
@@ -85,6 +86,7 @@ async function main() {
                 exportPathStr,
                 gitURL,
                 gitRef,
+                maxLocksOverride: metadata?.maxLocksOverride,
             };
             console.log(`FUNCTION_SPEC:${JSON.stringify(spec)}`);
         }

package/src/4-deploy/edgeBootstrap.ts

@@ -14,6 +14,9 @@ export const liveHashOverrideURL = new URLParam(liveHashOverrideParam, undefined
     time: number;
 });
 
+declare global {
+    var getEdgeNodeConfig: undefined | (() => Promise<EdgeNodeConfig>);
+}
 
 let getCachedConfig = cache(async (url: string): Promise<EdgeNodesIndex | undefined> => {
     setTimeout(() => {
@@ -218,6 +221,8 @@ async function edgeNodeFunction(config: {
         };
     }
 
+    globalThis.getEdgeNodeConfig = getEdgeNodeConfig;
+
     let cachedConfig = config.cachedConfig;
 
     let liveHashOverride = "";

package/src/4-deploy/edgeClientWatcher.tsx

@@ -2,10 +2,10 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { deploySchema } from "./deploySchema";
 import { getDomain, noSyncing } from "../config";
-import { throttleFunction, timeInMinute } from "socket-function/src/misc";
+import { throttleFunction, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { isNode } from "typesafecss";
 import { logErrors, timeoutToError } from "../errors";
-import { blue, red } from "socket-function/src/formatting/logColors";
+import { blue, red, yellow } from "socket-function/src/formatting/logColors";
 import { showModal } from "../5-diagnostics/Modal";
 import { qreact } from "../4-dom/qreact";
 import { liveHashOverrideURL } from "./edgeBootstrap";
@@ -13,6 +13,11 @@ import { css } from "typesafecss";
 import { formatNiceDateTime, formatTime } from "socket-function/src/formatting/format";
 import { delay } from "socket-function/src/batching";
 import { atomicObjectRead } from "../2-proxy/PathValueProxyWatcher";
+import { lazy } from "socket-function/src/caching";
+import { Button } from "../library-components/Button";
+import { updateRootDiscoveryLocation } from "../-f-node-discovery/NodeDiscovery";
+
+const SWITCH_SERVER_TIMEOUT = timeInSecond * 15;
 
 let lastHashServer = "";
 export function startEdgeNotifier() {
@@ -29,7 +34,7 @@ export function startEdgeNotifier() {
         }
         void notifyClients(liveHash, refreshThresholdTime);
     });
-    (async () => {
+    void (async () => {
         await delay(1);
         let { watchOnRollingUpdate } = await import("../deployManager/machineController");
         watchOnRollingUpdate({
@@ -119,7 +124,7 @@ function onLiveHashChange(liveHash: string, refreshThresholdTime: number) {
                             >
                                 <div className={css.vbox(10).maxWidth(250)}>
                                     <h3 className={css.margin(0)}>Server Update Available</h3>
-                                    <div>The server has been updated. Please refresh the page to ensure you don't experience incompatibility issues.</div>
+                                    <div>The server has been updated. Please refresh the page to ensure you don't experience compatibility issues.</div>
                                     {i !== notifyIntervals.length - 2 && <div>This notification will be shown again at {formatNiceDateTime(Date.now() + waitDuration)}</div>}
                                     {i === notifyIntervals.length - 2 && <div>The page will automatically refresh at {formatNiceDateTime(Date.now() + waitDuration)}</div>}
                                 </div>
@@ -168,6 +173,7 @@ function onLiveHashChange(liveHash: string, refreshThresholdTime: number) {
     })();
 }
 
+
 const EdgeNotifierClientController = SocketFunction.register(
     "EdgeNotifierClientController-ed122e41-6ad2-4161-9492-a3f8414bd4f0",
     new class EdgeNotifierClient {
@@ -188,10 +194,37 @@ if (!isNode()) {
     setImmediate(() => {
         void Querysub.optionalStartupWait().finally(async () => {
             curHash = await EdgeNotifierController.nodes[SocketFunction.browserNodeId()].watchUpdates();
+            void runRefreshLoop();
         });
         SocketFunction.expose(EdgeNotifierClientController);
     });
 }
+const runRefreshLoop = lazy(async () => {
+    let getEdgeNodeConfig = globalThis.getEdgeNodeConfig;
+    if (!getEdgeNodeConfig) throw new Error(`Missing getEdgeNodeConfig. This means we can't reconnect if the server disconnects. Did we not inject the edge node detection code into the bootstrapper?`);
+    let lastAliveTime = Date.now();
+    let doNotRefresh = false;
+    while (true) {
+        await delay(3000);
+        if (doNotRefresh) break;
+        let timeSinceLastAlive = Date.now() - lastAliveTime;
+        if (SocketFunction.isNodeConnected(SocketFunction.browserNodeId())) {
+            lastAliveTime = Date.now();
+        } else if (timeSinceLastAlive > SWITCH_SERVER_TIMEOUT) {
+            lastAliveTime = Date.now();
+            console.log(yellow(`Server timed out, finding new server`));
+            try {
+                let config = await getEdgeNodeConfig();
+                updateRootDiscoveryLocation(config);
+            } catch (e: any) {
+                console.warn(`Failed to find new edge node, trying again in 15 seconds: ${e.message}`);
+                await delay(15000);
+            }
+        } else {
+            console.log(yellow(`No edge node connected, trying again in 3 seconds`));
+        }
+    }
+});
 
 class EdgeNotifierControllerBase {
     public async watchUpdates() {

package/src/4-querysub/Querysub.ts

@@ -17,7 +17,7 @@ import { getHostedIP, getSNICerts, publishMachineARecords } from "../-e-certs/Ed
 import { LOCAL_DOMAIN, nodePathAuthority } from "../0-path-value-core/NodePathAuthorities";
 import { debugCoreMode, registerGetCompressNetwork, encodeParentFilter, registerGetCompressDisk, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { clientWatcher, ClientWatcher } from "../1-path-client/pathValueClientWatcher";
-import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions } from "../2-proxy/PathValueProxyWatcher";
+import { SyncWatcher, proxyWatcher, specialObjectWriteValue, isSynced, PathValueProxyWatcher, atomic, doAtomicWrites, noAtomicSchema, undeleteFromLookup, registerSchemaPrefix, WatcherOptions, doProxyOptions } from "../2-proxy/PathValueProxyWatcher";
 import { isInProxyDatabase, rawSchema } from "../2-proxy/pathDatabaseProxyBase";
 import { isValueProxy2, getProxyPath } from "../2-proxy/pathValueProxy";
 import { getCurrentCallAllowUndefined, getCurrentCall, CallSpec, PathFunctionRunner } from "../3-path-functions/PathFunctionRunner";
@@ -438,6 +438,10 @@ export class Querysub {
         onNestedSynced(callback);
     }
 
+    public static noLocks = (callback: () => void) => {
+        doProxyOptions({ noLocks: true }, callback);
+    };
+
     /** A more powerful version of omCommitFinished, which even waits for call predictions (or tries to).
      *      - Also see afterPredictionsSynced, which runs the callback in a write.
      *  NOTE: IDEALLY, you would just call a series of synced functions. They will be run in order,

package/src/4-querysub/QuerysubController.ts

@@ -34,7 +34,7 @@ import { PromiseObj } from "../promise";
 import { LoggingClient } from "../0-path-value-core/LoggingClient";
 import * as prediction from "./querysubPrediction";
 import { getCallResultPath } from "./querysubPrediction";
-import { pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
+import { nodePathAuthority, pathValueAuthority2 } from "../0-path-value-core/NodePathAuthorities";
 import { diskLog } from "../diagnostics/logs/diskLogger";
 import { assertIsManagementUser } from "../diagnostics/managementPages";
 import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
@@ -43,7 +43,7 @@ setFlag(require, "preact", "allowclient", true);
 import yargs from "yargs";
 import { mergeFilterables, parseFilterable, serializeFilterable } from "../misc/filterable";
 import { isManagementUser, onAllPredictionsFinished } from "../-0-hooks/hooks";
-import { isBootstrapOnly, isLocal } from "../config";
+import { getDomain, isBootstrapOnly, isLocal } from "../config";
 
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("fncfilter", { type: "string", default: "", desc: `Sets the filterable state for function calls, causing them to target specific FunctionRunners. If no FunctionRunner matches, all functions will fail to run. For example: "devtestserver" will match a FunctionRunner that uses the "devtestserver" filter. Merges with the existing filterable state if a client sets it explicitly.` })
@@ -55,40 +55,29 @@ const getFncFilter = lazy(() => parseFilterable(yargObj.fncfilter));
 export { Querysub, id };
 
 
-// NOTE: Eventually this will be split by domain, because the underlying getAllNodeIds
-//  will be split by domain.
-// NOTE: We don't need to shard querysub nodes within a domain, because... they don't
-//  store data, and don't run functions, and only really check permissions (which should be
-//  very lightweight, hopefully...)
-//  - I guess we track our watches, but even that... should be fairly light, and... there
-//      is little benefit to sharding that?
-export const querysubNodeId = lazy(() => {
-    if (isNode()) {
-        return getControllerNodeId(QuerysubController);
+// TODO: Eventually this will be split by domain, which required nodeIds to be split by domain, so that getControllerNodeId can check nodeIds on a specific domain.
+export async function querysubNodeId() {
+    if (isClient()) {
+        return nodePathAuthority.getSingleReadNodePromise(getPathStr1(getDomain()));
     } else {
-        // NOTE: We won't be able to directly connect to some querysub nodes (because they are from other
-        //  developers), so... just use the browser url.
-        return getBrowserUrlNode();
+        return getControllerNodeId(QuerysubController);
     }
-});
+}
+
 
 setImmediate(() => {
-    async function querysubWatchLatest(config: WatchConfig) {
-        let nodeId = await querysubNodeId();
-        if (!nodeId) throw new Error("No querysub node found");
-        await QuerysubController.nodes[nodeId].watch(config);
+    async function querysubWatchLatest(config: WatchConfig, authorityId: string) {
+        await QuerysubController.nodes[authorityId].watch(config);
     }
-    async function querysubUnwatchLatest(config: WatchConfig) {
-        let nodeId = await querysubNodeId();
-        if (!nodeId) throw new Error("No querysub node found");
-        await QuerysubController.nodes[nodeId].unwatch(config);
+    async function querysubUnwatchLatest(config: WatchConfig, authorityId: string) {
+        await QuerysubController.nodes[authorityId].unwatch(config);
     }
 
     let baseRemoteWatchFunction = RemoteWatcher.REMOTE_WATCH_FUNCTION;
     RemoteWatcher.REMOTE_WATCH_FUNCTION = async (config, authorityId) => {
         let weTrusted = isServer() && await errorToUndefined(isTrustedByNode(authorityId));
         if (!weTrusted) {
-            await querysubWatchLatest(config);
+            await querysubWatchLatest(config, authorityId);
         } else {
             await baseRemoteWatchFunction(config, authorityId);
         }
@@ -98,7 +87,7 @@ setImmediate(() => {
     RemoteWatcher.REMOTE_UNWATCH_FUNCTION = async (config, authorityIdBase) => {
         let weTrusted = isServer() && await isTrustedByNode(authorityIdBase);
         if (!weTrusted) {
-            logErrors(querysubUnwatchLatest(config));
+            logErrors(querysubUnwatchLatest(config, authorityIdBase));
         } else {
             logErrors(baseRemoteUnwatchFunction(config, authorityIdBase));
         }

package/src/5-diagnostics/memoryValueAudit.ts

@@ -313,7 +313,7 @@ async function auditSpecificPathsBase(config: {
             }
             sort(logObjs, x => x.log.time);
             let now = Date.now();
-            for (let { source, log } of logObjs) {
+            for (let { source, log } of logObjs.slice(-100)) {
                 console.log(`${source.padEnd(8, " ")} ${log.type.padEnd(20, " ")}`, now - log.time, log.values, log.time);
             }
             debugbreak(2);

package/src/deployManager/machineSchema.ts

@@ -106,7 +106,7 @@ async function registerNodeForMachineCleanup(nodeId: string) {
         console.log(green(`Registering node for machine cleanup at ${nodeIdPath}`));
         await fs.promises.writeFile(nodeIdPath, nodeId);
     } else {
-        console.log(red(`Not registering node for machine cleanup because we are not in the service folder: ${currentPath}`));
+        console.log(`Not registering node for machine cleanup because we are not in the service folder: ${currentPath}`);
     }
 }
 

package/src/deployManager/spec.txt

@@ -1,16 +1,11 @@
-TODO: It seems like every other time it fails to reload?
+// updateRootDiscoveryLocation("127-0-0-1.querysubtest.com:7008")
+
+Get updateRootDiscoveryLocation called when we disconnect, and don't reconnect?
 
-Uh... we get a failure in commitWrites, and then we never receive the path values we need?
-    - Although we only don't receive the values we need once, so... maybe the failed check is okay?
 
-1) Verify clientside notifications work when we rolling update cyoa
-2) Verify edge nodes are removed when we rolling update cyoa
-    - Eh... it removes most of the time?
-3) Update cyoa rolling time back to 4 hours
 
-1) Auto restart if the connect to the cyoa server is down, after 30 seconds, instead of just infinitely erroring out.
-    - Give a notification after 15 seconds warning them, with the ability to not restart if they click "do not restart"
-    - In edgeClientWatcher
+1) Update everything to latest
+3) Update cyoa rolling time back to 4 hours
 
 
 9) In machine info, support an object which can show current, and max, so we can show it as a bar and the max amount for:
@@ -25,5 +20,4 @@ Uh... we get a failure in commitWrites, and then we never receive the path value
     (Yes, it overlaps with node metrics, but that's okay, sometimes we want to look at machines, other times services, and the types of errors that show up in either changes).
 
 
-11) Restart the server and verify everything starts up nicely on reboot!
-    - Or... screw it, and move onto audio apis...
+11) A spinning animation on the FPS display, so we can visually see when the browser locks up (because the animation will stop?)

package/src/diagnostics/errorLogs/ErrorLogController.ts

@@ -344,7 +344,8 @@ export class ErrorLogControllerBase {
         if (ErrorLogControllerBase.logIssueNotifyEquals(prev, notify)) {
             return;
         }
-        console.log(magenta(`Received log notify from ${caller.nodeId} for ${notify?.oldestUngrouped?.message}`));
+        // Don't log when we receive it, otherwise errors show up on all servers which is very confusing and annoying!
+        //console.log(magenta(`Received log notify from ${caller.nodeId} for ${notify?.oldestUngrouped?.message}`));
         ErrorLogControllerBase.liveLogsIssueNotify.set(caller.nodeId, notify);
         logErrors(ErrorLogControllerBase.recalcAllNotify());
     }

package/src/diagnostics/logs/ObjectDisplay.tsx

@@ -89,9 +89,14 @@ export class ObjectDisplay extends qreact.Component<{
             </>;
         };
 
+        function log() {
+            (globalThis as any).debugValue = value;
+            console.log("debugValue=", value);
+        }
+
         if (!canHaveChildren(value)) {
             return <ShowMore maxHeight={maxHeight} className={css.maxWidth("50vw")}>
-                <span className={css.maxHeight("70vh").overflowAuto}>
+                <span className={css.maxHeight("70vh").overflowAuto} onClick={log}>
                     <div class={css.relative}>
                         {renderValue(value, [])}
                         <MeasureHeightCSS />
@@ -103,7 +108,7 @@ export class ObjectDisplay extends qreact.Component<{
             css.hbox(6).alignItems("start")
             + (!expanded && css.maxHeight(maxHeight).overflowHidden)
             + (expanded && css.maxHeight("70vh").overflowAuto)
-        }>
+        } onClick={log}>
             {canHaveChildren(value) && <Button
                 className={
                     css.hsl(270, 50, 50).background("hsl(270, 50%, 70%)", "hover")
@@ -135,7 +140,7 @@ export class PrimitiveDisplay extends qreact.Component<{
             return <span className={css.boldStyle.hslcolor(265, 70, 60)}>{value}</span>;
         }
         if (typeof value === "boolean") {
-            return <span className={css.boldStyle.hslcolor(207, 70, 60)}>{value}</span>;
+            return <span className={css.boldStyle.hslcolor(207, 70, 60)}>{JSON.stringify(value)}</span>;
         }
 
         if (typeof value === "string") {

package/src/diagnostics/managementPages.tsx

@@ -318,7 +318,7 @@ class ManagementRoot extends qreact.Component {
                             }
                         `}
                     </style>
-                    <div class={css.fillWidth.hbox(30).wrap.hsl(245, 25, 60).pad2(10).pointerEvents("all")}>
+                    <div class={css.fillWidth.hbox(30, 10).wrap.hsl(245, 25, 60).pad2(10).pointerEvents("all")}>
                         <LogNotify />
                         {pages.map(page =>
                             <ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>