querysub 0.441.0 → 0.443.0

package/.claude/settings.local.json

@@ -0,0 +1,8 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(Get-ChildItem *)",
+      "mcp__querysub-logs__searchIndexes"
+    ]
+  }
+}

package/bin/mcp-indexed-logs.js

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+
+process.argv.push("--local");
+
+require("typenode");
+require("../src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.441.0",
+  "version": "0.443.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -19,7 +19,9 @@
     "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx",
     "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild",
     "audit-imports": "yarn typenode ./src/diagnostics/auditImportViolations.ts",
-    "test": "yarn typenode ./test.ts"
+    "test": "yarn typenode ./test.ts",
+    "mcp": "yarn typenode ./src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts",
+    "mc": "yarn typenode ./src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts --cwd D:/repos/qs-cyoa/"
   },
   "bin": {
     "deploy": "./bin/deploy.js",
@@ -41,7 +43,8 @@
     "error-watch": "./bin/error-watch.js",
     "error-watch-public": "./bin/error-watch-public.js",
     "audit-imports": "./bin/audit-imports.js",
-    "audit-disk-values": "./bin/audit-disk-values.js"
+    "audit-disk-values": "./bin/audit-disk-values.js",
+    "mcp-indexed-logs": "./bin/mcp-indexed-logs.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",
@@ -61,7 +64,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
-    "socket-function": "^1.1.23",
+    "socket-function": "^1.1.33",
     "terser": "^5.31.0",
     "typesafecss": "^0.29.0",
     "yaml": "^2.5.0",

package/spec.txt

@@ -1,5 +1,6 @@
 
 
+
 Trigger values
     1) validStateComputer.ingestValuesAndValidStates
     1.5) authorityStorage.ingestValues

package/src/-a-archives/archiveCache.ts

@@ -17,7 +17,7 @@ import { measureWrap } from "socket-function/src/profiling/measure";
 
 const SIZE_LIMIT = new SizeLimiter({
     diskRoot: getStorageDir(),
-    maxBytes: isPublic() ? 1024 * 1024 * 1024 * 250 : 1024 * 1024 * 1024 * 50,
+    maxBytes: isPublic() ? 1024 * 1024 * 1024 * 250 : 1024 * 1024 * 1024 * 100,
     // Anything less than this and we can't even load enough weights models for a single task
     minBytes: 1024 * 1024 * 1024 * 8,
     maxDiskFraction: 0.3,

package/src/-e-certs/EdgeCertController.ts

@@ -119,16 +119,10 @@ async function EdgeCertController_watchHTTPSKeyCert(
 
 /** NOTE: Any SocketFunction.mount calls should probably call this, otherwise they won't be able to be called. */
 export async function publishMachineARecords() {
-    if (!SocketFunction.isMounted()) {
-        throw new Error(`Must mount before publishing machine A records`);
-    }
-
     let ip = await getHostedIP();
 
-    let selfNodeId = SocketFunction.mountedNodeId;
-    let nodeObj = getNodeIdLocation(selfNodeId);
-    if (!nodeObj) throw new Error(`Invalid nodeId ${selfNodeId}`);
-    let machineAddress = nodeObj.address.split(".").slice(1).join(".");
+    let machineAddress = getOwnMachineId() + "." + getDomain();
+
     let prevMachineIP = await getRecords("A", machineAddress);
     let ipDomain = await getIPDomain();
     let promises: Promise<void>[] = [];

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -49,6 +49,8 @@ let DISK_AUDIT_RATE = timeInMinute * 15;
 let API_AUDIT_RATE = timeInSecond * 30;
 let API_AUDIT_COUNT = 12;
 
+let FIRST_BROADCAST_TIMEOUT = timeInSecond * 5;
+
 
 let DEAD_NODE_POLL_COOLDOWN = timeInMinute * 5;
 
@@ -66,6 +68,7 @@ export function isNodeDiscoveryLogging() {
 
 
 function getAlternateNodeIds(nodeId: string): MaybePromise<string[] | undefined> {
+    // IMPORTANT! NEVER allow for reconnection of client ids. A lot of code depends on the fact that clients will reconnect with a new client node id when they disconnect!
     let machineId = certs.getMachineId(nodeId);
     if (machineId === getOwnMachineId()) {
         let decoded = decodeNodeId(nodeId);
@@ -356,7 +359,7 @@ async function runHeartbeatAuditLoop() {
     await runInfinitePollCallAtStart(CHECK_INTERVAL * 0.9, async () => {
         if (shutdown) return;
         // Wait a bit longer, to try to prevent all nodes from synchronizing their audit times.
-        console.log(magenta(`Auditing node list`));
+        console.info(magenta(`Auditing node list`));
         await delay(CHECK_INTERVAL * Math.random() * 0.1);
         //console.log(magenta(`Auditing node list`));
 
@@ -378,11 +381,11 @@ async function runHeartbeatAuditLoop() {
                 deadCount.set(nodeId, count);
                 if (count >= DEAD_CHECK_COUNT) {
                     removedNodeIds.push(nodeId);
-                    console.log(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
+                    console.info(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
                     await archives().del(nodeId);
                     deadCount.delete(nodeId);
                 } else {
-                    console.log(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
+                    console.info(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
                     pendingDeadCount++;
                 }
             } else {
@@ -453,8 +456,12 @@ async function writeHeartbeat() {
     await archives().set(nodeId, Buffer.from(now + ""));
 }
 
-async function runMainSyncLoops() {
-    await syncArchives();
+async function runServerSyncLoops() {
+    await Promise.all([
+        syncArchives(),
+        // NOTE: This is new, but I am pretty sure required. We don't want our node out there before we are trusted.
+        ensureWeAreTrusted(),
+    ]);
 
     discoveryReady.resolve();
 
@@ -473,7 +480,7 @@ async function runMainSyncLoops() {
         if (isOwnNodeId(nodeId)) return;
         // Ignore errors, but wait a bit, so hopefully 99.99% of the time we can be certain
         //  all other nodes know our node id at this point.
-        await timeoutToUndefinedSilent(timeInSecond * 5, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
+        await timeoutToUndefinedSilent(FIRST_BROADCAST_TIMEOUT, errorToUndefinedSilent(NodeDiscoveryController.nodes[nodeId].addNode(getOwnNodeId())));
     }));
 
     nodeBroadcasted.resolve();
@@ -527,7 +534,7 @@ if (isServer()) {
         logErrors(runMemoryAuditLoop());
         // NOTE: We used to wait until we mounted, but... we should be able to find nodes
         //  before we mount, right? (And what if we never mount?)
-        runMainSyncLoops().catch(e => {
+        runServerSyncLoops().catch(e => {
             discoveryReady.reject(e);
             logErrors(Promise.reject(e));
         });

package/src/-g-core-values/NodeCapabilities.ts

@@ -139,6 +139,9 @@ class NodeCapabilitiesControllerBase {
     public async getMemoryUsage() {
         return process.memoryUsage();
     }
+    public async getProcessId() {
+        return process.pid;
+    }
     public async getFunctionRunnerShards() {
         return getFunctionRunnerShards();
     }
@@ -189,6 +192,7 @@ export const NodeCapabilitiesController = SocketFunction.register(
         getEntryPoint: {},
         getStartupTime: {},
         getMemoryUsage: {},
+        getProcessId: {},
         getFunctionRunnerShards: {},
         getTrueTimeOffset: {},
         getInspectURL: { hooks: [requiresNetworkTrustHook] },

package/src/0-path-value-core/AuthorityLookup.ts

@@ -6,7 +6,7 @@ import { getDomain, isPublic } from "../config";
 import { cache, lazy } from "socket-function/src/caching";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { delay, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { getAllNodeIds, getOwnNodeId, isOwnNodeId, onNodeBroadcasted, syncNodesNow, watchDeltaNodeIds, watchNodeIds } from "../-f-node-discovery/NodeDiscovery";
+import { getAllNodeIds, getOwnNodeId, isOwnNodeId, onNodeBroadcasted, onNodeDiscoveryReady, syncNodesNow, watchDeltaNodeIds, watchNodeIds } from "../-f-node-discovery/NodeDiscovery";
 import { IdentityController_getCurrentReconnectNodeIdAssert } from "../-c-identity/IdentityController";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { isClient } from "../config2";
@@ -106,7 +106,9 @@ class AuthorityLookup {
 
 
     public startSyncing = lazy(async () => {
-        await onNodeBroadcasted();
+        // NOTE: This used to wait for onNodeBroadcasted. I know we had a reason at some point, but that reason might have went away after refactoring PathRouter. Waiting for node discovery is much faster. So hopefully we can leave this here because fast startup really helps development. If required we can probably do a check for is public and then do the faster case. 
+        await onNodeDiscoveryReady();
+
         let firstSync = true;
         let promises: Promise<void>[] = [];
         watchDeltaNodeIds(({ newNodeIds, removedNodeIds }) => {
@@ -168,6 +170,7 @@ class AuthorityLookup {
     private connectTo = cache((nodeId: string) => {
         let stopObj = { stop: false };
         let promise = runInfinitePollCallAtStart(NETWORK_POLL_INTERVAL, async () => {
+            // NOTE: We keep calling even on disconnect, until it is removed from node discovery, so we handle reconnections. AND if it is removed from node discovery and reconnects, that's fine, it will tell us about the rediscovery as if was a new node, and we will setup it again.
             await this.syncNodeSerial(nodeId)();
         }, stopObj);
 
@@ -181,6 +184,7 @@ class AuthorityLookup {
     });
 
 
+    // Only allow one outstanding call per node (but allow multiple if across different nodes)
     private syncNodeSerial = cache((nodeId: string) => {
         return runInSerial(async () => {
             await this.syncNodeNow(nodeId);
@@ -188,10 +192,11 @@ class AuthorityLookup {
     });
     private disconnectWatch = cache((nodeId: string) => {
         let onDisconnect = () => {
-            SocketFunction.onNextDisconnect(nodeId, onDisconnect);
+            SocketFunction.onNextDisconnect(nodeId, onDisconnect, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
+            // NOTE: Will be re-added once we receive an updatePaths call (or the node will be removed by NodeDiscovery, in which case we will be entirely cleaned up).
             this.topology.nodes.delete(nodeId);
         };
-        SocketFunction.onNextDisconnect(nodeId, onDisconnect);
+        SocketFunction.onNextDisconnect(nodeId, onDisconnect, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
     });
     private async syncNodeNow(nodeId: string) {
         try {

package/src/0-path-value-core/LockWatcher2.ts

@@ -131,6 +131,7 @@ class LockWatcher2 {
                 paths: pathsToUnwatch,
                 parentPaths: [],
                 callback: getLockWatcher2NodeId(),
+                reason: "LockWatcher2 cleanup",
             });
         }
 

package/src/0-path-value-core/PathValueController.ts

@@ -240,7 +240,7 @@ export class PathValueControllerBase {
                 auditLog("UNWATCHING PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId });
+        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" });
     }
 
     public static async getInitialValues(config: {

package/src/0-path-value-core/PathWatcher.ts

@@ -2,7 +2,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { cache } from "socket-function/src/caching";
 import { QueueLimited, keyByArray } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
-import { debugNodeThread } from "../-c-identity/IdentityController";
+import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
 import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { registerResource } from "../diagnostics/trackResources";
 import { ignoreErrors } from "../errors";
@@ -14,6 +14,7 @@ import { WatchConfig, authorityStorage, PathValue, NodeId, compareTime, isCoreQu
 import { matchesParentRangeFilter } from "./hackedPackedPathParentFiltering";
 import { isClient } from "../config2";
 import { delay } from "socket-function/src/batching";
+import { isClientNodeId } from "socket-function/src/nodeCache";
 
 setImmediate(() => import("../4-querysub/Querysub"));
 
@@ -216,8 +217,8 @@ class PathWatcher {
     }
     // NOTE: Automatically unwatches from remoteWatcher on all paths that are fully unwatched
     @measureFnc
-    public unwatchPath(config: WatchConfig & { callback: PathWatcherCallback; }) {
-        const { callback } = config;
+    public unwatchPath(config: WatchConfig & { callback: PathWatcherCallback; reason: string; }) {
+        const { callback, reason } = config;
 
         let fullyUnwatched: WatchConfig = {
             paths: [],
@@ -240,9 +241,9 @@ class PathWatcher {
             obj.watchers.delete(callback);
 
             if (isOwnNodeId(callback)) {
-                auditLog("local UNWATCH PARENT", { path });
+                auditLog("local UNWATCH PARENT", { path, reason });
             } else {
-                auditLog("non-local UNWATCH PARENT", { path, watcher: callback });
+                auditLog("non-local UNWATCH PARENT", { path, watcher: callback, remoteNodeId: debugNodeId(callback), remoteNodeThreadId: debugNodeThread(callback), reason });
             }
 
             if (obj.watchers.size === 0) {
@@ -277,9 +278,9 @@ class PathWatcher {
             watchers.watchers.delete(callback);
 
             if (isOwnNodeId(callback)) {
-                auditLog("local UNWATCH VALUE", { path });
+                auditLog("local UNWATCH VALUE", { path, reason });
             } else {
-                auditLog("non-local UNWATCH VALUE", { path, watcher: callback });
+                auditLog("non-local UNWATCH VALUE", { path, watcher: callback, reason });
             }
 
             if (watchers.watchers.size === 0) {
@@ -501,20 +502,17 @@ class PathWatcher {
     private ensureUnwatchingOnDisconnect = cache((nodeId: string) => {
         if (isOwnNodeId(nodeId)) return;
         if (nodeId.startsWith(getInternalValueWatchPrefix())) return;
+        if (!isClientNodeId(nodeId)) {
+            console.error(`UNEXPECTED SERVER NODE ID ${nodeId} ON PATH WATCHER. We need to handle the reconnection case here!`);
+        }
         SocketFunction.onNextDisconnect(nodeId, async () => {
             this.ensureUnwatchingOnDisconnect.clear(nodeId);
-
-            // Wait while, so Querysub doesn't thrash when clients connect and disconnect
-            //  TODO: We should be smarter about our wait time, waiting less if we have a lot of resource
-            //      pressure, and more (potentially forever), if we don't have much resource pressure
-            //      (memory, cpu, and network).
-            setTimeout(() => {
-                let watches = this.watchersToPaths.get(nodeId);
-                this.watchersToPaths.delete(nodeId);
-                if (watches) {
-                    this.unwatchPath({ paths: Array.from(watches.paths), parentPaths: Array.from(watches.parents), callback: nodeId });
-                }
-            }, MAX_CHANGE_AGE);
+            // NOTE: We used to wait here, but I think that's invalid. Client nodeIds will never reconnect, just unwatch now, as they will can never receive any values anyways.
+            let watches = this.watchersToPaths.get(nodeId);
+            this.watchersToPaths.delete(nodeId);
+            if (watches) {
+                this.unwatchPath({ paths: Array.from(watches.paths), parentPaths: Array.from(watches.parents), callback: nodeId, reason: "ensureUnwatchingOnDisconnect" });
+            }
         });
     });
 

package/src/0-path-value-core/pathValueArchives.ts

@@ -14,6 +14,8 @@ import { devDebugbreak, isNoNetwork } from "../config";
 import { wrapArchivesWithCache } from "../-a-archives/archiveCache";
 import { AuthoritySpec, PathRouter, debugSpec } from "./PathRouter";
 import { authorityLookup } from "./AuthorityLookup";
+import { delay } from "socket-function/src/batching";
+import { safeLoop } from "socket-function/src/batching";
 
 export const archives = lazy(() => wrapArchivesWithCache(getArchives("path-values/")));
 export const archivesLocks = lazy(() => getArchives("path-values-locks/"));
@@ -298,7 +300,7 @@ export class PathValueArchives {
         let notMatched = 0;
         let tooOldValues = 0;
 
-        for (let dataFile of dataPaths) {
+        await safeLoop({ data: dataPaths, name: blue("Load PathValues From Buffer") }, async dataFile => {
             let data = readCache.get(dataFile);
             if (!data) throw new Error(`Data file ${dataFile} not in lookup which we just checked?`);
             totalSize += data.byteLength;
@@ -325,7 +327,7 @@ export class PathValueArchives {
                     }
                 }
             }
-        }
+        });
 
         let relevantCount = totalCount - notMatched;
         parseTime = Date.now() - parseTime;
@@ -447,6 +449,22 @@ export class PathValueArchives {
         });
     }
 
+    private static valuePathCountCache: { time: number; count: Promise<number> } | undefined;
+    public static async getValuePathCount(): Promise<number> {
+        const ONE_HOUR = 60 * 60 * 1000;
+        let cache = PathValueArchives.valuePathCountCache;
+        if (cache && Date.now() - cache.time < ONE_HOUR) {
+            return cache.count;
+        }
+        let countPromise = (async () => {
+            let locker = await pathValueArchives.getArchiveLocker();
+            let allFiles = await locker.getAllValidFiles();
+            return allFiles.length;
+        })();
+        PathValueArchives.valuePathCountCache = { time: Date.now(), count: countPromise };
+        return countPromise;
+    }
+
     @measureFnc
     public async getValuePaths(authority: AuthoritySpec): Promise<{
         pickedPaths: string[];

package/src/0-path-value-core/pathValueCore.ts

@@ -4,7 +4,7 @@ import { addEpsilons, minusEpsilon } from "../bits";
 import { logErrors } from "../errors";
 import { appendToPathStr, getParentPathStr, getPathDepth, getPathIndexAssert, hack_getPackedPathSuffix, hack_setPackedPathSuffix, hack_stripPackedPath } from "../path";
 import { measureFnc, measureWrap } from "socket-function/src/profiling/measure";
-import { IdentityController_getOwnPubKeyShort } from "../-c-identity/IdentityController";
+import { IdentityController_getOwnPubKeyShort, debugNodeId, debugNodeThread } from "../-c-identity/IdentityController";
 import { pathValueArchives } from "./pathValueArchives";
 import { blue } from "socket-function/src/formatting/logColors";
 import { delay, runInfinitePoll } from "socket-function/src/batching";
@@ -567,14 +567,15 @@ class AuthorityPathValueStorage {
         if (this.DEBUG_UNWATCH) {
             console.log(blue(`Unsyncing path at ${Date.now()}`), path);
         }
-        if (isDiskAudit()) {
-            auditLog("DESTROY PATH", { path });
-        }
+        auditLog("DESTROY PATH", { path });
 
         this.isSyncedCache.delete(path);
         this.removePathFromStorage(path, "unwatched");
     }
     public markParentPathAsUnwatched(path: string) {
+        if (this.parentsSynced.has(path)) {
+            console.info(`Unwatching path that is a parent synced path (fine, but might be an issue)`, { path });
+        }
         // NOTE: I don't think we have to handle the case where we are the authority,
         //  because we don't delete any actual data here.
         this.parentsSynced.delete(path);
@@ -693,6 +694,10 @@ class AuthorityPathValueStorage {
 
     public addParentSyncs(parentSyncs: { parentPath: string; sourceNodeId: string }[]) {
         for (let obj of parentSyncs) {
+            if (isDebugLogEnabled()) {
+                auditLog("RECEIVED PARENT PATH", { parentPath: obj.parentPath, remoteNodeId: debugNodeId(obj.sourceNodeId), remoteNodeThreadId: debugNodeThread(obj.sourceNodeId) });
+            }
+
             let decoded = decodeParentFilter(obj.parentPath);
             let range = decoded ? { start: decoded.start, end: decoded.end } : { start: 0, end: 1 };
             let parentPath = hack_stripPackedPath(obj.parentPath);
@@ -1063,6 +1068,7 @@ class AuthorityPathValueStorage {
 
             if (pathsToClear.size > 0) {
                 for (let path of pathsToClear) {
+                    console.info(`Destroying event path`, { path });
                     this.destroyPath(path);
                     // I'm not sure if removing it as a parent is needed, or... maybe it is needed,
                     //  and this isn't enough of a check?

package/src/1-path-client/RemoteWatcher.ts

@@ -113,7 +113,8 @@ export class RemoteWatcher {
                 console.log(yellow(`Trying to find new authority for disconnected watches ${paths.length} paths and ${parentPaths.length} parent paths`));
                 logErrors(this.tryToReconnectNow());
             })());
-        });
+            // Reconnection is our loop running again, maybe matching the same nodeId, and then using it again.
+        }, "iKnowThatServerNodeIdsMayReconnect_andIHandleReconnections");
     });
 
     // NOTE: Lower numbers mean more lag on disconnected values, but... faster reconnection. AND... if a node is disconnected,
@@ -410,13 +411,11 @@ export class RemoteWatcher {
             // Only remote watch remotes!
             if (isOwnNodeId(authorityId)) continue;
 
-            if (isDevDebugbreak()) {
-                for (let path of paths) {
-                    auditLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
-                }
-                for (let path of parentPaths) {
-                    auditLog("remoteWatcher outer PARENT WATCH", { path, remoteNodeId: authorityId });
-                }
+            for (let path of paths) {
+                auditLog("remoteWatcher outer WATCH", { path, remoteNodeId: authorityId });
+            }
+            for (let path of parentPaths) {
+                auditLog("remoteWatcher outer PARENT WATCH", { path, remoteNodeId: authorityId });
             }
 
             if (STOP_KEYS_DOUBLE_SENDS) {
@@ -476,13 +475,11 @@ export class RemoteWatcher {
                     }
                 }
                 for (let [authorityId, { paths, parentPaths }] of byAuthority) {
-                    if (isDevDebugbreak()) {
-                        for (let path of paths) {
-                            auditLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
-                        }
-                        for (let path of parentPaths) {
-                            auditLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
-                        }
+                    for (let path of paths) {
+                        auditLog("remoteWatcher inner WATCH", { path, remoteNodeId: authorityId });
+                    }
+                    for (let path of parentPaths) {
+                        auditLog("remoteWatcher inner WATCH PARENT", { path, remoteNodeId: authorityId });
                     }
                     // NOTE: We log watches here because we want to log batched watches
                     ActionsHistory.OnNewWatches({ newPathsWatched: paths, newParentsWatched: parentPaths, debugName: batched[0].debugName, authorityId });
@@ -585,13 +582,11 @@ export class RemoteWatcher {
             });
         }
         for (let [authorityIdBase, { paths, parentPaths }] of unwatchesPerAuthority.entries()) {
-            if (isDevDebugbreak()) {
-                for (let path of paths) {
-                    auditLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
-                }
-                for (let path of parentPaths) {
-                    auditLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
-                }
+            for (let path of paths) {
+                auditLog("remoteWatcher inner UNWATCH", { path, remoteNodeId: authorityIdBase });
+            }
+            for (let path of parentPaths) {
+                auditLog("remoteWatcher inner UNWATCH PARENT", { path, remoteNodeId: authorityIdBase });
             }
             if (!isOwnNodeId(authorityIdBase)) {
                 logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({ paths, parentPaths }, authorityIdBase));

package/src/1-path-client/pathValueClientWatcher.ts

@@ -543,7 +543,7 @@ export class ClientWatcher {
                 //console.log(red("Unwatching paths"), pathsToUnwatch, parentPathsToUnwatch, Date.now());
                 let unwatchConfig: WatchConfig = { paths: Array.from(pathsToUnwatch), parentPaths: Array.from(parentPathsToUnwatch), };
                 // pathWatcher unwatches remotely as well
-                pathWatcher.unwatchPath({ callback: getOwnNodeId(), ...unwatchConfig });
+                pathWatcher.unwatchPath({ callback: getOwnNodeId(), ...unwatchConfig, reason: "pathValueClientWatcher local unwatch" });
             }
         });
     });

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -1451,7 +1451,7 @@ export class PathValueProxyWatcher {
                             if (watcher.disposed) return;
                             let remainingUnsynced = Array.from(watcher.lastUnsyncedAccesses).filter(x => !authorityStorage.isSynced(x));
                             let remainingUnsyncedParent = Array.from(watcher.lastUnsyncedParentAccesses).filter(x => !authorityStorage.isParentSynced(x));
-                            console.warn(red(`Did not sync ${watcher.debugName} after 30 seconds. Either we had a lot synchronous block, or we will never received the values we are waiting for.`), remainingUnsynced, remainingUnsyncedParent, watcher.options.watchFunction);
+                            console.warn(red(`Did not sync ${watcher.debugName} after 30 seconds. Either we had a lot synchronous block, or we will never received the values we are waiting for.`), { remainingUnsynced, remainingUnsyncedParent }, watcher.options.watchFunction);
                         }, 30 * 1000);
                         setTimeout(() => {
                             if (watcher.syncRunCount !== syncRunCount) return;
@@ -1471,11 +1471,11 @@ export class PathValueProxyWatcher {
                                 let notWatchingUnsynced = reallyUnsyncedAccesses.filter(x => !remoteWatcher.debugIsWatchingPath(x));
                                 let notWatchingUnsyncedParent = reallyUnsyncedParentAccesses.filter(x => !remoteWatcher.debugIsWatchingPath(x));
                                 if (notWatchingUnsynced.length !== 0 || notWatchingUnsyncedParent.length !== 0) {
-                                    console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("NOT REMOTE WATCHING REQUIRED PATHS")}. This means our sync or unsync (likely unsync) logic is broken, in remoteWatcher/clientWatcher. OR, there were no read nodes when we tried to sync (we don't handle missing read nodes correctly at the moment)`), notWatchingUnsynced, notWatchingUnsyncedParent, watcher.options.watchFunction);
+                                    console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("NOT REMOTE WATCHING REQUIRED PATHS")}. This means our sync or unsync (likely unsync) logic is broken, in remoteWatcher/clientWatcher. OR, there were no read nodes when we tried to sync (we don't handle missing read nodes correctly at the moment)`), { notWatchingUnsynced, notWatchingUnsyncedParent }, watcher.options.watchFunction);
                                     debugbreak(2);
                                     debugger;
                                 } else {
-                                    console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT RECEIVE PATH VALUES")}. This means PathValueServer is not responding to watches, either to specific paths, or for all paths`), reallyUnsyncedAccesses, reallyUnsyncedParentAccesses, watcher.options.watchFunction);
+                                    console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT RECEIVE PATH VALUES")}. This means PathValueServer is not responding to watches, either to specific paths, or for all paths`), { reallyUnsyncedAccesses, reallyUnsyncedParentAccesses }, watcher.options.watchFunction);
                                     // debugbreak(2);
                                     // debugger;
                                 }
@@ -1484,7 +1484,7 @@ export class PathValueProxyWatcher {
                                 debugbreak(2);
                                 debugger;
                             } else {
-                                console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT TRIGGER WATCHER")}. This means either ProxyWatcher is broken (and isn't triggering when it should, or isn't watching when it should), or ClientWatcher/PathWatcher are broken and are not properly informing callers of watchers.`), watcher.lastUnsyncedAccesses, watcher.lastUnsyncedParentAccesses, watcher.options.watchFunction);
+                                console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT TRIGGER WATCHER")}. This means either ProxyWatcher is broken (and isn't triggering when it should, or isn't watching when it should), or ClientWatcher/PathWatcher are broken and are not properly informing callers of watchers.`), { lastUnsyncedAccesses: watcher.lastUnsyncedAccesses, lastUnsyncedParentAccesses: watcher.lastUnsyncedParentAccesses }, watcher.options.watchFunction);
                                 debugbreak(2);
                                 debugger;
                             }
@@ -1494,7 +1494,7 @@ export class PathValueProxyWatcher {
                     if (watcher.countSinceLastFullSync > 10) {
                         require("debugbreak")(2);
                         debugger;
-                        console.warn(`Watcher ${watcher.debugName} has been unsynced for ${watcher.countSinceLastFullSync} times. This is fine, but maybe optimize it. Why is it cascading?`, watcher.lastUnsyncedAccesses, watcher.lastUnsyncedParentAccesses, watcher.options.watchFunction);
+                        console.warn(`Watcher ${watcher.debugName} has been unsynced for ${watcher.countSinceLastFullSync} times. This is fine, but maybe optimize it. Why is it cascading?`, { lastUnsyncedAccesses: watcher.lastUnsyncedAccesses, lastUnsyncedParentAccesses: watcher.lastUnsyncedParentAccesses }, watcher.options.watchFunction);
                     }
                     if (watcher.countSinceLastFullSync > 500) {
                         debugger;

package/src/3-path-functions/PathFunctionRunner.ts

@@ -453,7 +453,7 @@ export class PathFunctionRunner {
         let fraction = getRoutingOverridePart(callPath.CallId)?.route;
         if (fraction === undefined) {
             fraction = PathRouter.getSingleKeyRoute(callPath.CallId);
-            console.error(`No routing override found for callId, falling back to direct hash`, { callId: callPath.CallId, fraction });
+            console.error(`No routing override found for callId, falling back to direct hash`, { callId: callPath.CallId, fraction, domainName: callPath.DomainName, moduleId: callPath.ModuleId, functionId: callPath.FunctionId, callerMachineId: callPath.callerMachineId, callerIP: callPath.callerIP });
         }
         // It isn't secondary if it is primary
         if (shardRange.startFraction <= fraction && fraction < shardRange.endFraction) return 0;

package/src/4-querysub/Querysub.ts

@@ -32,7 +32,7 @@ import { inlineNestedCalls, syncSchema } from "../3-path-functions/syncSchema";
 import type { identityStorageKey, IdentityStorageType } from "../-a-auth/certs";
 
 import { ExternalRenderClass, qreact } from "../4-dom/qreact";
-import { configRootDiscoveryLocation, getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { configRootDiscoveryLocation, getOwnNodeId, onNodeBroadcasted, onNodeDiscoveryReady } from "../-f-node-discovery/NodeDiscovery";
 import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
 import debugbreak from "debugbreak";
 import { registerDynamicResource } from "../diagnostics/trackResources";
@@ -290,11 +290,16 @@ export class Querysub {
      *          isn't presently necessary in most serverside scripts.
      */
     @measureFnc
-    public static async optionalStartupWait() {
+    public static async optionalStartupWait(addTime?: (name: string) => void) {
+        // Used by authorityLookup, pulled out here so timing is more clear.
+        await onNodeDiscoveryReady();
+        addTime?.("onNodeDiscoveryReady");
         await authorityLookup.startSyncing();
+        addTime?.("authorityLookup.startSyncing");
         await measureBlock(async () => {
             await waitForFirstTimeSync();
         }, "waitForFirstTimeSync");
+        addTime?.("waitForFirstTimeSync");
     }
 
     public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
@@ -1049,18 +1054,30 @@ export class Querysub {
         if (isClient()) {
             throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
         }
+        let times: {
+            name: string;
+            duration: number
+        }[] = [];
+        let time = Date.now();
+        function addTime(name: string) {
+            times.push({ name, duration: Date.now() - time });
+            time = Date.now();
+        }
 
         this.socketFunctionInit();
-        await SocketFunction.mount({
+        let mountPromise = SocketFunction.mount({
             public: isPublic(),
             port,
             ...await getThreadKeyCert(),
         });
 
-        await publishMachineARecords();
+        let publishPromise = publishMachineARecords();
+        await Promise.all([mountPromise, publishPromise]);
+        addTime("mount & publish a records");
+
+        await Querysub.optionalStartupWait(addTime);
 
-        await Querysub.optionalStartupWait();
-        console.log(magenta(`Started hosting service ${name}`));
+        console.log(magenta(`Started hosting service ${name}`) + ` | ${times.map(t => `${blue(t.name)}: ${green(formatTime(t.duration))}`).join(" | ")}`);
     }
 
     /** Syncs keys AND values (as we won't return a key for a value that is undefined). */
@@ -1333,7 +1350,7 @@ import "../diagnostics/trackResources";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { getCountPerPaint } from "../functional/onNextPaint";
 import { addEpsilons } from "../bits";
-import { blue, magenta } from "socket-function/src/formatting/logColors";
+import { blue, green, magenta } from "socket-function/src/formatting/logColors";
 import { MachineController } from "../deployManager/machineController";
 import { getRecords, setRecord } from "../-b-authorities/dnsAuthority";
 import { testTCPIsListening } from "socket-function/src/networking";