querysub 0.525.0 → 0.526.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.525.0",
+  "version": "0.526.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/LatencyTracking.ts

@@ -8,7 +8,7 @@
 */
 
 import { SocketFunction } from "socket-function/SocketFunction";
-import { timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
 import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isServer } from "../config2";
@@ -23,6 +23,8 @@ const LATENCY_POLL_INTERVAL = timeInMinute;
 const PING_TIMEOUT = timeInSecond * 5;
 // Rolling average size. Small, so latency changes show up quickly.
 const LATENCY_SAMPLE_LIMIT = 20;
+// Raw sample history kept per node, so callers can ask for a median over the last N with a known confidence.
+const LATENCY_HISTORY_LIMIT = 100;
 // Forget nodes we haven't been able to reach for this long.
 const NODE_EXPIRY_TIME = LATENCY_POLL_INTERVAL * 5;
 
@@ -30,6 +32,7 @@ export type NodeLatencyInfo = {
     averageLatency: number;
     sampleCount: number;
     lastSeen: number;
+    history: number[];
 };
 
 // otherNodeId => our rolling latency to it
@@ -42,6 +45,14 @@ export function getNodeLatencyInfo(nodeId: string): NodeLatencyInfo | undefined
 export function getNodeLatency(nodeId: string): number | undefined {
     return getNodeLatencyInfo(nodeId)?.averageLatency;
 }
+// Median of the last historyCount raw samples, plus how many samples were actually available — so callers can scale how much they trust the number.
+export function getNodeLatencyMedian(config: { nodeId: string; historyCount: number }): { latency: number; historyUsed: number } | undefined {
+    let history = getNodeLatencyInfo(config.nodeId)?.history;
+    if (!history || history.length === 0) return undefined;
+    let samples = history.slice(-config.historyCount);
+    sort(samples, x => x);
+    return { latency: samples[Math.floor(samples.length / 2)], historyUsed: samples.length };
+}
 
 export function getCachedNodeLatencyInfoList(): Map<string, NodeLatencyInfo> {
     return latencyByNode;
@@ -60,10 +71,16 @@ export function getOwnLatencies(): { [nodeId: string]: number } {
 function recordLatency(nodeId: string, latency: number) {
     let prev = latencyByNode.get(nodeId);
     let sampleCount = Math.min(prev?.sampleCount || 0, LATENCY_SAMPLE_LIMIT);
+    let history = prev?.history || [];
+    history.push(latency);
+    if (history.length > LATENCY_HISTORY_LIMIT) {
+        history.shift();
+    }
     latencyByNode.set(nodeId, {
         averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
         sampleCount: sampleCount + 1,
         lastSeen: Date.now(),
+        history,
     });
 }
 

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -20,7 +20,7 @@ import { formatNumber, formatPercent, formatTime } from "socket-function/src/for
 setFlag(require, "cbor-x", "allowclient", true);
 
 import * as pako from "pako";
-import { delay } from "socket-function/src/batching";
+import { delay, safeLoop } from "socket-function/src/batching";
 import { LZ4 } from "../storage/LZ4";
 import { unblockLoop } from "socket-function/src/batching";
 setFlag(require, "pako", "allowclient", true);
@@ -345,6 +345,7 @@ class PathValueSerializer {
         let encodedValues: (Buffer | number)[] = [];
 
         let i = 0;
+        let valuesSinceYield = 0;
         for (let values of valuesGroups) {
             measureBlock(() => {
                 // NOTE: Writing one value at a time is about 1.6X slower. BUT, it allows us to very efficient decode
@@ -378,21 +379,23 @@ class PathValueSerializer {
                     i++;
                 }
             }, "valuesWriteLoop");
-            await delay("paintLoop");
+            valuesSinceYield += values.length;
+            if (valuesSinceYield >= 10_000) {
+                valuesSinceYield = 0;
+                await delay("paintLoop");
+            }
         }
 
         measureBlock(function valuesWriteTypesBuffer() {
             writer.writeBuffer(types);
         });
-        // Break encodeValues into groups of 100, so we can check the time and delay
-        //      if we are taking too long, but not check it EVERY loop, as that would
-        //      be too slow.
+        // Break encodeValues into groups, and yield based on the amount of values written — NOT by measuring time, as timing every loop is itself too slow.
         let valueGroups: (Buffer | number)[][] = [];
         const VALUE_GROUP_SIZE = 1000;
         for (let i = 0; i < encodedValues.length; i += VALUE_GROUP_SIZE) {
             valueGroups.push(encodedValues.slice(i, i + VALUE_GROUP_SIZE));
         }
-        let prevTime = Date.now();
+        let encodedSinceYield = 0;
         for (let encodedValues of valueGroups) {
             measureBlock(function valuesWriteTypes() {
                 for (let value of encodedValues) {
@@ -403,10 +406,10 @@ class PathValueSerializer {
                     }
                 }
             });
-            let now = Date.now();
-            if (now - prevTime > 10) {
+            encodedSinceYield += encodedValues.length;
+            if (encodedSinceYield >= 10_000) {
+                encodedSinceYield = 0;
                 await delay("paintLoop");
-                prevTime = now;
             }
         }
     }
@@ -550,9 +553,14 @@ class PathValueSerializer {
         }
 
         startNewBuffer("pathValues");
+        let pathValuesSinceYield = 0;
         for (let values of valueGroups) {
             this.pathValuesWrite(writer, values, settings);
-            await delay("afterPaint");
+            pathValuesSinceYield += values.length;
+            if (pathValuesSinceYield >= 10_000) {
+                pathValuesSinceYield = 0;
+                await delay("afterPaint");
+            }
         }
 
         if (!settings.noLocks) {
@@ -593,15 +601,20 @@ class PathValueSerializer {
             curCount += str.length;
         }
 
+        let stringBytesSinceYield = 0;
         for (let strings of stringParts) {
             let stringBuffer = StringSerialize.serializeStrings(strings);
             settings.bufferMap!.push("strings");
             outputBuffers.push(stringBuffer);
-            await delay("paintLoop");
+            stringBytesSinceYield += stringBuffer.length;
+            if (stringBytesSinceYield >= 1_000_000) {
+                stringBytesSinceYield = 0;
+                await delay("paintLoop");
+            }
         }
 
         if (settings.compression === "lz4") {
-            let compressedBuffers = await unblockLoop(outputBuffers, x => LZ4.compress(x));
+            let compressedBuffers = await measureBlock(() => safeLoop({ data: outputBuffers, name: "PathValueSerializer.serialize|lz4" }, x => LZ4.compress(x)), "PathValueSerializer.serialize|lz4");
 
             // If the compress factor is less than a threshold, use the uncompressed buffers
             let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
@@ -723,6 +736,7 @@ class PathValueSerializer {
         }
         if (!config?.skipStrings) {
             let stringArrays: string[][] = [];
+            let bytesSinceYield = 0;
             for (let stringBuffer of stringBuffers) {
                 let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
                 while (true) {
@@ -731,11 +745,14 @@ class PathValueSerializer {
                         break;
                     }
                     stringArrays.push(nextStrings);
-                    if (stringArrays.length > 1) {
+                    // Each getNextStrings call decodes ~1MB, so on buffers past that we yield per chunk decoded.
+                    if (stringBuffer.length > 1_000_000) {
                         await delay("paintLoop");
                     }
                 }
-                if (stringBuffers.length > 1) {
+                bytesSinceYield += stringBuffer.length;
+                if (bytesSinceYield >= 1_000_000) {
+                    bytesSinceYield = 0;
                     await delay("paintLoop");
                 }
             }
@@ -848,6 +865,7 @@ class PathValueSerializer {
             }
 
             let stringArrays: string[][] = [];
+            let bytesSinceYield = 0;
             for (let bufferIndex of stringBufferIndexes) {
                 let stringBuffer = buffers[bufferIndex];
                 let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
@@ -857,13 +875,12 @@ class PathValueSerializer {
                         break;
                     }
                     stringArrays.push(nextStrings);
-                    if (stringArrays.length > 1) {
+                    bytesSinceYield += stringBuffer.length;
+                    if (bytesSinceYield >= 1_000_000) {
+                        bytesSinceYield = 0;
                         await delay("paintLoop");
                     }
                 }
-                if (stringBufferIndexes.length > 1) {
-                    await delay("paintLoop");
-                }
             }
             strings = stringArrays.flat();
         }

package/src/0-path-value-core/AuthorityLookup.ts

@@ -1,4 +1,4 @@
-import { timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
 import { nestArchives } from "../-a-archives/archives";
 import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
 import { archiveJSONT } from "../-a-archives/archivesJSONT";
@@ -31,7 +31,7 @@ let CALL_TIMEOUT = timeInSecond * 5;
 const SYNC_JITTER_WINDOW = timeInSecond * 10;
 
 // Nodes this close to their scheduled shutdown are avoided as sources (only used if nothing else can satisfy the request).
-const SHUTDOWN_AVOID_WINDOW = timeInMinute * 10;
+const SHUTDOWN_AVOID_WINDOW = timeInMinute * 5;
 
 export type AuthorityEntry = {
     nodeId: string;
@@ -82,6 +82,8 @@ class AuthorityLookup {
     public async setOurSpec(spec: AuthoritySpec) {
         if (!SocketFunction.isMounted()) throw new Error("Cannot call setOurPaths without mounting first (use Querysub.hostService).");
         spec.nodeId = getOwnNodeId();
+        // AuthoritySpec.prefixes promises to be sorted, and this is the choke point every published spec goes through.
+        sort(spec.prefixes, prefix => prefix.originalPrefix);
 
 
         if (this.setSpec) {

package/src/0-path-value-core/PathRouter.ts

@@ -5,7 +5,7 @@ import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
 import { fastHash } from "../misc/hash";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { getNodeLatency } from "../-f-node-discovery/LatencyTracking";
+import { getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
 import { unique } from "../misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
@@ -22,15 +22,18 @@ export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
 // fine (two nearby servers just talk to each other). Replaces the old uniform-random pick between candidates.
 const LATENCY_CANDIDATE_LIMIT = 5;
 const LATENCY_WEIGHT_OFFSET = 10;
-// A candidate whose weight is below (the current max weight / this factor) is dropped to zero probability. So a node
-// weighted 10x another (10x lower effective latency) is picked over it 100% of the time.
-const LATENCY_WEIGHT_CUTOFF_FACTOR = 10;
+// How many latency samples we use per node (median of the last N). The more we actually got, the more we trust the number, and the more aggressively we cut slow candidates.
+const LATENCY_HISTORY_COUNT = 10;
+// A candidate whose weight is below (the current max weight / (this - its historyUsed)) is dropped to zero probability. So with full history the factor is 2 (2x the effective latency of the best is never picked), and with a single sample it is 11 (only extreme outliers are cut).
+const LATENCY_CUTOFF_BASE_FACTOR = 12;
 // Latency assumed for a node we haven't measured yet, so new/unreached nodes stay eligible but aren't preferred.
 const UNKNOWN_LATENCY_MS = 50;
 
-function getRoutingLatency(nodeId: string): number {
-    if (isOwnNodeId(nodeId)) return 0;
-    return getNodeLatency(nodeId) ?? UNKNOWN_LATENCY_MS;
+function getRoutingLatency(nodeId: string): { latency: number; historyUsed: number } {
+    if (isOwnNodeId(nodeId)) return { latency: 0, historyUsed: LATENCY_HISTORY_COUNT };
+    let median = getNodeLatencyMedian({ nodeId, historyCount: LATENCY_HISTORY_COUNT });
+    if (!median) return { latency: UNKNOWN_LATENCY_MS, historyUsed: 0 };
+    return median;
 }
 
 // In-place reorder of candidates by a latency-weighted random draw over the lowest LATENCY_CANDIDATE_LIMIT
@@ -39,16 +42,22 @@ function getRoutingLatency(nodeId: string): number {
 //       since an overloaded server's latency spikes, which already makes it less likely to be chosen here.
 function latencyWeightedShuffle<T extends { nodeId: string }>(arr: T[]) {
     if (arr.length <= 1) return;
-    let withLatency = arr.map(x => ({ x, latency: getRoutingLatency(x.nodeId) }));
+    let withLatency = arr.map(x => {
+        let routingLatency = getRoutingLatency(x.nodeId);
+        return { x, latency: routingLatency.latency, historyUsed: routingLatency.historyUsed };
+    });
     sort(withLatency, e => e.latency);
     let pool = withLatency.slice(0, LATENCY_CANDIDATE_LIMIT);
     let rest = withLatency.slice(LATENCY_CANDIDATE_LIMIT);
     let ordered: T[] = [];
     while (pool.length > 0) {
         let weights = pool.map(e => 1 / (e.latency + LATENCY_WEIGHT_OFFSET));
-        let cutoff = Math.max(...weights) / LATENCY_WEIGHT_CUTOFF_FACTOR;
+        let maxWeight = Math.max(...weights);
         for (let i = 0; i < weights.length; i++) {
-            if (weights[i] < cutoff) weights[i] = 0;
+            let cutoffFactor = LATENCY_CUTOFF_BASE_FACTOR - Math.min(pool[i].historyUsed, LATENCY_HISTORY_COUNT);
+            if (weights[i] < maxWeight / cutoffFactor) {
+                weights[i] = 0;
+            }
         }
         let total = 0;
         for (let w of weights) {
@@ -90,6 +99,7 @@ export type AuthoritySpec = {
     routeEnd: number;
     // If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
     //  - For now, let's just never add overlapping prefixes.
+    //  - Sorted by originalPrefix (authorityLookup.setOurSpec sorts before publishing), so specs can be compared element-wise.
     prefixes: PrefixMatcher[];
     //  - Make sure to set this if you just want the prefix values, otherwise you will get all that that prefixes don't match (the prefixes exclude prefix match but where route does not match).
     excludeDefault?: boolean;
@@ -162,6 +172,18 @@ export function parsePrefixMatcher(prefixPath: string): PrefixMatcher {
     };
 }
 
+// Full-overlap equivalence: same range, same default matching, and the same prefixes (prefixes are sorted, so element-wise comparison works).
+export function areAuthoritySpecsEquivalent(a: AuthoritySpec, b: AuthoritySpec): boolean {
+    if (a.routeStart !== b.routeStart) return false;
+    if (a.routeEnd !== b.routeEnd) return false;
+    if (!a.excludeDefault !== !b.excludeDefault) return false;
+    if (a.prefixes.length !== b.prefixes.length) return false;
+    for (let [i, prefix] of a.prefixes.entries()) {
+        if (prefix.originalPrefix !== b.prefixes[i].originalPrefix) return false;
+    }
+    return true;
+}
+
 export function debugSpec(spec: AuthoritySpec) {
     return {
         info: `${spec.routeStart}-${spec.routeEnd} (${spec.prefixes.length} prefixes${spec.excludeDefault ? " excluding default" : ""})`,

package/src/1-path-client/RemoteWatcher.ts

@@ -6,7 +6,9 @@ import { isNode, sort, timeInMinute, timeInSecond } from "socket-function/src/mi
 import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { PathValueController } from "../0-path-value-core/PathValueController";
-import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
+import { AuthoritySpec, PathRouter, areAuthoritySpecsEquivalent } from "../0-path-value-core/PathRouter";
+import { getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
+import { formatTime } from "socket-function/src/formatting/format";
 import { WatchConfig, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { ActionsHistory } from "../diagnostics/ActionsHistory";
@@ -29,9 +31,15 @@ setImmediate(() => import("../4-querysub/Querysub"));
 const STOP_KEYS_DOUBLE_SENDS = true;
 
 // How close to an authority's scheduled shutdown we start treating it like a disconnect (rehoming all the paths we watch on it).
-const SHUTDOWN_REHOME_WINDOW = timeInMinute * 5;
+const SHUTDOWN_REHOME_WINDOW = timeInMinute * 3;
 const SHUTDOWN_REHOME_POLL_INTERVAL = timeInSecond * 30;
 
+// Rehome the watches on an authority when an equivalent authority has this factor lower latency. At 2 the router's full-confidence cutoff factor (also 2) then reliably routes everything to the better node on the rewatch.
+const LATENCY_REHOME_FACTOR = 2;
+// Both latencies must be medians over this many samples — with fewer we don't trust either number enough to move watches over it.
+const LATENCY_REHOME_HISTORY_COUNT = 10;
+const LATENCY_REHOME_POLL_INTERVAL = timeInMinute;
+
 // NOTE: If a parent watch is broken up between multiple nodes, we generally watch everything on all those nodes and then filter when we receive the data. This isn't efficient, and we should probably change it. However, in practice, it's probably fine, as it's unlikely for the watches to be broken up to a much finer granularity than the network (it would require very strange partially overlapping sharding cases which no reasonable sharding setup would ever satisfy).
 export class RemoteWatcher {
     public static DEBUG = false;
@@ -152,6 +160,40 @@ export class RemoteWatcher {
             }
         }
     }
+
+    private latencyRehomeLoop = runInfinitePoll(LATENCY_REHOME_POLL_INTERVAL, () => this.rehomeHighLatencyAuthorities());
+    private rehomeHighLatencyAuthorities() {
+        let authorityIds = new Set<string>(this.remoteWatchPaths.values());
+        for (let watchObj of this.remoteWatchParents2.values()) {
+            for (let range of watchObj.ranges) {
+                authorityIds.add(range.authorityId);
+            }
+        }
+        // Also guards getTopologySync — if we watch anything remotely, the authority lookup has synced.
+        if (authorityIds.size === 0) return;
+        let topology = authorityLookup.getTopologySync();
+        for (let authorityId of authorityIds) {
+            if (isOwnNodeId(authorityId)) continue;
+            let entry = topology.find(candidate => candidate.nodeId === authorityId);
+            if (!entry) continue;
+            let current = getNodeLatencyMedian({ nodeId: authorityId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
+            if (!current || current.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
+            for (let candidate of topology) {
+                if (candidate.nodeId === authorityId) continue;
+                if (isOwnNodeId(candidate.nodeId)) continue;
+                if (!candidate.isReady) continue;
+                if (authorityLookup.nodeIsShuttingDownSoon(candidate.nodeId)) continue;
+                if (!areAuthoritySpecsEquivalent(entry.authoritySpec, candidate.authoritySpec)) continue;
+                let candidateLatency = getNodeLatencyMedian({ nodeId: candidate.nodeId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
+                if (!candidateLatency || candidateLatency.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
+                if (candidateLatency.latency * LATENCY_REHOME_FACTOR > current.latency) continue;
+                console.log(yellow(`Authority ${authorityId} has high latency (${formatTime(current.latency)}), and the equivalent authority ${candidate.nodeId} is much faster (${formatTime(candidateLatency.latency)}), so we are rehoming all paths watched on it`));
+                logErrors(this.refreshAllWatches(authorityId));
+                break;
+            }
+        }
+    }
+
     private async tryToReconnectNow() {
         if (!this.disconnectedPaths.size && !this.disconnectedParents.size) return;
 
@@ -709,6 +751,8 @@ export class RemoteWatcher {
             }
             let paths = Array.from(pathsToWatch);
 
+            console.log(yellow(`Refreshing all watches on ${authorityNodeId}: ${pathsToWatch.size} paths, ${parentPathsToRewatch.size} parent paths (${parentRemotePathsToUnwatch.size} remote parent ranges)`));
+
             logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({
                 paths,
                 parentPaths: Array.from(parentRemotePathsToUnwatch)

package/src/4-querysub/FunctionRunnerTracking.ts

@@ -7,13 +7,13 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { timeInMinute, timeInSecond, sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
-import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isClient } from "../config2";
 import { t } from "../2-proxy/schema2";
 import { createLocalSchema } from "./schemaHelpers";
 import { Querysub } from "./Querysub";
 import { getAllNodeIds, getOwnNodeId, watchDeltaNodeIds } from "../-f-node-discovery/NodeDiscovery";
-import { getNodeLatencyInfo } from "../-f-node-discovery/LatencyTracking";
+import { getNodeLatencyInfo, getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
 import { FunctionRunnerInfoController, FunctionStatsSummary } from "../3-path-functions/PathFunctionRunner";
 import { URLParam } from "../library-components/URLParam";
 import { logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
@@ -37,6 +37,12 @@ const CALL_TIME_WEIGHT_BASE = 1000;
 // If every runner on a network will be shut down within this window, the network is dying and we switch off of it.
 const NETWORK_DYING_WINDOW = timeInMinute * 2;
 
+// Once selected we stick with a network — only switching when another network's best node has this factor lower latency than our network's best node (comparing medians, both with full history, so we never switch on noisy data).
+const NETWORK_SWITCH_LATENCY_FACTOR = 2;
+const NETWORK_SWITCH_HISTORY_COUNT = 10;
+// The latency data is already collected by LatencyTracking, so checking is cheap and can run often.
+const NETWORK_SWITCH_POLL_INTERVAL = timeInMinute;
+
 // The forced network is a URL parameter, so it survives reloads and is shareable / obvious in the URL.
 export const forcedNetworkURL = new URLParam("network", "");
 
@@ -306,11 +312,75 @@ export function getAutoSelectedNetwork(): string | undefined {
     return candidates[0]?.network;
 }
 
+// The lowest full-history median latency among the network's runners (the best node is what we compare networks by).
+function getNetworkBestNodeLatency(network: string): number | undefined {
+    let index = getFunctionRunnerIndex();
+    if (!index) return undefined;
+    let best: number | undefined;
+    for (let node of index.nodes) {
+        if (!node.networks.includes(network)) continue;
+        let median = getNodeLatencyMedian({ nodeId: node.nodeId, historyCount: NETWORK_SWITCH_HISTORY_COUNT });
+        if (!median || median.historyUsed < NETWORK_SWITCH_HISTORY_COUNT) continue;
+        if (best === undefined || median.latency < best) {
+            best = median.latency;
+        }
+    }
+    return best;
+}
+
+let stickyNetwork: string | undefined;
+function checkNetworkSwitch() {
+    if (!stickyNetwork) return;
+    let currentLatency = getNetworkBestNodeLatency(stickyNetwork);
+    if (currentLatency === undefined) return;
+
+    let candidates = getNetworkSelectionInfos().filter(x => !x.dying && x.network !== stickyNetwork);
+    if (isPublic()) {
+        candidates = candidates.filter(x => x.isPublic);
+    }
+    let upLongEnough = candidates.filter(x => x.upLongEnough);
+    if (upLongEnough.length > 0) {
+        candidates = upLongEnough;
+    }
+
+    let bestNetwork: string | undefined;
+    let bestLatency: number | undefined;
+    for (let candidate of candidates) {
+        let latency = getNetworkBestNodeLatency(candidate.network);
+        if (latency === undefined) continue;
+        if (bestLatency === undefined || latency < bestLatency) {
+            bestLatency = latency;
+            bestNetwork = candidate.network;
+        }
+    }
+    if (bestNetwork === undefined || bestLatency === undefined) return;
+    if (bestLatency * NETWORK_SWITCH_LATENCY_FACTOR > currentLatency) return;
+
+    console.log(yellow(`Switching the selected function network from ${stickyNetwork} (best node latency ${formatTime(currentLatency)}) to ${bestNetwork} (best node latency ${formatTime(bestLatency)}), so all new function calls use the faster network`));
+    stickyNetwork = bestNetwork;
+}
+const startNetworkSwitchPoll = lazy(() => {
+    logErrors(runInfinitePoll(NETWORK_SWITCH_POLL_INTERVAL, checkNetworkSwitch));
+});
+
+// Sticky: the score-based pick only initializes (or replaces a dead/dying selection); after that only checkNetworkSwitch changes it, so calls don't bounce between networks on small score changes.
+function getStickyNetwork(): string | undefined {
+    void startNetworkSwitchPoll();
+    if (stickyNetwork) {
+        let info = getNetworkSelectionInfos().find(x => x.network === stickyNetwork);
+        if (info && !info.dying && (!isPublic() || info.isPublic)) {
+            return stickyNetwork;
+        }
+    }
+    stickyNetwork = getAutoSelectedNetwork();
+    return stickyNetwork;
+}
+
 /** Safe to call from any context. Returns the network new function calls should be put on. Throws if no network is available (calls without a network can never run), unless noThrow is set. */
 export function getSelectedNetwork(): string;
 export function getSelectedNetwork(config: { noThrow: boolean }): string | undefined;
 export function getSelectedNetwork(config?: { noThrow?: boolean }): string | undefined {
-    let network = forcedNetworkURL.value || getAutoSelectedNetwork();
+    let network = forcedNetworkURL.value || getStickyNetwork();
     if (!network && !config?.noThrow) {
         throw new Error(`No function runner discovered during discovery, so this call cannot be given a network and would never run (we keep polling for function runners, every ${formatTime(POLL_INTERVAL)} when tracking, every ${formatTime(INDEX_POLL_INTERVAL)} for the client index)`);
     }

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -433,6 +433,35 @@ class NodeInfoTable extends qreact.Component<{
     }
 }
 
+// Nodes that never reported their latencies — all we have is their nodeId (and possibly a DNS-resolved machine IP), so this is a much narrower table than NodeInfoTable, still grouped by machine.
+class UnresponsiveNodesTable extends qreact.Component<{ nodeIds: string[]; machineIp: Map<string, string> }> {
+    render() {
+        let rows = this.props.nodeIds.map(nodeId => ({ nodeId, machineId: machineIdOf(nodeId) }));
+        let sep = String.fromCharCode(1);
+        sort(rows, row => `${row.machineId}${sep}${threadLabel(row.nodeId)}`);
+        return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
+            <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
+                <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>IP</div>
+                <div className={css.width(96).flexShrink0.pad2(6).ellipsis}>Machine</div>
+                <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>Thread:Port</div>
+                <div className={css.width(520).flexShrink0.pad2(6).ellipsis}>Node Id</div>
+            </div>
+            {rows.map((row, i) => {
+                let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
+                return <div
+                    className={css.hbox(0).fillWidth.hsl(0, 0, 99)
+                        .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
+                >
+                    <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>{firstOfMachine && this.props.machineIp.get(row.machineId) || ""}</div>
+                    <div className={css.width(96).flexShrink0.pad2(6).ellipsis}>{firstOfMachine && row.machineId.slice(0, ID_CHARS) || ""}</div>
+                    <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>{threadLabel(row.nodeId)}</div>
+                    <div className={css.width(520).flexShrink0.pad2(6).ellipsis.colorhsl(0, 0, 45)} title={row.nodeId}>{row.nodeId}</div>
+                </div>;
+            })}
+        </div>;
+    }
+}
+
 class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: NodeAuthorityInfo[] }> {
     render() {
         let nodeIds = this.props.nodeIds;
@@ -625,11 +654,13 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
 
         // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
         // Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
-        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
+        let rows = nodeIds.filter(nodeId => respondedSet.has(nodeId)).map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
         let selected = selectedMachineParam.value;
         let sep = String.fromCharCode(1);
         sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
 
+        let unresponsiveNodeIds = nodeIds.filter(nodeId => !respondedSet.has(nodeId));
+
         return <div className={css.vbox(8).fillWidth}>
             <div className={css.hbox(14)}>
                 <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
@@ -656,6 +687,10 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             <div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
             <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
             <NodeInfoTable rows={rows} selectedMachine={selected} machineColumns={machineColumns} nodeMachineLatency={nodeMachineLatency} />
+            {unresponsiveNodeIds.length > 0 && <>
+                <h2 className={css.margin(0)}>Unresponsive Nodes ({unresponsiveNodeIds.length})</h2>
+                <UnresponsiveNodesTable nodeIds={unresponsiveNodeIds} machineIp={machineIp} />
+            </>}
         </div>;
     }
 }

package/src/diagnostics/periodic.ts

@@ -78,7 +78,8 @@ if (isNode()) {
         }
 
         if (data.toString().includes("\r")) {
-            logAll();
+            import("./watchdog").then(m => m.logUnfiltered());
+            //logAll();
         }
     });
 

package/src/diagnostics/watchdog.ts

@@ -116,6 +116,17 @@ function logProfileMeasuresTimingsNow() {
         thresholdInTable: 0
     });
 };
+export function logUnfiltered(depth = 2) {
+    let profile = measureObj.finish();
+    measureObj = startMeasure();
+    logMeasureTable(profile, {
+        name: `all logs at ${new Date().toLocaleString()}`,
+        mergeDepth: depth,
+        minTimeToLog: 0,
+        maxTableEntries: 10000000,
+        thresholdInTable: 0
+    });
+};
 
 
 registerPeriodic(logProfileMeasuresTimingsNow);

package/src/server.ts

@@ -1,3 +1,10 @@
+import { disableMeasurements } from "socket-function/src/profiling/measure";
+// NOTE: Profiling seems to make us use about twice as much memory, at least in the bad case where we have many tiny watchers.
+if (typeof document === "undefined" && (process.argv.includes("--noprofile") || process.argv.includes("--nprofile"))) {
+    disableMeasurements();
+}
+
+
 import "./forceProduction";
 import "./inject";