querysub 0.524.0 → 0.526.0

package/src/0-path-value-core/PathValueController.ts

@@ -23,11 +23,18 @@ import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
 import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
 import { delay } from "socket-function/src/batching";
 import { isDefined } from "../misc";
+import { measureBlock } from "socket-function/src/profiling/measure";
 export { pathValueCommitter };
 
 let pathValueSendCount = 0;
 export function getPathValueSendCount() { return pathValueSendCount; }
 
+// Debug hook, called with every batch of values sendData receives, before any filtering or ingest — so tests can observe exactly when values arrive, without worrying about values being ignored or batched downstream.
+let debugOnSendData: ((values: PathValue[], sourceNodeId: string) => void) | undefined;
+export function setDebugOnSendData(callback: typeof debugOnSendData) {
+    debugOnSendData = callback;
+}
+
 // ONLY returns non-canGCValues that are valid
 export type AuditSnapshotEntry = {
     path: string;
@@ -53,7 +60,7 @@ export class PathValueControllerBase {
         let { pathValues, nodeId } = config;
 
         pathValueSendCount += pathValues.length;
-        let serializedValues = await pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK });
+        let serializedValues = await measureBlock(() => pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK }), "createValues|serialize");
         if (isDebugLogEnabled()) {
             for (let value of pathValues) {
                 auditLog("SEND CREATED VALUE", {
@@ -111,11 +118,11 @@ export class PathValueControllerBase {
         let changes = config.pathValues;
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
-        recordPathValuesSent(changes.length);
-        let buffers = await pathValueSerializer.serialize(changes, {
+        recordPathValuesSent({ count: changes.length, nodeId });
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(changes, {
             noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
-        });
+        }), "sendValues|serialize");
         this.logSendValues({ nodeId, pathValues: changes, initialTriggers, reason: config.reason });
         return await PathValueController.nodes[nodeId].sendData({
             valueBuffers: buffers,
@@ -137,7 +144,7 @@ export class PathValueControllerBase {
 
     }): Promise<void | "refused"> {
         let callerId = SocketFunction.getCaller().nodeId;
-        let { initialCreation, valueBuffers } = config;
+        const { initialCreation, valueBuffers } = config;
 
         let slowdown = getSlowdown();
         if (slowdown) {
@@ -146,28 +153,34 @@ export class PathValueControllerBase {
 
         let values: PathValue[] = [];
         if (valueBuffers) {
-            values = await pathValueSerializer.deserialize(valueBuffers);
+            values = await measureBlock(() => pathValueSerializer.deserialize(valueBuffers), "sendData|deserialize");
             ActionsHistory.OnRead(values);
-            recordPathValuesReceived(values.length);
+            recordPathValuesReceived({ count: values.length, nodeId: callerId });
+        }
+        if (debugOnSendData) {
+            debugOnSendData(values, callerId);
         }
 
         if (initialCreation) {
-            let sourceNodeId = debugNodeId(callerId);
-            let rejected = false;
-            let threshold = Date.now() - MAX_CHANGE_AGE;
-            for (let value of values) {
-                let pastThreshold = threshold - value.time.time;
-                if (pastThreshold > 0) {
-                    rejected = true;
-                    console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
-                        path: value.path,
-                        timeId: value.time.time,
-                        sourceNodeId,
-                        sourceNodeThreadId: decodeNodeId(sourceNodeId, getDomain())?.threadId,
-                        totalValueCount: values.length,
-                    });
+            let rejected = measureBlock(() => {
+                let sourceNodeId = debugNodeId(callerId);
+                let anyRejected = false;
+                let threshold = Date.now() - MAX_CHANGE_AGE;
+                for (let value of values) {
+                    let pastThreshold = threshold - value.time.time;
+                    if (pastThreshold > 0) {
+                        anyRejected = true;
+                        console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
+                            path: value.path,
+                            timeId: value.time.time,
+                            sourceNodeId,
+                            sourceNodeThreadId: decodeNodeId(sourceNodeId, getDomain())?.threadId,
+                            totalValueCount: values.length,
+                        });
+                    }
                 }
-            }
+                return anyRejected;
+            }, "sendData|ageCheck");
             if (rejected) {
                 return "refused";
             }
@@ -189,7 +202,7 @@ export class PathValueControllerBase {
         }
 
         // Note which values are genuinely new to us BEFORE ingesting. We only re-share new values with the other authorities below; if we already had a value we already shared it when we first received it, so re-sharing is redundant and can clobber a newer value with an older copy.
-        let valuesNewToUs = config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [];
+        let valuesNewToUs = measureBlock(() => config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [], "sendData|newToUsCheck");
 
         try {
             let initialTriggers = config.initialTriggers || { values: new Set(), parentPaths: new Set() };
@@ -205,7 +218,8 @@ export class PathValueControllerBase {
 
         if (config.initialCreation && valuesNewToUs.length > 0) {
             // Always shared the latest. If we don't have it, maybe something with ingestion delayed, we should still share it so we don't lose the value. If we haven't even ingested it, then it is our latest version of the value.
-            valuesNewToUs = valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value);
+            valuesNewToUs = measureBlock(() => valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value), "sendData|newToUsRemap");
+            // Not measured: this waits on network sends to the other authorities.
             await PathValueControllerBase.authorityShareValues({ pathValues: valuesNewToUs });
         }
     }
@@ -242,13 +256,13 @@ export class PathValueControllerBase {
                 auditLog("WATCH PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.watchPath({
+        measureBlock(() => pathWatcher.watchPath({
             nodeId: callerId,
             paths: config.paths,
             parentPaths: config.parentPaths,
             initialTrigger: true,
             fullHistory: config.fullHistory,
-        });
+        }), "watchLatest|watchPath");
     }
     public async unwatchLatest(config: WatchConfig) {
         let callerId = SocketFunction.getCaller().nodeId;
@@ -261,7 +275,7 @@ export class PathValueControllerBase {
                 auditLog("UNWATCHING PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" });
+        measureBlock(() => pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" }), "unwatchLatest|unwatchPath");
     }
 
     public static async getInitialValues(config: {
@@ -281,11 +295,11 @@ export class PathValueControllerBase {
         startTime: number;
         endTime: number;
     }): Promise<Buffer[]> {
-        let values = authorityStorage.getAllValues(config);
-        let buffers = await pathValueSerializer.serialize(values, {
+        let values = measureBlock(() => authorityStorage.getAllValues(config), "getInitialValues|getAllValues");
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(values, {
             noLocks: true,
             compress: getCompressNetwork(),
-        });
+        }), "getInitialValues|serialize");
         return buffers;
     }
 
@@ -304,23 +318,25 @@ export class PathValueControllerBase {
         let { spec } = config;
         let { compareTime } = await import("./pathValueCore");
 
-        let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
+        return measureBlock(() => {
+            let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
 
-        let pathToLatest = new Map<string, Time>();
-        for (let value of allValues) {
-            if (!value.valid) continue;
-            if (value.canGCValue) continue;
+            let pathToLatest = new Map<string, Time>();
+            for (let value of allValues) {
+                if (!value.valid) continue;
+                if (value.canGCValue) continue;
 
-            let existing = pathToLatest.get(value.path);
-            if (!existing || compareTime(value.time, existing) > 0) {
-                pathToLatest.set(value.path, value.time);
+                let existing = pathToLatest.get(value.path);
+                if (!existing || compareTime(value.time, existing) > 0) {
+                    pathToLatest.set(value.path, value.time);
+                }
             }
-        }
 
-        let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
+            let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
 
-        let encoded = encodeCborx(entries);
-        return LZ4.compress(encoded);
+            let encoded = encodeCborx(entries);
+            return LZ4.compress(encoded);
+        }, "getAuditSnapshot|local");
     }
 
 
@@ -333,19 +349,22 @@ export class PathValueControllerBase {
     }
 
     public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
-        let values: PathValue[] = [];
-        for (let entry of entries) {
-            let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
-            if (!value) continue;
-            if (!value.valid) continue;
-            if (value.isTransparent) continue;
-            if (value.canGCValue) continue;
-            values.push(value);
-        }
-        let buffers = await pathValueSerializer.serialize(values, {
+        let values = measureBlock(() => {
+            let matched: PathValue[] = [];
+            for (let entry of entries) {
+                let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
+                if (!value) continue;
+                if (!value.valid) continue;
+                if (value.isTransparent) continue;
+                if (value.canGCValue) continue;
+                matched.push(value);
+            }
+            return matched;
+        }, "getValuesByPathAndTime|gather");
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(values, {
             noLocks: true,
             compress: getCompressNetwork(),
-        });
+        }), "getValuesByPathAndTime|serialize");
         return buffers;
     }
 }
@@ -367,6 +386,7 @@ export const PathValueController = SocketFunction.register(
         hooks: [requiresNetworkTrustHook],
     }),
     {
-        noFunctionMeasure: !isNode(),
+        // Most of these functions spend their time waiting on batching or remote calls, so auto-measuring them is misleading — instead the local work inside them is wrapped in individual measureBlocks.
+        noFunctionMeasure: true,
     }
 );

package/src/0-path-value-core/ValidStateComputer.ts

@@ -2,7 +2,7 @@ import { keyByArray, binarySearchIndex } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { isDiskAudit } from "../config";
-import { getParentPathStr, getPathFromStr } from "../path";
+import { getParentPathStr, getPathFromStr, getPathStr4 } from "../path";
 import { auditLog } from "./auditLogs";
 import { PathRouter } from "./PathRouter";
 import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE, createMissingEpochValue, Time, MISSING_TRANSACTION_PART_TIMEOUT, isOurPrediction, DEFER_LOCK_WINDOW } from "./pathValueCore";
@@ -21,8 +21,9 @@ class ValidStateComputer {
         parentSyncs: { parentPath: string; sourceNodeId: string }[];
         initialTriggers: { values: Set<string>; parentPaths: Set<string> };
         doNotArchive?: boolean;
+        forceValidStates?: boolean;
     }) {
-        let { pathValues, parentSyncs, doNotArchive } = config;
+        let { pathValues, parentSyncs, doNotArchive, forceValidStates } = config;
         let initialTriggers = { ...config.initialTriggers, initialTriggerNonHistoryWatchers: new Set<string>() };
 
         // TODO: We might want to add back optimizations for "no watches and no locks"?
@@ -30,6 +31,13 @@ class ValidStateComputer {
 
         let now = Date.now();
 
+        let forcedValidStates = new Map<string, boolean>();
+        if (forceValidStates) {
+            for (let value of pathValues) {
+                forcedValidStates.set(getForcedValidStateKey(value), !!value.valid);
+            }
+        }
+
         authorityStorage.addParentSyncs(parentSyncs);
 
         // Dedup by (path, time): if multiple values for the same (path, time) arrive in this batch,
@@ -216,6 +224,7 @@ class ValidStateComputer {
                 Array.from(dependenciesChanged),
                 now,
                 initialPrevValidStates,
+                forcedValidStates,
             );
             let validStateChanged = result.changed;
             for (let d of result.deferred) {
@@ -319,6 +328,7 @@ class ValidStateComputer {
         valuePaths: PathValue[],
         now: number,
         prevValidStates: Map<PathValue, boolean | undefined>,
+        forcedValidStates?: Map<string, boolean>,
     ): { changed: PathValue[]; deferred: PathValue[] } {
         let changed: PathValue[] = [];
         let deferred: PathValue[] = [];
@@ -404,6 +414,11 @@ class ValidStateComputer {
 
             for (let pathValue of valueGroup) {
                 let prevValidState = prevValidStates.get(pathValue);
+                let newValid: boolean = valid;
+                let forcedValid = forcedValidStates?.get(getForcedValidStateKey(pathValue));
+                if (forcedValid !== undefined) {
+                    newValid = forcedValid;
+                }
                 /* We used to log these values until 2026 June 28th 6:30 am
                 if (valid && prevValidState === false) {
                     console.info(`Accepting value that was previously rejected`, {
@@ -428,31 +443,32 @@ class ValidStateComputer {
                 }
                 */
                 // IMPORTANT! This means if it didn't previously exist and it's presently rejected, we count that as a change, as it this is going from undefined to false. This is actually very useful, as a lot of places want to know if a write is rejected, so they can display it in the UI for the developer.
-                if (valid === prevValidState && pathValue.valid === valid) continue;
+                if (newValid === prevValidState && pathValue.valid === newValid) continue;
 
                 // NOTE: We might remove this logging later as it's pretty heavy, but right now we still have bugs somewhere here.
                 {
-                    console.info(`Changed valid state ${JSON.stringify(prevValidState)} to ${JSON.stringify(valid)} (and path is ${JSON.stringify(pathValue.valid)})`, {
+                    console.info(`Changed valid state ${JSON.stringify(prevValidState)} to ${JSON.stringify(newValid)} (and path is ${JSON.stringify(pathValue.valid)})`, {
                         path: pathValue.path,
                         timeId: pathValue.time.time,
                         timeIdFull: pathValue.time,
-                        valid,
+                        valid: newValid,
+                        forced: forcedValid !== undefined,
                         prevValidState,
                     });
                 }
 
                 changed.push(pathValue);
 
-                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it.
+                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it. Anyone who sets the valid state anywhere else will be absolutely fired.
                 // @ts-expect-error
-                pathValue.valid = valid;
+                pathValue.valid = newValid;
 
                 // HACK: There are times when we might be evaluating the same path value multiple times at once (ex, multiple initial syncs at once). In which case we might not equal the stored path value, so we need to forcefully update the valid state of it.
                 let storedPathValue = authorityStorage.getValueExactMaybeRejected(pathValue.path, pathValue.time);
                 // I think this is fine. I think it happens if we receive it multiple times due to sharding. It's a bit inefficient, but... SHOULD be fine.
                 if (storedPathValue && pathValue !== storedPathValue) {
                     // @ts-expect-error
-                    storedPathValue.valid = valid;
+                    storedPathValue.valid = newValid;
                 }
             }
         }
@@ -541,3 +557,7 @@ class ValidStateComputer {
     }
 }
 export const validStateComputer = new ValidStateComputer();
+
+function getForcedValidStateKey(pathValue: { time: Time; path: string }): string {
+    return getPathStr4(pathValue.path, String(pathValue.time.time), String(pathValue.time.version), String(pathValue.time.creatorId));
+}

package/src/1-path-client/RemoteWatcher.ts

@@ -6,7 +6,9 @@ import { isNode, sort, timeInMinute, timeInSecond } from "socket-function/src/mi
 import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { PathValueController } from "../0-path-value-core/PathValueController";
-import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
+import { AuthoritySpec, PathRouter, areAuthoritySpecsEquivalent } from "../0-path-value-core/PathRouter";
+import { getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
+import { formatTime } from "socket-function/src/formatting/format";
 import { WatchConfig, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { ActionsHistory } from "../diagnostics/ActionsHistory";
@@ -29,9 +31,15 @@ setImmediate(() => import("../4-querysub/Querysub"));
 const STOP_KEYS_DOUBLE_SENDS = true;
 
 // How close to an authority's scheduled shutdown we start treating it like a disconnect (rehoming all the paths we watch on it).
-const SHUTDOWN_REHOME_WINDOW = timeInMinute * 5;
+const SHUTDOWN_REHOME_WINDOW = timeInMinute * 3;
 const SHUTDOWN_REHOME_POLL_INTERVAL = timeInSecond * 30;
 
+// Rehome the watches on an authority when an equivalent authority has this factor lower latency. At 2 the router's full-confidence cutoff factor (also 2) then reliably routes everything to the better node on the rewatch.
+const LATENCY_REHOME_FACTOR = 2;
+// Both latencies must be medians over this many samples — with fewer we don't trust either number enough to move watches over it.
+const LATENCY_REHOME_HISTORY_COUNT = 10;
+const LATENCY_REHOME_POLL_INTERVAL = timeInMinute;
+
 // NOTE: If a parent watch is broken up between multiple nodes, we generally watch everything on all those nodes and then filter when we receive the data. This isn't efficient, and we should probably change it. However, in practice, it's probably fine, as it's unlikely for the watches to be broken up to a much finer granularity than the network (it would require very strange partially overlapping sharding cases which no reasonable sharding setup would ever satisfy).
 export class RemoteWatcher {
     public static DEBUG = false;
@@ -152,6 +160,40 @@ export class RemoteWatcher {
             }
         }
     }
+
+    private latencyRehomeLoop = runInfinitePoll(LATENCY_REHOME_POLL_INTERVAL, () => this.rehomeHighLatencyAuthorities());
+    private rehomeHighLatencyAuthorities() {
+        let authorityIds = new Set<string>(this.remoteWatchPaths.values());
+        for (let watchObj of this.remoteWatchParents2.values()) {
+            for (let range of watchObj.ranges) {
+                authorityIds.add(range.authorityId);
+            }
+        }
+        // Also guards getTopologySync — if we watch anything remotely, the authority lookup has synced.
+        if (authorityIds.size === 0) return;
+        let topology = authorityLookup.getTopologySync();
+        for (let authorityId of authorityIds) {
+            if (isOwnNodeId(authorityId)) continue;
+            let entry = topology.find(candidate => candidate.nodeId === authorityId);
+            if (!entry) continue;
+            let current = getNodeLatencyMedian({ nodeId: authorityId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
+            if (!current || current.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
+            for (let candidate of topology) {
+                if (candidate.nodeId === authorityId) continue;
+                if (isOwnNodeId(candidate.nodeId)) continue;
+                if (!candidate.isReady) continue;
+                if (authorityLookup.nodeIsShuttingDownSoon(candidate.nodeId)) continue;
+                if (!areAuthoritySpecsEquivalent(entry.authoritySpec, candidate.authoritySpec)) continue;
+                let candidateLatency = getNodeLatencyMedian({ nodeId: candidate.nodeId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
+                if (!candidateLatency || candidateLatency.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
+                if (candidateLatency.latency * LATENCY_REHOME_FACTOR > current.latency) continue;
+                console.log(yellow(`Authority ${authorityId} has high latency (${formatTime(current.latency)}), and the equivalent authority ${candidate.nodeId} is much faster (${formatTime(candidateLatency.latency)}), so we are rehoming all paths watched on it`));
+                logErrors(this.refreshAllWatches(authorityId));
+                break;
+            }
+        }
+    }
+
     private async tryToReconnectNow() {
         if (!this.disconnectedPaths.size && !this.disconnectedParents.size) return;
 
@@ -656,6 +698,19 @@ export class RemoteWatcher {
         return Array.from(new Set(this.remoteWatchPaths.values()));
     }
 
+    public getWatchedPathCountsPerNodeId(): Map<string, number> {
+        let counts = new Map<string, number>();
+        for (let nodeId of this.remoteWatchPaths.values()) {
+            counts.set(nodeId, (counts.get(nodeId) || 0) + 1);
+        }
+        for (let watchObj of this.remoteWatchParents2.values()) {
+            for (let range of watchObj.ranges) {
+                counts.set(range.authorityId, (counts.get(range.authorityId) || 0) + 1);
+            }
+        }
+        return counts;
+    }
+
 
 
     public async refreshAllWatches(authorityNodeId: string) {
@@ -696,6 +751,8 @@ export class RemoteWatcher {
             }
             let paths = Array.from(pathsToWatch);
 
+            console.log(yellow(`Refreshing all watches on ${authorityNodeId}: ${pathsToWatch.size} paths, ${parentPathsToRewatch.size} parent paths (${parentRemotePathsToUnwatch.size} remote parent ranges)`));
+
             logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({
                 paths,
                 parentPaths: Array.from(parentRemotePathsToUnwatch)

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,5 +1,5 @@
 import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
-import { recordFunctionExecuted } from "../-f-node-discovery/TrafficTracking";
+import { recordFunctionExecuted, recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { cache, lazy } from "socket-function/src/caching";
 import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
 import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
@@ -56,6 +56,9 @@ setImmediate(() => {
 });
 
 export function commitCall(call: CallSpec) {
+    // Counted here (not in QuerysubController.addCall) so trusted nodes' direct writes are counted too — both the
+    // remote addCall path and the direct-write path funnel through here.
+    recordQuerysubCall();
     if (!call.network) {
         throw new Error(`Call has no network, so it would never run (no FunctionRunner would pick it up). Call: ${debugCallSpec(call)}`);
     }

package/src/4-deploy/edgeBootstrap.ts

@@ -357,7 +357,7 @@ async function edgeNodeFunction(config: {
         // Probes ALL the given nodes (even private / non-live ones), writing latencies to globalThis.EDGE_NODE_STATS (so the edge node dropdown can show them), and returns a latency-weighted random pick among `pickableHosts` (undefined if none respond).
         async function probeAndPick(probeNodes: EdgeNodeConfig[], pickableHosts: Set<string>): Promise<EdgeNodeConfig | undefined> {
             // Having this much lower latency than another node means we pick it 100% of the time (the weight scales linearly, from 1 at the best latency down to 0 at the best latency + this)
-            const LATENCY_WEIGHT_WINDOW = 1000;
+            const LATENCY_WEIGHT_WINDOW = 300;
             // How many probes are in flight at once. The first batch fires simultaneously, so anything that responds more than LATENCY_WEIGHT_WINDOW after the first response has a latency at least that much higher than it, and so would never have been picked anyway.
             const PROBE_BATCH_SIZE = 5;
 

package/src/4-querysub/FunctionRunnerTracking.ts

@@ -7,13 +7,13 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { timeInMinute, timeInSecond, sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
-import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isClient } from "../config2";
 import { t } from "../2-proxy/schema2";
 import { createLocalSchema } from "./schemaHelpers";
 import { Querysub } from "./Querysub";
 import { getAllNodeIds, getOwnNodeId, watchDeltaNodeIds } from "../-f-node-discovery/NodeDiscovery";
-import { getNodeLatencyInfo } from "../-f-node-discovery/LatencyTracking";
+import { getNodeLatencyInfo, getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
 import { FunctionRunnerInfoController, FunctionStatsSummary } from "../3-path-functions/PathFunctionRunner";
 import { URLParam } from "../library-components/URLParam";
 import { logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
@@ -37,6 +37,12 @@ const CALL_TIME_WEIGHT_BASE = 1000;
 // If every runner on a network will be shut down within this window, the network is dying and we switch off of it.
 const NETWORK_DYING_WINDOW = timeInMinute * 2;
 
+// Once selected we stick with a network — only switching when another network's best node has this factor lower latency than our network's best node (comparing medians, both with full history, so we never switch on noisy data).
+const NETWORK_SWITCH_LATENCY_FACTOR = 2;
+const NETWORK_SWITCH_HISTORY_COUNT = 10;
+// The latency data is already collected by LatencyTracking, so checking is cheap and can run often.
+const NETWORK_SWITCH_POLL_INTERVAL = timeInMinute;
+
 // The forced network is a URL parameter, so it survives reloads and is shareable / obvious in the URL.
 export const forcedNetworkURL = new URLParam("network", "");
 
@@ -306,11 +312,75 @@ export function getAutoSelectedNetwork(): string | undefined {
     return candidates[0]?.network;
 }
 
+// The lowest full-history median latency among the network's runners (the best node is what we compare networks by).
+function getNetworkBestNodeLatency(network: string): number | undefined {
+    let index = getFunctionRunnerIndex();
+    if (!index) return undefined;
+    let best: number | undefined;
+    for (let node of index.nodes) {
+        if (!node.networks.includes(network)) continue;
+        let median = getNodeLatencyMedian({ nodeId: node.nodeId, historyCount: NETWORK_SWITCH_HISTORY_COUNT });
+        if (!median || median.historyUsed < NETWORK_SWITCH_HISTORY_COUNT) continue;
+        if (best === undefined || median.latency < best) {
+            best = median.latency;
+        }
+    }
+    return best;
+}
+
+let stickyNetwork: string | undefined;
+function checkNetworkSwitch() {
+    if (!stickyNetwork) return;
+    let currentLatency = getNetworkBestNodeLatency(stickyNetwork);
+    if (currentLatency === undefined) return;
+
+    let candidates = getNetworkSelectionInfos().filter(x => !x.dying && x.network !== stickyNetwork);
+    if (isPublic()) {
+        candidates = candidates.filter(x => x.isPublic);
+    }
+    let upLongEnough = candidates.filter(x => x.upLongEnough);
+    if (upLongEnough.length > 0) {
+        candidates = upLongEnough;
+    }
+
+    let bestNetwork: string | undefined;
+    let bestLatency: number | undefined;
+    for (let candidate of candidates) {
+        let latency = getNetworkBestNodeLatency(candidate.network);
+        if (latency === undefined) continue;
+        if (bestLatency === undefined || latency < bestLatency) {
+            bestLatency = latency;
+            bestNetwork = candidate.network;
+        }
+    }
+    if (bestNetwork === undefined || bestLatency === undefined) return;
+    if (bestLatency * NETWORK_SWITCH_LATENCY_FACTOR > currentLatency) return;
+
+    console.log(yellow(`Switching the selected function network from ${stickyNetwork} (best node latency ${formatTime(currentLatency)}) to ${bestNetwork} (best node latency ${formatTime(bestLatency)}), so all new function calls use the faster network`));
+    stickyNetwork = bestNetwork;
+}
+const startNetworkSwitchPoll = lazy(() => {
+    logErrors(runInfinitePoll(NETWORK_SWITCH_POLL_INTERVAL, checkNetworkSwitch));
+});
+
+// Sticky: the score-based pick only initializes (or replaces a dead/dying selection); after that only checkNetworkSwitch changes it, so calls don't bounce between networks on small score changes.
+function getStickyNetwork(): string | undefined {
+    void startNetworkSwitchPoll();
+    if (stickyNetwork) {
+        let info = getNetworkSelectionInfos().find(x => x.network === stickyNetwork);
+        if (info && !info.dying && (!isPublic() || info.isPublic)) {
+            return stickyNetwork;
+        }
+    }
+    stickyNetwork = getAutoSelectedNetwork();
+    return stickyNetwork;
+}
+
 /** Safe to call from any context. Returns the network new function calls should be put on. Throws if no network is available (calls without a network can never run), unless noThrow is set. */
 export function getSelectedNetwork(): string;
 export function getSelectedNetwork(config: { noThrow: boolean }): string | undefined;
 export function getSelectedNetwork(config?: { noThrow?: boolean }): string | undefined {
-    let network = forcedNetworkURL.value || getAutoSelectedNetwork();
+    let network = forcedNetworkURL.value || getStickyNetwork();
     if (!network && !config?.noThrow) {
         throw new Error(`No function runner discovered during discovery, so this call cannot be given a network and would never run (we keep polling for function runners, every ${formatTime(POLL_INTERVAL)} when tracking, every ${formatTime(INDEX_POLL_INTERVAL)} for the client index)`);
     }

package/src/4-querysub/QuerysubController.ts

@@ -1,6 +1,5 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { cache, lazy } from "socket-function/src/caching";
-import { recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
 import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
@@ -535,7 +534,6 @@ export class QuerysubControllerBase {
 
     // NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
     public async addCall(call: CallSpec) {
-        recordQuerysubCall();
         if (isBootstrapOnly()) throw new Error(`Cannot add calls to bootstrap only server`);
         if (Querysub.DEBUG_CALLS) {
             console.log(`[Querysub] addCall @${debugTime(call.runAtTime)}: ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -182,11 +182,11 @@ export class IndexedLogs<T> {
             };
             let writeBuffers = async (buffers: Buffer[]) => {
                 if (Date.now() > endTime) {
+                    await newStreamer();
                     let timeBlockObj = this.getTimeBlock(Date.now());
                     startTime = timeBlockObj.startTime;
                     endTime = timeBlockObj.endTime;
                     path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath(timeBlockObj);
-                    await newStreamer();
                 }
 
                 let maxSize = this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA;