querysub 0.518.0 → 0.520.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.518.0",
+  "version": "0.520.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -70,7 +70,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.5",
+    "socket-function": "^1.2.11",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-f-node-discovery/LatencyTracking.ts

@@ -0,0 +1,116 @@
+/*
+    The single source of truth for node-to-node latency. Each server periodically pings every other node
+    (reusing NodeDiscovery's isAlive as a lightweight round-trip) and keeps a rolling-average latency per node.
+
+    Like FunctionRunnerTracking, this is PER SERVER, in memory (latency depends on who is measuring) and is NEVER
+    written to the database. Polls are spread out over the interval (spreadCallsOverTime) plus a small random jitter,
+    so we never hit every node at once.
+*/
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { timeInMinute, timeInSecond } from "socket-function/src/misc";
+import { lazy } from "socket-function/src/caching";
+import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { isServer } from "../config2";
+import { spreadCallsOverTime } from "../misc";
+import { logErrors, timeoutToUndefinedSilent } from "../errors";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { NodeDiscoveryController, getAllNodeIds, getOwnNodeId, isOwnNodeId } from "./NodeDiscovery";
+import { blue } from "socket-function/src/formatting/logColors";
+import { formatTime } from "socket-function/src/formatting/format";
+
+const LATENCY_POLL_INTERVAL = timeInMinute;
+const PING_TIMEOUT = timeInSecond * 5;
+// Rolling average size. Small, so latency changes show up quickly.
+const LATENCY_SAMPLE_LIMIT = 20;
+// Forget nodes we haven't been able to reach for this long.
+const NODE_EXPIRY_TIME = LATENCY_POLL_INTERVAL * 5;
+
+export type NodeLatencyInfo = {
+    averageLatency: number;
+    sampleCount: number;
+    lastSeen: number;
+};
+
+// otherNodeId => our rolling latency to it
+let latencyByNode = new Map<string, NodeLatencyInfo>();
+
+export function getNodeLatencyInfo(nodeId: string): NodeLatencyInfo | undefined {
+    void startLatencyTracking();
+    return latencyByNode.get(nodeId);
+}
+export function getNodeLatency(nodeId: string): number | undefined {
+    return getNodeLatencyInfo(nodeId)?.averageLatency;
+}
+
+/** Our measured latency to every node we have reached, as a plain map (for sending over the wire). */
+export function getOwnLatencies(): { [nodeId: string]: number } {
+    void startLatencyTracking();
+    let result: { [nodeId: string]: number } = {};
+    for (let [nodeId, info] of latencyByNode) {
+        result[nodeId] = info.averageLatency;
+    }
+    return result;
+}
+
+function recordLatency(nodeId: string, latency: number) {
+    let prev = latencyByNode.get(nodeId);
+    let sampleCount = Math.min(prev?.sampleCount || 0, LATENCY_SAMPLE_LIMIT);
+    latencyByNode.set(nodeId, {
+        averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
+        sampleCount: sampleCount + 1,
+        lastSeen: Date.now(),
+    });
+}
+
+async function pingNode(nodeId: string) {
+    // A little extra jitter on top of the spread, so pings don't line up across nodes.
+    await delay(Math.random() * 2000);
+    let start = Date.now();
+    let alive = await timeoutToUndefinedSilent(PING_TIMEOUT, NodeDiscoveryController.nodes[nodeId].isAlive());
+    if (!alive) return;
+    recordLatency(nodeId, Date.now() - start);
+}
+
+async function pollLatencies() {
+    let nodeIds = (await getAllNodeIds()).filter(nodeId => !isOwnNodeId(nodeId));
+    await spreadCallsOverTime(nodeIds, LATENCY_POLL_INTERVAL, pingNode);
+
+    let now = Date.now();
+    for (let [nodeId, info] of Array.from(latencyByNode)) {
+        if (info.lastSeen < now - NODE_EXPIRY_TIME) {
+            latencyByNode.delete(nodeId);
+        }
+    }
+}
+
+export const startLatencyTracking = lazy(async () => {
+    if (!isServer()) return;
+    console.log(blue(`Starting latency tracking (pinging every node every ${formatTime(LATENCY_POLL_INTERVAL)})`));
+    await runInfinitePollCallAtStart(LATENCY_POLL_INTERVAL, pollLatencies);
+});
+
+class LatencyControllerBase {
+    /** This node's measured latency to every other node it has reached. */
+    public async getLatencies(): Promise<{ [nodeId: string]: number }> {
+        return getOwnLatencies();
+    }
+    public async getOwnNodeId(): Promise<string> {
+        return getOwnNodeId();
+    }
+}
+export const LatencyController = SocketFunction.register(
+    "LatencyController-4c1f8a20-2e73-4f6b-9a1e-6b0d3c5a7e91",
+    new LatencyControllerBase(),
+    () => ({
+        getLatencies: { hooks: [requiresNetworkTrustHook] },
+        getOwnNodeId: { hooks: [requiresNetworkTrustHook] },
+    }),
+    () => ({})
+);
+
+if (isServer()) {
+    setImmediate(() => {
+        logErrors(startLatencyTracking());
+    });
+}

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -156,6 +156,14 @@ export async function getAllNodeIds() {
     return Array.from(allNodeIds2);
 }
 
+/** Synchronous, best-effort: the last cached node list, without waiting for any sync. */
+export function getCachedNodeIds(): string[] {
+    if (nodeOverrides) {
+        return nodeOverrides;
+    }
+    return Array.from(allNodeIds2);
+}
+
 export async function syncNodesNow() {
     await syncArchives();
 }
@@ -680,7 +688,7 @@ class NodeDiscoveryControllerBase {
         return isNoNetwork();
     }
 }
-const NodeDiscoveryController = SocketFunction.register(
+export const NodeDiscoveryController = SocketFunction.register(
     "NodeDiscoveryController-7991037e-fd9e-4085-b1db-52035487e72c",
     new NodeDiscoveryControllerBase(),
     () => ({

package/src/-f-node-discovery/TrafficTracking.ts

@@ -0,0 +1,159 @@
+/*
+    Per-server, in-memory traffic counters (never written to the database). Everything is just an integer increment
+    into a 5-minute bucket; we keep the last 60 minutes (12 buckets) and drop older ones, so memory is bounded.
+
+    Byte traffic comes from SocketFunction.trackMessageSizes (upload/download per connection). The connection nodeId is
+    sometimes a raw client-connection id, so we map it to the nice node id (debugNodeId) lazily at read time — the hot
+    path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node buckets and a single
+    "outside the network" aggregate for anything not in our cached node list (clients, etc.).
+*/
+
+import { SocketFunction } from "socket-function/SocketFunction";
+import { timeInMinute } from "socket-function/src/misc";
+import { isClientNodeId } from "socket-function/src/nodeCache";
+import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
+import { getCachedNodeIds } from "./NodeDiscovery";
+import { debugNodeId } from "../-c-identity/IdentityController";
+
+const BUCKET_MS = timeInMinute * 5;
+const BUCKET_COUNT = 12;
+// Sentinel key for traffic that isn't a network node (client connections, etc.). Aggregated so memory stays O(nodes),
+// not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
+const OUTSIDE_KEY = "";
+
+export type NodeDataTraffic = { sent: number; received: number; };
+export type TrafficStats = {
+    functionsExecuted: number;
+    pathValuesSent: number;
+    pathValuesReceived: number;
+    querysubCalls: number;
+    // Bytes to/from each known network node, plus one aggregate for everything outside the node list.
+    perNode: { [nodeId: string]: NodeDataTraffic };
+    outside: NodeDataTraffic;
+};
+
+function currentBucket() {
+    return Math.floor(Date.now() / BUCKET_MS);
+}
+function oldestBucket() {
+    return currentBucket() - BUCKET_COUNT + 1;
+}
+
+// bucket => count
+let functionsExecuted = new Map<number, number>();
+let pathValuesSent = new Map<number, number>();
+let pathValuesReceived = new Map<number, number>();
+let querysubCalls = new Map<number, number>();
+// raw connection nodeId => (bucket => bytes)
+let bytesSent = new Map<string, Map<number, number>>();
+let bytesReceived = new Map<string, Map<number, number>>();
+
+function addSimple(buckets: Map<number, number>, amount: number) {
+    let bucket = currentBucket();
+    buckets.set(bucket, (buckets.get(bucket) || 0) + amount);
+}
+function addKeyed(map: Map<string, Map<number, number>>, key: string, amount: number) {
+    let buckets = map.get(key);
+    if (!buckets) {
+        buckets = new Map();
+        map.set(key, buckets);
+    }
+    addSimple(buckets, amount);
+}
+
+export function recordFunctionExecuted() {
+    addSimple(functionsExecuted, 1);
+}
+export function recordPathValuesSent(count: number) {
+    if (count <= 0) return;
+    addSimple(pathValuesSent, count);
+}
+export function recordPathValuesReceived(count: number) {
+    if (count <= 0) return;
+    addSimple(pathValuesReceived, count);
+}
+export function recordQuerysubCall() {
+    addSimple(querysubCalls, 1);
+}
+
+// The hot path is O(1): a client-id check (a string prefix test) collapses all non-network traffic into one key, and
+// everything else is keyed by the raw (server-style) id — bounded by node count — and resolved to a nice id at read.
+function trafficKey(nodeId: string): string {
+    return isClientNodeId(nodeId) ? OUTSIDE_KEY : nodeId;
+}
+SocketFunction.trackMessageSizes.upload.push((size, nodeId) => addKeyed(bytesSent, trafficKey(nodeId), size));
+SocketFunction.trackMessageSizes.download.push((size, nodeId) => addKeyed(bytesReceived, trafficKey(nodeId), size));
+
+function sumRecent(buckets: Map<number, number>): number {
+    let oldest = oldestBucket();
+    let total = 0;
+    for (let [bucket, count] of Array.from(buckets)) {
+        if (bucket < oldest) {
+            buckets.delete(bucket);
+            continue;
+        }
+        total += count;
+    }
+    return total;
+}
+
+function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
+    let known = new Set(getCachedNodeIds());
+    let perNode: { [nodeId: string]: NodeDataTraffic } = {};
+    let outside: NodeDataTraffic = { sent: 0, received: 0 };
+    // At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
+    // the rare server-style id that isn't already a known node.
+    let attribute = (rawNodeId: string, field: "sent" | "received", amount: number) => {
+        if (amount <= 0) return;
+        if (rawNodeId === OUTSIDE_KEY) {
+            outside[field] += amount;
+            return;
+        }
+        let nice = known.has(rawNodeId) ? rawNodeId : debugNodeId(rawNodeId);
+        if (known.has(nice)) {
+            let entry = perNode[nice];
+            if (!entry) {
+                entry = { sent: 0, received: 0 };
+                perNode[nice] = entry;
+            }
+            entry[field] += amount;
+        } else {
+            outside[field] += amount;
+        }
+    };
+    for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
+        attribute(rawNodeId, "sent", sumRecent(buckets));
+        if (buckets.size === 0) bytesSent.delete(rawNodeId);
+    }
+    for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
+        attribute(rawNodeId, "received", sumRecent(buckets));
+        if (buckets.size === 0) bytesReceived.delete(rawNodeId);
+    }
+    return { perNode, outside };
+}
+
+export function getTrafficStats(): TrafficStats {
+    let data = aggregateData();
+    return {
+        functionsExecuted: sumRecent(functionsExecuted),
+        pathValuesSent: sumRecent(pathValuesSent),
+        pathValuesReceived: sumRecent(pathValuesReceived),
+        querysubCalls: sumRecent(querysubCalls),
+        perNode: data.perNode,
+        outside: data.outside,
+    };
+}
+
+class TrafficControllerBase {
+    public async getTrafficStats(): Promise<TrafficStats> {
+        return getTrafficStats();
+    }
+}
+export const TrafficController = SocketFunction.register(
+    "TrafficController-9d2b7e14-6a3f-4c81-b5e2-0f7c9a1d3e64",
+    new TrafficControllerBase(),
+    () => ({
+        getTrafficStats: { hooks: [requiresNetworkTrustHook] },
+    }),
+    () => ({})
+);

package/src/0-path-value-core/PathRouter.ts

@@ -5,6 +5,7 @@ import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
 import { fastHash } from "../misc/hash";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { getNodeLatency } from "../-f-node-discovery/LatencyTracking";
 import { unique } from "../misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
@@ -16,6 +17,53 @@ import { getBufferInt } from "socket-function/src/bits";
 import { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH } from "./PathRouterConstants";
 export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
 
+// Latency-preferring routing: among candidate nodes, consider only the lowest few latencies and pick randomly,
+// weighted by 1/(latency + offset) — so 0ms is twice as likely as 10ms. One very close node can dominate, which is
+// fine (two nearby servers just talk to each other). Replaces the old uniform-random pick between candidates.
+const LATENCY_CANDIDATE_LIMIT = 5;
+const LATENCY_WEIGHT_OFFSET = 10;
+// Latency assumed for a node we haven't measured yet, so new/unreached nodes stay eligible but aren't preferred.
+const UNKNOWN_LATENCY_MS = 50;
+
+function getRoutingLatency(nodeId: string): number {
+    if (isOwnNodeId(nodeId)) return 0;
+    return getNodeLatency(nodeId) ?? UNKNOWN_LATENCY_MS;
+}
+
+// In-place reorder of candidates by a latency-weighted random draw over the lowest LATENCY_CANDIDATE_LIMIT
+// latencies, with the remaining nodes appended in latency order as fallback for range coverage.
+// TODO: Also weight by each server's load. For now we don't — but single-threaded servers self-limit somewhat,
+//       since an overloaded server's latency spikes, which already makes it less likely to be chosen here.
+function latencyWeightedShuffle<T extends { nodeId: string }>(arr: T[]) {
+    if (arr.length <= 1) return;
+    let withLatency = arr.map(x => ({ x, latency: getRoutingLatency(x.nodeId) }));
+    sort(withLatency, e => e.latency);
+    let pool = withLatency.slice(0, LATENCY_CANDIDATE_LIMIT);
+    let rest = withLatency.slice(LATENCY_CANDIDATE_LIMIT);
+    let ordered: T[] = [];
+    while (pool.length > 0) {
+        let total = 0;
+        for (let e of pool) {
+            total += 1 / (e.latency + LATENCY_WEIGHT_OFFSET);
+        }
+        let r = Math.random() * total;
+        let idx = 0;
+        while (idx < pool.length - 1) {
+            r -= 1 / (pool[idx].latency + LATENCY_WEIGHT_OFFSET);
+            if (r <= 0) break;
+            idx++;
+        }
+        ordered.push(pool[idx].x);
+        pool.splice(idx, 1);
+    }
+    for (let e of rest) {
+        ordered.push(e.x);
+    }
+    for (let i = 0; i < arr.length; i++) {
+        arr[i] = ordered[i];
+    }
+}
+
 // Cases
 // 1) Whole path hash
 // 2) Prefix + child override
@@ -496,7 +544,7 @@ export class PathRouter {
 
         let preferredNodeIds = new Set(config?.preferredNodeIds ?? []);
         for (let group of cohesiveGroups) {
-            shuffle(group, Math.random());
+            latencyWeightedShuffle(group);
             sort(group, x => (authorityLookup.nodeIsShuttingDownSoon(x.nodeId) && 2 || 0) + (preferredNodeIds.has(x.nodeId) && 0 || 1));
             let targetStart = target.routeStart;
             let targetEnd = target.routeEnd;
@@ -579,7 +627,7 @@ export class PathRouter {
         // Direct prefix. This happens for things like calls and functions, it requires more advanced routing as it means we're going to route between multiple servers, but... it is important
         let hasPrefix = allSources.filter(x => x.authoritySpec.prefixes.some(y => isPrefixParent(y, path))).map(x => x.authoritySpec);
         if (hasPrefix.length > 0) {
-            shuffle(hasPrefix, Math.random());
+            latencyWeightedShuffle(hasPrefix);
             sort(hasPrefix, x => (authorityLookup.nodeIsShuttingDownSoon(x.nodeId) && 2 || 0) + (preferredNodeIds.has(x.nodeId) && 0 || 1));
 
             let missingRanges: { start: number; end: number }[] = [{
@@ -621,7 +669,7 @@ export class PathRouter {
             );
         });
         if (nestedMatches.length > 0) {
-            shuffle(nestedMatches, Math.random());
+            latencyWeightedShuffle(nestedMatches);
             sort(nestedMatches, x => (authorityLookup.nodeIsShuttingDownSoon(x.nodeId) && 2 || 0) + (preferredNodeIds.has(x.nodeId) && 0 || 1));
             sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
             return {
@@ -640,7 +688,7 @@ export class PathRouter {
         });
         // Same as prefix matches. Not preferred, and not preferred over being under a prefix, but required for some root data, or data with no prefixes.
         if (fullPathMatches.length > 0) {
-            shuffle(fullPathMatches, Math.random());
+            latencyWeightedShuffle(fullPathMatches);
             sort(fullPathMatches, x => (authorityLookup.nodeIsShuttingDownSoon(x.nodeId) && 2 || 0) + (preferredNodeIds.has(x.nodeId) && 0 || 1));
             sort(allSources, x => isOwnNodeId(x.nodeId) ? -1 : 1);
             let missingRanges: { start: number; end: number }[] = [{
@@ -688,7 +736,7 @@ export class PathRouter {
     @measureFnc
     public static getReadyAuthority(path: string): AuthorityEntry | undefined {
         let candidates = authorityLookup.getTopologySync();
-        shuffle(candidates, Math.random());
+        latencyWeightedShuffle(candidates);
         // Only use nodes that are about to shut down if nothing else matches
         sort(candidates, x => authorityLookup.nodeIsShuttingDownSoon(x.nodeId) && 1 || 0);
         for (let candidate of candidates) {

package/src/0-path-value-core/PathValueController.ts

@@ -12,6 +12,7 @@ import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController"
 import { getSlowdown, isDiskAudit, getDomain } from "../config";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
+import { recordPathValuesSent, recordPathValuesReceived } from "../-f-node-discovery/TrafficTracking";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
 import { authorityLookup } from "./AuthorityLookup";
 import { timeoutToError } from "../errors";
@@ -110,6 +111,7 @@ export class PathValueControllerBase {
         let changes = config.pathValues;
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
+        recordPathValuesSent(changes.length);
         let buffers = await pathValueSerializer.serialize(changes, {
             noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
@@ -146,6 +148,7 @@ export class PathValueControllerBase {
         if (valueBuffers) {
             values = await pathValueSerializer.deserialize(valueBuffers);
             ActionsHistory.OnRead(values);
+            recordPathValuesReceived(values.length);
         }
 
         if (initialCreation) {

package/src/0-path-value-core/pathValueArchives.ts

@@ -199,7 +199,8 @@ export class PathValueArchives {
                 //  break existing servers (and it could cause security issues, etc).
                 if (time < maxAttemptArchiveTime) {
                     devDebugbreak();
-                    console.error(`Tried to archive a value which is too far into the past, ${time} < ${maxAttemptArchiveTime}. To commit this to disk would break locks, and could leave the database in an invalid state. Killing server so clients will resync with a server that is in a better state.`);
+                    let pastDistance = Date.now() - time;
+                    console.error(`Tried to archive a value which is too far into the past, ${time} < ${maxAttemptArchiveTime} (${formatTime(pastDistance)} ago, ${formatTime(maxAttemptArchiveTime - time)} beyond our limit). To commit this to disk would break locks, and could leave the database in an invalid state. Killing server so clients will resync with a server that is in a better state.`);
                     process.exit();
                 }
                 if (time < oldestTime) oldestTime = time;

package/src/0-path-value-core/startupAuthority.ts

@@ -34,7 +34,7 @@ export async function startupAuthority(spec: AuthoritySpec) {
             pathValues: values,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: true,
+            doNotArchive: new Set(values),
         });
         console.log(blue(`Finished ingesting values from source ${source.nodeId} (values: ${formatNumber(values.length)}`), { nodeId: source.nodeId, spec: debugSpec(source), values: values.length });
     }
@@ -62,7 +62,7 @@ export async function startupAuthority(spec: AuthoritySpec) {
         let flat = Object.values(snapshot.values).flat();
         console.log(blue(`Loaded snapshot in memory, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
         // NOTE: If it's on disk, it can't be rejected, so there's no point in passing it through our validStateComputer (and passing it through is VERY slow).
-        authorityStorage.ingestValues(flat, { doNotArchive: true });
+        authorityStorage.ingestValues(flat, { doNotArchive: new Set(flat) });
         console.log(blue(`Finished ingesting values from snapshot, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
     })();
 

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,4 +1,5 @@
 import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
+import { recordFunctionExecuted } from "../-f-node-discovery/TrafficTracking";
 import { cache, lazy } from "socket-function/src/caching";
 import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
 import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
@@ -645,6 +646,7 @@ export class PathFunctionRunner {
     private callStats = new Map<string, CallStats>();
 
     private async runCall(callSpec: CallSpec, functionSpec: FunctionSpec): Promise<void> {
+        recordFunctionExecuted();
         let callId = callSpec.CallId;
         let stats = this.callStats.get(callId);
         if (!stats) {

package/src/4-querysub/FunctionRunnerTracking.ts

@@ -13,6 +13,7 @@ import { t } from "../2-proxy/schema2";
 import { createLocalSchema } from "./schemaHelpers";
 import { Querysub } from "./Querysub";
 import { getAllNodeIds, getOwnNodeId, watchDeltaNodeIds } from "../-f-node-discovery/NodeDiscovery";
+import { getNodeLatencyInfo } from "../-f-node-discovery/LatencyTracking";
 import { FunctionRunnerInfoController, FunctionStatsSummary } from "../3-path-functions/PathFunctionRunner";
 import { URLParam } from "../library-components/URLParam";
 import { logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
@@ -27,8 +28,6 @@ const POLL_INTERVAL = timeInMinute;
 const INDEX_POLL_INTERVAL = timeInMinute * 5;
 // Forget runners we haven't been able to reach for this long
 const NODE_EXPIRY_TIME = POLL_INTERVAL * 5;
-// Rolling latency average size. Small, so latency changes show up quickly.
-const LATENCY_SAMPLE_LIMIT = 20;
 // Only prefer runners that have been up at least this long
 const MIN_UP_TIME = timeInMinute * 5;
 // Only weight by call time once a network has at least this much total recorded call time
@@ -179,7 +178,6 @@ async function pollFunctionRunner(nodeId: string, logWarnings?: "logWarnings" |
     // Wait a bit so the node has time to set itself up, also to stagger the polling a little bit more.
     await delay(2000 + Math.random() * 3000);
     polledNodeIds.add(nodeId);
-    let start = Date.now();
     // Only function runners expose this, so a failed call just means the node isn't a runner
     let runnerInfoPromise = FunctionRunnerInfoController.nodes[nodeId].getRunnerInfo({
         writeNodeId: getOwnNodeId(),
@@ -198,7 +196,6 @@ async function pollFunctionRunner(nodeId: string, logWarnings?: "logWarnings" |
         }
         return;
     }
-    let latency = Date.now() - start;
 
     let networks = new Set<string>();
     for (let shard of runnerInfo.shards) {
@@ -207,11 +204,11 @@ async function pollFunctionRunner(nodeId: string, logWarnings?: "logWarnings" |
         }
     }
 
-    let prev = nodeInfos.get(nodeId);
-    let sampleCount = Math.min(prev?.latencySampleCount || 0, LATENCY_SAMPLE_LIMIT);
     if (!nodeInfos.has(nodeId)) {
         console.log(green(`[${formatDateTime(Date.now())}] Function runner ${nodeId} discovered, adding it to the runner index`), { nodeId });
     }
+    // Latency is now owned by LatencyTracking (the single source of truth), not measured here.
+    let latencyInfo = getNodeLatencyInfo(nodeId);
     nodeInfos.set(nodeId, {
         nodeId,
         entryPoint: runnerInfo.entryPoint,
@@ -220,8 +217,8 @@ async function pollFunctionRunner(nodeId: string, logWarnings?: "logWarnings" |
         isPublic: runnerInfo.shards.some(x => x.isPublic),
         networks: Array.from(networks),
         shards: runnerInfo.shards,
-        averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
-        latencySampleCount: sampleCount + 1,
+        averageLatency: latencyInfo?.averageLatency ?? 0,
+        latencySampleCount: latencyInfo?.sampleCount ?? 0,
         scheduledShutdownTime: runnerInfo.scheduledShutdownTime,
     });
     nodeTimings.set(nodeId, runnerInfo.timings);

package/src/4-querysub/Querysub.ts

@@ -846,7 +846,8 @@ export class Querysub {
         }
 
         let mountedNodeId = await SocketFunction.mount({
-            public: isPublic(),
+            // IMPORTANT! It really breaks our network when there's nodes that want to be on the network but don't want to listen. So everything now has to be public and has to port forward. 
+            public: true,
             port: config.port,
             autoForwardPort: true,
             ...await getThreadKeyCert(getDomain()),
@@ -1081,7 +1082,8 @@ export class Querysub {
 
         this.socketFunctionInit();
         let mountPromise = SocketFunction.mount({
-            public: isPublic(),
+            // See hostServer for why this has to be always true
+            public: true,
             port,
             autoForwardPort: true,
             ...await getThreadKeyCert(getDomain()),

package/src/4-querysub/QuerysubController.ts

@@ -1,5 +1,6 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { cache, lazy } from "socket-function/src/caching";
+import { recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
 import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
@@ -534,6 +535,7 @@ export class QuerysubControllerBase {
 
     // NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
     public async addCall(call: CallSpec) {
+        recordQuerysubCall();
         if (isBootstrapOnly()) throw new Error(`Cannot add calls to bootstrap only server`);
         if (Querysub.DEBUG_CALLS) {
             console.log(`[Querysub] addCall @${debugTime(call.runAtTime)}: ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);

package/src/4-querysub/querysubPrediction.ts

@@ -308,7 +308,7 @@ function predictCallBase(config: {
             pathValues: predictions.writes,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: true,
+            doNotArchive: new Set(predictions.writes),
         });
 
         if (Querysub.DEBUG_PREDICTIONS) {
@@ -324,19 +324,20 @@ function predictCallBase(config: {
         didCancel = true;
         if (!predictions) return;
         // Reject our predictions, as the call likely never got committed, so it will never be written
+        let rejectedWrites = predictions.writes.map(write => ({
+            path: write.path,
+            value: write.value,
+            locks: [],
+            lockCount: 0,
+            valid: false,
+            time: predictResultWrite.time,
+            isTransparent: false,
+        }));
         validStateComputer.ingestValuesAndValidStates({
-            pathValues: predictions.writes.map(write => ({
-                path: write.path,
-                value: write.value,
-                locks: [],
-                lockCount: 0,
-                valid: false,
-                time: predictResultWrite.time,
-                isTransparent: false,
-            })),
+            pathValues: rejectedWrites,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: true,
+            doNotArchive: new Set(rejectedWrites),
         });
     }
 

package/src/deployManager/components/MachineDetailPage.tsx

@@ -1,6 +1,6 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { qreact } from "../../4-dom/qreact";
-import { MACHINE_RESYNC_INTERVAL, MachineServiceController, ServiceConfig } from "../machineSchema";
+import { MACHINE_RESYNC_INTERVAL, MachineServiceController, ServiceConfig, getMachineIdList } from "../machineSchema";
 import { css } from "typesafecss";
 import { currentViewParam, selectedMachineIdParam, selectedServiceIdParam } from "../urlParams";
 import { formatNumber, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
@@ -34,7 +34,7 @@ export class MachineDetailPage extends qreact.Component {
         for (let serviceId of serviceList) {
             let serviceConfig = controller.getServiceConfig(serviceId);
             if (!serviceConfig?.parameters.deploy) continue;
-            if (serviceConfig && (serviceConfig.parameters.machineIds || []).includes(selectedMachineId)) {
+            if (serviceConfig && getMachineIdList(serviceConfig.parameters).includes(selectedMachineId)) {
                 relevantServiceConfigs.set(serviceId, serviceConfig);
             }
         }