querysub 0.524.0 → 0.525.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.524.0",
+  "version": "0.525.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -70,7 +70,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.12",
+    "socket-function": "^1.2.14",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-c-identity/IdentityController.ts

@@ -19,7 +19,7 @@ import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { red } from "socket-function/src/formatting/logColors";
 import { isNode } from "typesafecss";
 import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
-import { timeInMinute } from "socket-function/src/misc";
+import { timeInMinute, isIpDomain } from "socket-function/src/misc";
 import { isClient, isServer } from "../config2";
 import { getDomain } from "../config";
 
@@ -242,7 +242,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         certIssuer: issuer.cert.toString(),
         mountedPort: getNodeIdLocation(SocketFunction.mountedNodeId)?.port,
         debugEntryPoint: isServer() ? process.argv[1] : "browser",
-        clientIsNode: isServer(),
+        clientIsNode: isServer() && !isIpDomain(nodeId),
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(

package/src/-d-trust/NetworkTrust2.ts

@@ -111,7 +111,7 @@ export async function isNodeTrusted(nodeId: string) {
     return await isTrusted(machineId);
 }
 
-const loadServerCert = cache(async (machineId: string) => {
+export const loadServerCert = cache(async (machineId: string) => {
     // This cert isn't stored in the archives, but... this is fine?
     if (machineId === "127-0-0-1." + getDomain()) return;
     let certFile = await archives().get(machineId);
@@ -131,7 +131,7 @@ export const ensureWeAreTrusted = lazy(measureWrap(async () => {
     }
 }));
 
-async function loadTrustCerts(nodeId: string) {
+export async function loadTrustCerts(nodeId: string) {
     let location = getNodeIdLocation(nodeId);
     if (location) {
         let machineId = getMachineId(location.address, getDomain());

package/src/-f-node-discovery/LatencyTracking.ts

@@ -43,6 +43,10 @@ export function getNodeLatency(nodeId: string): number | undefined {
     return getNodeLatencyInfo(nodeId)?.averageLatency;
 }
 
+export function getCachedNodeLatencyInfoList(): Map<string, NodeLatencyInfo> {
+    return latencyByNode;
+}
+
 /** Our measured latency to every node we have reached, as a plain map (for sending over the wire). */
 export function getOwnLatencies(): { [nodeId: string]: number } {
     void startLatencyTracking();
@@ -72,9 +76,11 @@ async function pingNode(nodeId: string) {
     recordLatency(nodeId, Date.now() - start);
 }
 
+let firstCall = true;
 async function pollLatencies() {
     let nodeIds = (await getAllNodeIds()).filter(nodeId => !isOwnNodeId(nodeId));
-    await spreadCallsOverTime(nodeIds, LATENCY_POLL_INTERVAL, pingNode);
+    let spread = firstCall ? 0 : LATENCY_POLL_INTERVAL;
+    await spreadCallsOverTime(nodeIds, spread, pingNode);
 
     let now = Date.now();
     for (let [nodeId, info] of Array.from(latencyByNode)) {
@@ -82,6 +88,7 @@ async function pollLatencies() {
             latencyByNode.delete(nodeId);
         }
     }
+    firstCall = false;
 }
 
 export const startLatencyTracking = lazy(async () => {

package/src/-f-node-discovery/TrafficTracking.ts

@@ -26,14 +26,14 @@ const BUCKET_HISTORY = BUCKET_SIZE * 20;
 // not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
 const OUTSIDE_KEY = "";
 
-export type NodeDataTraffic = { sent: number; received: number; };
+export type NodeDataTraffic = { sent: number; received: number; pathValuesSent: number; pathValuesReceived: number; };
 // All numbers are PER-SECOND rates (see file header).
 export type TrafficStats = {
     functionsExecuted: number;
     pathValuesSent: number;
     pathValuesReceived: number;
     querysubCalls: number;
-    // Byte rates to/from each known network node, plus one aggregate for everything outside the node list.
+    // Byte and path value rates to/from each known network node, plus one aggregate for everything outside the node list.
     perNode: { [nodeId: string]: NodeDataTraffic };
     outside: NodeDataTraffic;
 };
@@ -48,6 +48,8 @@ let querysubCalls: BucketMap = new Map();
 // raw connection nodeId => buckets
 let bytesSent = new Map<string, BucketMap>();
 let bytesReceived = new Map<string, BucketMap>();
+let pathValuesSentPerNode = new Map<string, BucketMap>();
+let pathValuesReceivedPerNode = new Map<string, BucketMap>();
 
 function addSimple(buckets: BucketMap, amount: number) {
     let bucket = Math.floor(Date.now() / BUCKET_SIZE) * BUCKET_SIZE;
@@ -65,13 +67,15 @@ function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
 export function recordFunctionExecuted() {
     addSimple(functionsExecuted, 1);
 }
-export function recordPathValuesSent(count: number) {
-    if (count <= 0) return;
-    addSimple(pathValuesSent, count);
+export function recordPathValuesSent(config: { count: number; nodeId: string }) {
+    if (config.count <= 0) return;
+    addSimple(pathValuesSent, config.count);
+    addKeyed(pathValuesSentPerNode, trafficKey(config.nodeId), config.count);
 }
-export function recordPathValuesReceived(count: number) {
-    if (count <= 0) return;
-    addSimple(pathValuesReceived, count);
+export function recordPathValuesReceived(config: { count: number; nodeId: string }) {
+    if (config.count <= 0) return;
+    addSimple(pathValuesReceived, config.count);
+    addKeyed(pathValuesReceivedPerNode, trafficKey(config.nodeId), config.count);
 }
 export function recordQuerysubCall() {
     addSimple(querysubCalls, 1);
@@ -111,10 +115,10 @@ function bucketRate(buckets: BucketMap, windowSize: number): number {
 function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
     let known = new Set(getCachedNodeIds());
     let perNode: { [nodeId: string]: NodeDataTraffic } = {};
-    let outside: NodeDataTraffic = { sent: 0, received: 0 };
+    let outside: NodeDataTraffic = { sent: 0, received: 0, pathValuesSent: 0, pathValuesReceived: 0 };
     // At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
     // the rare server-style id that isn't already a known node.
-    let attribute = (rawNodeId: string, field: "sent" | "received", rate: number) => {
+    let attribute = (rawNodeId: string, field: keyof NodeDataTraffic, rate: number) => {
         if (rate <= 0) return;
         if (rawNodeId === OUTSIDE_KEY) {
             outside[field] += rate;
@@ -124,7 +128,7 @@ function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeD
         if (known.has(nice)) {
             let entry = perNode[nice];
             if (!entry) {
-                entry = { sent: 0, received: 0 };
+                entry = { sent: 0, received: 0, pathValuesSent: 0, pathValuesReceived: 0 };
                 perNode[nice] = entry;
             }
             entry[field] += rate;
@@ -132,14 +136,16 @@ function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeD
             outside[field] += rate;
         }
     };
-    for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
-        attribute(rawNodeId, "sent", bucketRate(buckets, windowSize));
-        if (buckets.size === 0) bytesSent.delete(rawNodeId);
-    }
-    for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
-        attribute(rawNodeId, "received", bucketRate(buckets, windowSize));
-        if (buckets.size === 0) bytesReceived.delete(rawNodeId);
-    }
+    let aggregateMap = (map: Map<string, BucketMap>, field: keyof NodeDataTraffic) => {
+        for (let [rawNodeId, buckets] of Array.from(map)) {
+            attribute(rawNodeId, field, bucketRate(buckets, windowSize));
+            if (buckets.size === 0) map.delete(rawNodeId);
+        }
+    };
+    aggregateMap(bytesSent, "sent");
+    aggregateMap(bytesReceived, "received");
+    aggregateMap(pathValuesSentPerNode, "pathValuesSent");
+    aggregateMap(pathValuesReceivedPerNode, "pathValuesReceived");
     return { perNode, outside };
 }
 

package/src/0-path-value-core/PathRouter.ts

@@ -22,6 +22,9 @@ export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
 // fine (two nearby servers just talk to each other). Replaces the old uniform-random pick between candidates.
 const LATENCY_CANDIDATE_LIMIT = 5;
 const LATENCY_WEIGHT_OFFSET = 10;
+// A candidate whose weight is below (the current max weight / this factor) is dropped to zero probability. So a node
+// weighted 10x another (10x lower effective latency) is picked over it 100% of the time.
+const LATENCY_WEIGHT_CUTOFF_FACTOR = 10;
 // Latency assumed for a node we haven't measured yet, so new/unreached nodes stay eligible but aren't preferred.
 const UNKNOWN_LATENCY_MS = 50;
 
@@ -42,14 +45,19 @@ function latencyWeightedShuffle<T extends { nodeId: string }>(arr: T[]) {
     let rest = withLatency.slice(LATENCY_CANDIDATE_LIMIT);
     let ordered: T[] = [];
     while (pool.length > 0) {
+        let weights = pool.map(e => 1 / (e.latency + LATENCY_WEIGHT_OFFSET));
+        let cutoff = Math.max(...weights) / LATENCY_WEIGHT_CUTOFF_FACTOR;
+        for (let i = 0; i < weights.length; i++) {
+            if (weights[i] < cutoff) weights[i] = 0;
+        }
         let total = 0;
-        for (let e of pool) {
-            total += 1 / (e.latency + LATENCY_WEIGHT_OFFSET);
+        for (let w of weights) {
+            total += w;
         }
         let r = Math.random() * total;
         let idx = 0;
         while (idx < pool.length - 1) {
-            r -= 1 / (pool[idx].latency + LATENCY_WEIGHT_OFFSET);
+            r -= weights[idx];
             if (r <= 0) break;
             idx++;
         }

package/src/0-path-value-core/PathValueCommitter.ts

@@ -144,7 +144,7 @@ class PathValueCommitter {
     }
 
     private broadcastValues = batchFunction(
-        { delay: 10, throttleWindow: 500, noMeasure: true },
+        { delay: 1, throttleWindow: 2000, noMeasure: true },
         async function internal_forwardWrites(valuesBatched: {
             values: Set<PathValue>;
             tryCount?: number;
@@ -327,7 +327,7 @@ class PathValueCommitter {
 
 
     public ingestRemoteValuesAndValidStates = batchFunction(
-        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
+        { delay: 1, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
         async (batched: RemoteValueAndValidState[]) => {
             const { remoteWatcher } = await import("../1-path-client/RemoteWatcher");
 

package/src/0-path-value-core/PathValueController.ts

@@ -23,11 +23,18 @@ import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
 import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
 import { delay } from "socket-function/src/batching";
 import { isDefined } from "../misc";
+import { measureBlock } from "socket-function/src/profiling/measure";
 export { pathValueCommitter };
 
 let pathValueSendCount = 0;
 export function getPathValueSendCount() { return pathValueSendCount; }
 
+// Debug hook, called with every batch of values sendData receives, before any filtering or ingest — so tests can observe exactly when values arrive, without worrying about values being ignored or batched downstream.
+let debugOnSendData: ((values: PathValue[], sourceNodeId: string) => void) | undefined;
+export function setDebugOnSendData(callback: typeof debugOnSendData) {
+    debugOnSendData = callback;
+}
+
 // ONLY returns non-canGCValues that are valid
 export type AuditSnapshotEntry = {
     path: string;
@@ -53,7 +60,7 @@ export class PathValueControllerBase {
         let { pathValues, nodeId } = config;
 
         pathValueSendCount += pathValues.length;
-        let serializedValues = await pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK });
+        let serializedValues = await measureBlock(() => pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK }), "createValues|serialize");
         if (isDebugLogEnabled()) {
             for (let value of pathValues) {
                 auditLog("SEND CREATED VALUE", {
@@ -111,11 +118,11 @@ export class PathValueControllerBase {
         let changes = config.pathValues;
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
-        recordPathValuesSent(changes.length);
-        let buffers = await pathValueSerializer.serialize(changes, {
+        recordPathValuesSent({ count: changes.length, nodeId });
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(changes, {
             noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
-        });
+        }), "sendValues|serialize");
         this.logSendValues({ nodeId, pathValues: changes, initialTriggers, reason: config.reason });
         return await PathValueController.nodes[nodeId].sendData({
             valueBuffers: buffers,
@@ -137,7 +144,7 @@ export class PathValueControllerBase {
 
     }): Promise<void | "refused"> {
         let callerId = SocketFunction.getCaller().nodeId;
-        let { initialCreation, valueBuffers } = config;
+        const { initialCreation, valueBuffers } = config;
 
         let slowdown = getSlowdown();
         if (slowdown) {
@@ -146,28 +153,34 @@ export class PathValueControllerBase {
 
         let values: PathValue[] = [];
         if (valueBuffers) {
-            values = await pathValueSerializer.deserialize(valueBuffers);
+            values = await measureBlock(() => pathValueSerializer.deserialize(valueBuffers), "sendData|deserialize");
             ActionsHistory.OnRead(values);
-            recordPathValuesReceived(values.length);
+            recordPathValuesReceived({ count: values.length, nodeId: callerId });
+        }
+        if (debugOnSendData) {
+            debugOnSendData(values, callerId);
         }
 
         if (initialCreation) {
-            let sourceNodeId = debugNodeId(callerId);
-            let rejected = false;
-            let threshold = Date.now() - MAX_CHANGE_AGE;
-            for (let value of values) {
-                let pastThreshold = threshold - value.time.time;
-                if (pastThreshold > 0) {
-                    rejected = true;
-                    console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
-                        path: value.path,
-                        timeId: value.time.time,
-                        sourceNodeId,
-                        sourceNodeThreadId: decodeNodeId(sourceNodeId, getDomain())?.threadId,
-                        totalValueCount: values.length,
-                    });
+            let rejected = measureBlock(() => {
+                let sourceNodeId = debugNodeId(callerId);
+                let anyRejected = false;
+                let threshold = Date.now() - MAX_CHANGE_AGE;
+                for (let value of values) {
+                    let pastThreshold = threshold - value.time.time;
+                    if (pastThreshold > 0) {
+                        anyRejected = true;
+                        console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
+                            path: value.path,
+                            timeId: value.time.time,
+                            sourceNodeId,
+                            sourceNodeThreadId: decodeNodeId(sourceNodeId, getDomain())?.threadId,
+                            totalValueCount: values.length,
+                        });
+                    }
                 }
-            }
+                return anyRejected;
+            }, "sendData|ageCheck");
             if (rejected) {
                 return "refused";
             }
@@ -189,7 +202,7 @@ export class PathValueControllerBase {
         }
 
         // Note which values are genuinely new to us BEFORE ingesting. We only re-share new values with the other authorities below; if we already had a value we already shared it when we first received it, so re-sharing is redundant and can clobber a newer value with an older copy.
-        let valuesNewToUs = config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [];
+        let valuesNewToUs = measureBlock(() => config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [], "sendData|newToUsCheck");
 
         try {
             let initialTriggers = config.initialTriggers || { values: new Set(), parentPaths: new Set() };
@@ -205,7 +218,8 @@ export class PathValueControllerBase {
 
         if (config.initialCreation && valuesNewToUs.length > 0) {
             // Always shared the latest. If we don't have it, maybe something with ingestion delayed, we should still share it so we don't lose the value. If we haven't even ingested it, then it is our latest version of the value.
-            valuesNewToUs = valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value);
+            valuesNewToUs = measureBlock(() => valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value), "sendData|newToUsRemap");
+            // Not measured: this waits on network sends to the other authorities.
             await PathValueControllerBase.authorityShareValues({ pathValues: valuesNewToUs });
         }
     }
@@ -242,13 +256,13 @@ export class PathValueControllerBase {
                 auditLog("WATCH PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.watchPath({
+        measureBlock(() => pathWatcher.watchPath({
             nodeId: callerId,
             paths: config.paths,
             parentPaths: config.parentPaths,
             initialTrigger: true,
             fullHistory: config.fullHistory,
-        });
+        }), "watchLatest|watchPath");
     }
     public async unwatchLatest(config: WatchConfig) {
         let callerId = SocketFunction.getCaller().nodeId;
@@ -261,7 +275,7 @@ export class PathValueControllerBase {
                 auditLog("UNWATCHING PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" });
+        measureBlock(() => pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" }), "unwatchLatest|unwatchPath");
     }
 
     public static async getInitialValues(config: {
@@ -281,11 +295,11 @@ export class PathValueControllerBase {
         startTime: number;
         endTime: number;
     }): Promise<Buffer[]> {
-        let values = authorityStorage.getAllValues(config);
-        let buffers = await pathValueSerializer.serialize(values, {
+        let values = measureBlock(() => authorityStorage.getAllValues(config), "getInitialValues|getAllValues");
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(values, {
             noLocks: true,
             compress: getCompressNetwork(),
-        });
+        }), "getInitialValues|serialize");
         return buffers;
     }
 
@@ -304,23 +318,25 @@ export class PathValueControllerBase {
         let { spec } = config;
         let { compareTime } = await import("./pathValueCore");
 
-        let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
+        return measureBlock(() => {
+            let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
 
-        let pathToLatest = new Map<string, Time>();
-        for (let value of allValues) {
-            if (!value.valid) continue;
-            if (value.canGCValue) continue;
+            let pathToLatest = new Map<string, Time>();
+            for (let value of allValues) {
+                if (!value.valid) continue;
+                if (value.canGCValue) continue;
 
-            let existing = pathToLatest.get(value.path);
-            if (!existing || compareTime(value.time, existing) > 0) {
-                pathToLatest.set(value.path, value.time);
+                let existing = pathToLatest.get(value.path);
+                if (!existing || compareTime(value.time, existing) > 0) {
+                    pathToLatest.set(value.path, value.time);
+                }
             }
-        }
 
-        let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
+            let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
 
-        let encoded = encodeCborx(entries);
-        return LZ4.compress(encoded);
+            let encoded = encodeCborx(entries);
+            return LZ4.compress(encoded);
+        }, "getAuditSnapshot|local");
     }
 
 
@@ -333,19 +349,22 @@ export class PathValueControllerBase {
     }
 
     public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
-        let values: PathValue[] = [];
-        for (let entry of entries) {
-            let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
-            if (!value) continue;
-            if (!value.valid) continue;
-            if (value.isTransparent) continue;
-            if (value.canGCValue) continue;
-            values.push(value);
-        }
-        let buffers = await pathValueSerializer.serialize(values, {
+        let values = measureBlock(() => {
+            let matched: PathValue[] = [];
+            for (let entry of entries) {
+                let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
+                if (!value) continue;
+                if (!value.valid) continue;
+                if (value.isTransparent) continue;
+                if (value.canGCValue) continue;
+                matched.push(value);
+            }
+            return matched;
+        }, "getValuesByPathAndTime|gather");
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(values, {
             noLocks: true,
             compress: getCompressNetwork(),
-        });
+        }), "getValuesByPathAndTime|serialize");
         return buffers;
     }
 }
@@ -367,6 +386,7 @@ export const PathValueController = SocketFunction.register(
         hooks: [requiresNetworkTrustHook],
     }),
     {
-        noFunctionMeasure: !isNode(),
+        // Most of these functions spend their time waiting on batching or remote calls, so auto-measuring them is misleading — instead the local work inside them is wrapped in individual measureBlocks.
+        noFunctionMeasure: true,
     }
 );

package/src/0-path-value-core/ValidStateComputer.ts

@@ -2,7 +2,7 @@ import { keyByArray, binarySearchIndex } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { isDiskAudit } from "../config";
-import { getParentPathStr, getPathFromStr } from "../path";
+import { getParentPathStr, getPathFromStr, getPathStr4 } from "../path";
 import { auditLog } from "./auditLogs";
 import { PathRouter } from "./PathRouter";
 import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE, createMissingEpochValue, Time, MISSING_TRANSACTION_PART_TIMEOUT, isOurPrediction, DEFER_LOCK_WINDOW } from "./pathValueCore";
@@ -21,8 +21,9 @@ class ValidStateComputer {
         parentSyncs: { parentPath: string; sourceNodeId: string }[];
         initialTriggers: { values: Set<string>; parentPaths: Set<string> };
         doNotArchive?: boolean;
+        forceValidStates?: boolean;
     }) {
-        let { pathValues, parentSyncs, doNotArchive } = config;
+        let { pathValues, parentSyncs, doNotArchive, forceValidStates } = config;
         let initialTriggers = { ...config.initialTriggers, initialTriggerNonHistoryWatchers: new Set<string>() };
 
         // TODO: We might want to add back optimizations for "no watches and no locks"?
@@ -30,6 +31,13 @@ class ValidStateComputer {
 
         let now = Date.now();
 
+        let forcedValidStates = new Map<string, boolean>();
+        if (forceValidStates) {
+            for (let value of pathValues) {
+                forcedValidStates.set(getForcedValidStateKey(value), !!value.valid);
+            }
+        }
+
         authorityStorage.addParentSyncs(parentSyncs);
 
         // Dedup by (path, time): if multiple values for the same (path, time) arrive in this batch,
@@ -216,6 +224,7 @@ class ValidStateComputer {
                 Array.from(dependenciesChanged),
                 now,
                 initialPrevValidStates,
+                forcedValidStates,
             );
             let validStateChanged = result.changed;
             for (let d of result.deferred) {
@@ -319,6 +328,7 @@ class ValidStateComputer {
         valuePaths: PathValue[],
         now: number,
         prevValidStates: Map<PathValue, boolean | undefined>,
+        forcedValidStates?: Map<string, boolean>,
     ): { changed: PathValue[]; deferred: PathValue[] } {
         let changed: PathValue[] = [];
         let deferred: PathValue[] = [];
@@ -404,6 +414,11 @@ class ValidStateComputer {
 
             for (let pathValue of valueGroup) {
                 let prevValidState = prevValidStates.get(pathValue);
+                let newValid: boolean = valid;
+                let forcedValid = forcedValidStates?.get(getForcedValidStateKey(pathValue));
+                if (forcedValid !== undefined) {
+                    newValid = forcedValid;
+                }
                 /* We used to log these values until 2026 June 28th 6:30 am
                 if (valid && prevValidState === false) {
                     console.info(`Accepting value that was previously rejected`, {
@@ -428,31 +443,32 @@ class ValidStateComputer {
                 }
                 */
                 // IMPORTANT! This means if it didn't previously exist and it's presently rejected, we count that as a change, as it this is going from undefined to false. This is actually very useful, as a lot of places want to know if a write is rejected, so they can display it in the UI for the developer.
-                if (valid === prevValidState && pathValue.valid === valid) continue;
+                if (newValid === prevValidState && pathValue.valid === newValid) continue;
 
                 // NOTE: We might remove this logging later as it's pretty heavy, but right now we still have bugs somewhere here.
                 {
-                    console.info(`Changed valid state ${JSON.stringify(prevValidState)} to ${JSON.stringify(valid)} (and path is ${JSON.stringify(pathValue.valid)})`, {
+                    console.info(`Changed valid state ${JSON.stringify(prevValidState)} to ${JSON.stringify(newValid)} (and path is ${JSON.stringify(pathValue.valid)})`, {
                         path: pathValue.path,
                         timeId: pathValue.time.time,
                         timeIdFull: pathValue.time,
-                        valid,
+                        valid: newValid,
+                        forced: forcedValid !== undefined,
                         prevValidState,
                     });
                 }
 
                 changed.push(pathValue);
 
-                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it.
+                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it. Anyone who sets the valid state anywhere else will be absolutely fired.
                 // @ts-expect-error
-                pathValue.valid = valid;
+                pathValue.valid = newValid;
 
                 // HACK: There are times when we might be evaluating the same path value multiple times at once (ex, multiple initial syncs at once). In which case we might not equal the stored path value, so we need to forcefully update the valid state of it.
                 let storedPathValue = authorityStorage.getValueExactMaybeRejected(pathValue.path, pathValue.time);
                 // I think this is fine. I think it happens if we receive it multiple times due to sharding. It's a bit inefficient, but... SHOULD be fine.
                 if (storedPathValue && pathValue !== storedPathValue) {
                     // @ts-expect-error
-                    storedPathValue.valid = valid;
+                    storedPathValue.valid = newValid;
                 }
             }
         }
@@ -541,3 +557,7 @@ class ValidStateComputer {
     }
 }
 export const validStateComputer = new ValidStateComputer();
+
+function getForcedValidStateKey(pathValue: { time: Time; path: string }): string {
+    return getPathStr4(pathValue.path, String(pathValue.time.time), String(pathValue.time.version), String(pathValue.time.creatorId));
+}

package/src/1-path-client/RemoteWatcher.ts

@@ -656,6 +656,19 @@ export class RemoteWatcher {
         return Array.from(new Set(this.remoteWatchPaths.values()));
     }
 
+    public getWatchedPathCountsPerNodeId(): Map<string, number> {
+        let counts = new Map<string, number>();
+        for (let nodeId of this.remoteWatchPaths.values()) {
+            counts.set(nodeId, (counts.get(nodeId) || 0) + 1);
+        }
+        for (let watchObj of this.remoteWatchParents2.values()) {
+            for (let range of watchObj.ranges) {
+                counts.set(range.authorityId, (counts.get(range.authorityId) || 0) + 1);
+            }
+        }
+        return counts;
+    }
+
 
 
     public async refreshAllWatches(authorityNodeId: string) {

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,5 +1,5 @@
 import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
-import { recordFunctionExecuted } from "../-f-node-discovery/TrafficTracking";
+import { recordFunctionExecuted, recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { cache, lazy } from "socket-function/src/caching";
 import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
 import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
@@ -56,6 +56,9 @@ setImmediate(() => {
 });
 
 export function commitCall(call: CallSpec) {
+    // Counted here (not in QuerysubController.addCall) so trusted nodes' direct writes are counted too — both the
+    // remote addCall path and the direct-write path funnel through here.
+    recordQuerysubCall();
     if (!call.network) {
         throw new Error(`Call has no network, so it would never run (no FunctionRunner would pick it up). Call: ${debugCallSpec(call)}`);
     }

package/src/4-deploy/edgeBootstrap.ts

@@ -357,7 +357,7 @@ async function edgeNodeFunction(config: {
         // Probes ALL the given nodes (even private / non-live ones), writing latencies to globalThis.EDGE_NODE_STATS (so the edge node dropdown can show them), and returns a latency-weighted random pick among `pickableHosts` (undefined if none respond).
         async function probeAndPick(probeNodes: EdgeNodeConfig[], pickableHosts: Set<string>): Promise<EdgeNodeConfig | undefined> {
             // Having this much lower latency than another node means we pick it 100% of the time (the weight scales linearly, from 1 at the best latency down to 0 at the best latency + this)
-            const LATENCY_WEIGHT_WINDOW = 1000;
+            const LATENCY_WEIGHT_WINDOW = 300;
             // How many probes are in flight at once. The first batch fires simultaneously, so anything that responds more than LATENCY_WEIGHT_WINDOW after the first response has a latency at least that much higher than it, and so would never have been picked anyway.
             const PROBE_BATCH_SIZE = 5;
 

package/src/4-querysub/QuerysubController.ts

@@ -1,6 +1,5 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { cache, lazy } from "socket-function/src/caching";
-import { recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
 import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
@@ -535,7 +534,6 @@ export class QuerysubControllerBase {
 
     // NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
     public async addCall(call: CallSpec) {
-        recordQuerysubCall();
         if (isBootstrapOnly()) throw new Error(`Cannot add calls to bootstrap only server`);
         if (Querysub.DEBUG_CALLS) {
             console.log(`[Querysub] addCall @${debugTime(call.runAtTime)}: ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);

package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts

@@ -182,11 +182,11 @@ export class IndexedLogs<T> {
             };
             let writeBuffers = async (buffers: Buffer[]) => {
                 if (Date.now() > endTime) {
+                    await newStreamer();
                     let timeBlockObj = this.getTimeBlock(Date.now());
                     startTime = timeBlockObj.startTime;
                     endTime = timeBlockObj.endTime;
                     path = new TimeFileTree(this.getLocalLogs()).getNewPendingPath(timeBlockObj);
-                    await newStreamer();
                 }
 
                 let maxSize = this.config.maxSingleFileData || MAX_SINGLE_FILE_DATA;

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -20,6 +20,7 @@ import { getFunctionRunnerIndex, FunctionRunnerNodeInfo } from "../../4-querysub
 import { formatTime, formatNumber } from "socket-function/src/formatting/format";
 import { LatencyGraph, LatencyGraphNode, LatencyGraphLink, LatencyGraphLabelLine } from "../../library-components/LatencyGraph";
 import { URLParam } from "../../library-components/URLParam";
+import { mainResets } from "../../library-components/urlResetGroups";
 
 const ID_CHARS = 8;
 // Green means querysub, blue means path value, purple means function runner.
@@ -31,10 +32,11 @@ const PROBE_TIMEOUT_MS = 5000;
 const RANGE_BAR_WIDTH_PX = 360;
 const RANGE_BAR_HEIGHT_PX = 14;
 const LATENCY_GRAPH_HEIGHT_PX = 720;
+const MACHINE_LATENCY_WIDTH_PX = 235;
 
 // Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
-const selectedMachineParam = new URLParam("rtSelectedMachine", "");
-const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60);
+const selectedMachineParam = new URLParam("rtSelectedMachine", "", { reset: [mainResets] });
+const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60, { reset: [mainResets] });
 
 // The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
@@ -227,6 +229,43 @@ function machineIdOf(nodeId: string): string {
     return parts?.machineId || nodeId;
 }
 
+function threadLabel(nodeId: string): string {
+    let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
+    return `${(parts?.threadId || "?").slice(0, ID_CHARS)}:${parts?.port ?? "?"}`;
+}
+
+// min · median · max of one source node's latencies to a machine's threads (plus a tooltip naming the farthest
+// thread), and the node's total path values sent/received to that machine (summed — the per-thread split isn't interesting).
+function machineLatencyCell(config: {
+    sourceLatencies: { [nodeId: string]: number } | undefined;
+    threads: string[];
+    traffic: TrafficStats | undefined;
+}): { text: string; tooltip: string } {
+    let { sourceLatencies, threads, traffic } = config;
+    if (!sourceLatencies) return { text: "", tooltip: "" };
+    let entries: { thread: string; ms: number }[] = [];
+    for (let thread of threads) {
+        let ms = sourceLatencies[thread];
+        if (Number.isFinite(ms)) entries.push({ thread, ms });
+    }
+    if (!entries.length) return { text: "", tooltip: "" };
+    sort(entries, entry => entry.ms);
+    let farthest = entries[entries.length - 1];
+    let text = `${formatTime(entries[0].ms)} · ${formatTime(entries[Math.floor(entries.length / 2)].ms)} · ${formatTime(farthest.ms)}`;
+    let pvSent = 0;
+    let pvReceived = 0;
+    for (let thread of threads) {
+        let data = traffic?.perNode?.[thread];
+        if (!data) continue;
+        pvSent += data.pathValuesSent || 0;
+        pvReceived += data.pathValuesReceived || 0;
+    }
+    if (pvSent || pvReceived) {
+        text += `  ↑${formatNumber(pvSent)}/s ↓${formatNumber(pvReceived)}/s values`;
+    }
+    return { text, tooltip: `${text}\nfarthest: ${threadLabel(farthest.thread)} (${formatTime(farthest.ms)})` };
+}
+
 // Summed traffic metrics across a set of nodes (a machine's threads).
 function sumTraffic(threads: string[], trafficMaps: Map<string, TrafficStats>) {
     let dataSent = 0;
@@ -313,7 +352,7 @@ function buildNodeRow(config: {
     if (runner) {
         if (runner.networks.length) fnParts.push(runner.networks.join(" "));
         for (let shard of runner.shards) {
-            fnParts.push(`fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
+            fnParts.push(`${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
         }
         fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)}/s calls`);
         if (!runner.isPublic) fnParts.push("PRIVATE");
@@ -322,7 +361,7 @@ function buildNodeRow(config: {
     let pvParts: string[] = [];
     let spec = info?.spec;
     if (spec && spec.routeStart >= 0 && spec.routeEnd >= 0) {
-        pvParts.push(`pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
+        pvParts.push(`${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
     }
     if (traffic && (traffic.pathValuesSent || traffic.pathValuesReceived)) {
         pvParts.push(`↑${formatNumber(traffic.pathValuesSent)}/s ↓${formatNumber(traffic.pathValuesReceived)}/s values`);
@@ -344,19 +383,30 @@ function buildNodeRow(config: {
     };
 }
 
-class NodeInfoTable extends qreact.Component<{ rows: NodeRow[]; selectedMachine: string }> {
+class NodeInfoTable extends qreact.Component<{
+    rows: NodeRow[];
+    selectedMachine: string;
+    machineColumns: { id: string; label: string }[];
+    nodeMachineLatency: Map<string, { [machineId: string]: { text: string; tooltip: string } }>;
+}> {
     render() {
         let rows = this.props.rows;
         let selected = this.props.selectedMachine;
+        let machineColumns = this.props.machineColumns;
+        let nodeMachineLatency = this.props.nodeMachineLatency;
         return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
             <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
                 {NODE_TABLE_COLUMNS.map(col =>
                     <div className={css.width(col.width).flexShrink0.pad2(6).ellipsis}>{col.label}</div>
                 )}
+                {machineColumns.map(col =>
+                    <div className={css.width(MACHINE_LATENCY_WIDTH_PX).flexShrink0.pad2(6).ellipsis} title={`latency and path values to ${col.label}`}>→ {col.label}</div>
+                )}
             </div>
             {rows.map((row, i) => {
                 let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
                 let isSelected = row.machineId === selected;
+                let latencies = nodeMachineLatency.get(row.nodeId) || {};
                 return <div
                     className={css.hbox(0).button.fillWidth.hsl(0, 0, isSelected ? 92 : 99)
                         .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
@@ -370,6 +420,13 @@ class NodeInfoTable extends qreact.Component<{ rows: NodeRow[]; selectedMachine:
                             title={text}
                         >{text}</div>;
                     })}
+                    {machineColumns.map(col => {
+                        let cell = latencies[col.id];
+                        return <div
+                            className={css.width(MACHINE_LATENCY_WIDTH_PX).flexShrink0.pad2(6).ellipsis.colorhsl(0, 0, 25)}
+                            title={cell?.tooltip || ""}
+                        >{cell?.text || ""}</div>;
+                    })}
                 </div>;
             })}
         </div>;
@@ -453,19 +510,56 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             if (unique.size) machineNetworks.set(machineId, [...unique]);
         }
 
+        // One latency column per machine: each row (node) gets its own min · median · max latency to that machine's
+        // threads (a machine has several threads, so a cell is a range, not one number).
+        let machineColumns = [...machineAllThreads.keys()].map(id => {
+            let networks = machineNetworks.get(id) || [];
+            let label = networks.length ? `${id.slice(0, ID_CHARS)} (${networks.join(" | ")})` : id.slice(0, ID_CHARS);
+            return { id, label, threads: machineAllThreads.get(id) || [] };
+        });
+        sort(machineColumns, column => column.id);
+        let nodeMachineLatency = new Map<string, { [machineId: string]: { text: string; tooltip: string } }>();
+        for (let nodeId of nodeIds) {
+            let sourceLatencies = latencyMaps.get(nodeId);
+            let traffic = trafficMaps.get(nodeId);
+            let byMachine: { [machineId: string]: { text: string; tooltip: string } } = {};
+            for (let column of machineColumns) {
+                let cell = machineLatencyCell({ sourceLatencies, threads: column.threads, traffic });
+                if (cell.text) byMachine[column.id] = cell;
+            }
+            nodeMachineLatency.set(nodeId, byMachine);
+        }
+
         // Total bytes on each thread-pair link (both directions). Both endpoints report the same total, so take the max.
         let pairKey = (a: string, b: string) => a < b ? `${a}|${b}` : `${b}|${a}`;
         let pairTraffic = new Map<string, number>();
+        // Path values per thread pair, split by direction (fwd = lower id → higher id). Both endpoints report each
+        // direction (one as sent, one as received), so take the max of the two estimates.
+        let pairPv = new Map<string, { fwd: number; back: number }>();
         for (let [reporter, traffic] of trafficMaps) {
             for (let [peer, data] of Object.entries(traffic.perNode || {})) {
                 if (reporter === peer || !respondedSet.has(peer)) continue;
                 let key = pairKey(reporter, peer);
                 pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, (data?.sent || 0) + (data?.received || 0)));
+                let pv = pairPv.get(key);
+                if (!pv) {
+                    pv = { fwd: 0, back: 0 };
+                    pairPv.set(key, pv);
+                }
+                let sentRate = data?.pathValuesSent || 0;
+                let receivedRate = data?.pathValuesReceived || 0;
+                if (reporter < peer) {
+                    pv.fwd = Math.max(pv.fwd, sentRate);
+                    pv.back = Math.max(pv.back, receivedRate);
+                } else {
+                    pv.fwd = Math.max(pv.fwd, receivedRate);
+                    pv.back = Math.max(pv.back, sentRate);
+                }
             }
         }
 
-        // Machine-to-machine latency = mean over all cross-machine thread-pair latencies; traffic = summed pair traffic.
-        let machinePairLatency = new Map<string, { sum: number; count: number; }>();
+        // Machine-to-machine latency = minimum over all cross-machine thread-pair latencies; traffic = summed pair traffic.
+        let machinePairLatency = new Map<string, number>();
         for (let [sourceId, latencies] of latencyMaps) {
             let sourceMachine = machineIdOf(sourceId);
             for (let [destId, latencyMs] of Object.entries(latencies)) {
@@ -473,13 +567,8 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
                 let destMachine = machineIdOf(destId);
                 if (sourceMachine === destMachine) continue;
                 let key = pairKey(sourceMachine, destMachine);
-                let agg = machinePairLatency.get(key);
-                if (!agg) {
-                    agg = { sum: 0, count: 0 };
-                    machinePairLatency.set(key, agg);
-                }
-                agg.sum += latencyMs;
-                agg.count++;
+                let existing = machinePairLatency.get(key);
+                machinePairLatency.set(key, existing === undefined ? latencyMs : Math.min(existing, latencyMs));
             }
         }
         let machinePairTraffic = new Map<string, number>();
@@ -491,6 +580,28 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             let mk = pairKey(ma, mb);
             machinePairTraffic.set(mk, (machinePairTraffic.get(mk) || 0) + weight);
         }
+        // Sum thread-pair path value flows up to machine pairs, keeping direction (fwd = lower machine id → higher).
+        let machinePairPv = new Map<string, { fwd: number; back: number }>();
+        for (let [key, pv] of pairPv) {
+            let [a, b] = key.split("|");
+            let ma = machineIdOf(a);
+            let mb = machineIdOf(b);
+            if (ma === mb) continue;
+            let mk = pairKey(ma, mb);
+            let entry = machinePairPv.get(mk);
+            if (!entry) {
+                entry = { fwd: 0, back: 0 };
+                machinePairPv.set(mk, entry);
+            }
+            // The thread pair's fwd direction may be flipped relative to the machine pair's ordering.
+            if (ma < mb) {
+                entry.fwd += pv.fwd;
+                entry.back += pv.back;
+            } else {
+                entry.fwd += pv.back;
+                entry.back += pv.fwd;
+            }
+        }
 
         let nodes: LatencyGraphNode[] = [...machineRespondedThreads.keys()].map(machineId => {
             let allThreads = machineAllThreads.get(machineId) || [];
@@ -502,9 +613,14 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             };
         });
         let links: LatencyGraphLink[] = [];
-        for (let [key, agg] of machinePairLatency) {
+        for (let [key, latencyMs] of machinePairLatency) {
             let [source, destination] = key.split("|");
-            links.push({ source, destination, latencyMs: agg.sum / agg.count, weight: machinePairTraffic.get(key) || 0 });
+            let pv = machinePairPv.get(key);
+            let extraLabel: LatencyGraphLabelLine | undefined;
+            if (pv && (pv.fwd || pv.back)) {
+                extraLabel = { text: `↑${formatNumber(pv.fwd)}/s ↓${formatNumber(pv.back)}/s values`, color: PATHVALUE_COLOR };
+            }
+            links.push({ source, destination, latencyMs, weight: machinePairTraffic.get(key) || 0, extraLabel });
         }
 
         // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
@@ -539,7 +655,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             </div>
             <div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
             <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
-            <NodeInfoTable rows={rows} selectedMachine={selected} />
+            <NodeInfoTable rows={rows} selectedMachine={selected} machineColumns={machineColumns} nodeMachineLatency={nodeMachineLatency} />
         </div>;
     }
 }

package/src/diagnostics/pathAuditer.ts

@@ -204,6 +204,7 @@ async function processPendingValidityChecks(now: number) {
     let splitIndex = binarySearchBasic2(pendingValidityChecks, check => check.discoveredAt, { discoveredAt: threshold } as PendingValidityCheck);
     if (splitIndex < 0) splitIndex = ~splitIndex;
 
+    let pathsToResolve = new Set<string>();
     for (let i = 0; i < splitIndex; i++) {
         let pendingCheck = pendingValidityChecks[i];
         let currentValue = authorityStorage.getValueAtOrBeforeTime(pendingCheck.path);
@@ -215,15 +216,14 @@ async function processPendingValidityChecks(now: number) {
                 ourValid: pendingCheck.ourValid,
                 remoteValid: pendingCheck.remoteValid,
                 remoteNodeId: pendingCheck.remoteNodeId,
-                reason: "Pending validity check aged past MAX_CHANGE_AGE. Forcing sync to all nodes.",
+                reason: "Pending validity check aged past MAX_CHANGE_AGE. Resolving the valid state to valid on all authorities.",
             });
-            let allAuthorities = PathRouter.getAllAuthoritiesForValues([currentValue]);
-            for (let [authorityNodeId, _] of allAuthorities.entries()) {
-                let serialized = await pathValueSerializer.serialize([currentValue]);
-                await PathAuditerController.nodes[authorityNodeId].ingestPathValues(serialized);
-            }
+            pathsToResolve.add(pendingCheck.path);
         }
     }
+    if (pathsToResolve.size > 0) {
+        await resolveValidStateDisagreements(pathsToResolve);
+    }
 
     if (splitIndex > 0) {
         for (let i = 0; i < splitIndex; i++) {
@@ -233,6 +233,73 @@ async function processPendingValidityChecks(now: number) {
     }
 }
 
+async function resolveValidStateDisagreements(paths: Set<string>) {
+    let values: PathValue[] = [];
+    for (let path of paths) {
+        let value = authorityStorage.getValueAtOrBeforeTime(path);
+        if (value) {
+            values.push(value);
+        }
+    }
+    if (values.length === 0) return;
+
+    console.info(`Resolving valid state disagreements`, { pathCount: values.length });
+
+    let otherAuthorities = PathRouter.getAllAuthoritiesForValues(values);
+    let anyValidByPath = new Map<string, boolean>();
+    for (let value of values) {
+        anyValidByPath.set(value.path, !!value.valid);
+    }
+    for (let [authorityNodeId, authorityValues] of otherAuthorities.entries()) {
+        let requests = authorityValues.map(value => ({ path: value.path, time: value.time }));
+        let responses = await PathAuditerController.nodes[authorityNodeId].getValidStates(requests);
+        for (let response of responses) {
+            if (response.valid) {
+                anyValidByPath.set(response.path, true);
+            }
+        }
+    }
+
+    let forcedByPath = new Map<string, PathValue>();
+    for (let value of values) {
+        if (!anyValidByPath.get(value.path)) continue;
+        console.info(`Resolving valid state disagreement to valid`, {
+            path: value.path,
+            timeId: value.time.time,
+            timeIdFull: value.time,
+            ourValid: value.valid,
+        });
+        if (value.valid) {
+            forcedByPath.set(value.path, value);
+        } else {
+            forcedByPath.set(value.path, { ...value, valid: true });
+        }
+        stats.fixedCount++;
+    }
+    if (forcedByPath.size === 0) return;
+
+    for (let [authorityNodeId, authorityValues] of otherAuthorities.entries()) {
+        let toSend: PathValue[] = [];
+        for (let value of authorityValues) {
+            let forced = forcedByPath.get(value.path);
+            if (forced) {
+                toSend.push(forced);
+            }
+        }
+        if (toSend.length === 0) continue;
+        let serialized = await pathValueSerializer.serialize(toSend);
+        await PathAuditerController.nodes[authorityNodeId].ingestPathValues(serialized, true);
+    }
+
+    validStateComputer.ingestValuesAndValidStates({
+        pathValues: Array.from(forcedByPath.values()),
+        parentSyncs: [],
+        initialTriggers: { values: new Set(), parentPaths: new Set() },
+        doNotArchive: true,
+        forceValidStates: true,
+    });
+}
+
 function trackSyncAge(config: {
     path: string;
     ourTimeId: number;
@@ -364,7 +431,7 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
                         ourValid: ourExact.valid,
                         remoteValid: response.valid,
                         remoteNodeId: nodeId,
-                        reason: "Remote says our value is invalid, but we think it's valid. Telling all nodes about this value to ensure it's in sync everywhere.",
+                        reason: "Remote says our value is invalid, but we think it's valid. Resolving the valid state to valid on all authorities.",
                     });
                 } else {
                     if (!pendingValidityCheckPaths.has(request.path)) {
@@ -439,16 +506,7 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
     }
 
     if (pathsToForceSync.size > 0) {
-        for (let path of pathsToForceSync) {
-            let valueToForce = authorityStorage.getValueAtOrBeforeTime(path);
-            if (valueToForce) {
-                let allAuthorities = PathRouter.getAllAuthoritiesForValues([valueToForce]);
-                for (let [authorityNodeId, _] of allAuthorities.entries()) {
-                    let serialized = await pathValueSerializer.serialize([valueToForce]);
-                    await PathAuditerController.nodes[authorityNodeId].ingestPathValues(serialized);
-                }
-            }
-        }
+        await resolveValidStateDisagreements(pathsToForceSync);
     }
 
     stats.totalAudited += pathsToAudit.length;
@@ -505,13 +563,14 @@ class PathAuditerService {
         return await pathValueSerializer.serialize(results);
     }
 
-    public async ingestPathValues(serializedValues: Buffer[]): Promise<void> {
+    public async ingestPathValues(serializedValues: Buffer[], forceValidStates?: boolean): Promise<void> {
         let values = await pathValueSerializer.deserialize(serializedValues);
         validStateComputer.ingestValuesAndValidStates({
             pathValues: values,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
             doNotArchive: true,
+            forceValidStates,
         });
     }
 

package/src/library-components/LatencyGraph.tsx

@@ -1,6 +1,7 @@
 import { qreact } from "../4-dom/qreact";
 import { Button } from "./Button";
 import { URLParam } from "./URLParam";
+import { mainResets } from "./urlResetGroups";
 import { InputLabelURL } from "./InputLabel";
 import { css } from "typesafecss";
 import { sort } from "socket-function/src/misc";
@@ -8,7 +9,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 
 export type LatencyGraphLabelLine = { text: string; color?: string; };
 export type LatencyGraphNode = { id: string; label?: string; latitude?: number; longitude?: number; labelLines?: LatencyGraphLabelLine[]; weight?: number; };
-export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; };
+export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; extraLabel?: LatencyGraphLabelLine; };
 export type LatencyGraphProps = {
     nodes: LatencyGraphNode[];
     links: LatencyGraphLink[];
@@ -22,15 +23,15 @@ export type LatencyGraphProps = {
 
 const DEFAULT_CONNECTIONS = 4;
 // Nearest neighbors drawn per node (the layout still uses every measured latency; this only thins the drawn lines).
-const connectionsParam = new URLParam("lgConnections", DEFAULT_CONNECTIONS);
+const connectionsParam = new URLParam("lgConnections", DEFAULT_CONNECTIONS, { reset: [mainResets] });
 const DEFAULT_LATENCY_EXPONENT = 0.5;
 // Exponent on (latency / reference) when mapping to layout distance, so far nodes push apart harder as it grows.
-const latencyExponentParam = new URLParam("lgLatencyExponent", DEFAULT_LATENCY_EXPONENT);
-const geoParam = new URLParam("lgGeo", false);
+const latencyExponentParam = new URLParam("lgLatencyExponent", DEFAULT_LATENCY_EXPONENT, { reset: [mainResets] });
+const geoParam = new URLParam("lgGeo", false, { reset: [mainResets] });
 // On by default: each drawn connection shows its latency and (when a weight formatter is given) its traffic.
-const showLatenciesParam = new URLParam("lgShowLatencies", true);
+const showLatenciesParam = new URLParam("lgShowLatencies", true, { reset: [mainResets] });
 // The config panel is collapsed by default so it doesn't eat the canvas; clicking the header expands it.
-const configOpenParam = new URLParam("lgConfigOpen", false);
+const configOpenParam = new URLParam("lgConfigOpen", false, { reset: [mainResets] });
 // Pixels per degree of latitude/longitude in geographic mode (equirectangular projection).
 const GEO_SCALE = 6;
 
@@ -62,7 +63,6 @@ const LINE_MAX_WIDTH = 7;
 const LABEL_LINE_HEIGHT = 13;
 const ZOOM_STEP = 1.1;
 const MIN_SCALE = 0.05;
-const MAX_SCALE = 12;
 // Screen-space margins left around the content when auto-fitting. The top gets much more room because node labels
 // stack upward above the nodes (and are drawn at a fixed screen size regardless of zoom).
 const FIT_PAD = 60;
@@ -84,6 +84,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
     maxNodeWeight = 0;
     pairWeight = new Map<number, number>();
     maxPairWeight = 0;
+    pairExtraLabel = new Map<number, LatencyGraphLabelLine>();
     // Cached from props in render() so draw() (a rAF callback, not synced) never reads this.props.
     formatWeight: ((weight: number) => string) | undefined = undefined;
     selectedId: string | undefined = undefined;
@@ -135,6 +136,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         this.positionsY = new Float64Array(n);
         this.ingestLinks();
         this.computeWeights();
+        this.computePairLabels();
         this.computeSolveEdges();
         this.computeRenderEdges();
         this.applyLayout();
@@ -166,6 +168,19 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         }
     }
 
+    computePairLabels() {
+        let n = this.nodes.length;
+        let index = this.nodeIndex();
+        this.pairExtraLabel = new Map();
+        for (let link of this.props.links) {
+            if (!link.extraLabel) continue;
+            let a = index.get(link.source);
+            let b = index.get(link.destination);
+            if (a === undefined || b === undefined || a === b) continue;
+            this.pairExtraLabel.set(Math.min(a, b) * n + Math.max(a, b), link.extraLabel);
+        }
+    }
+
     nodeIndex() {
         let index = new Map<string, number>();
         for (let [i, node] of this.nodes.entries()) {
@@ -519,7 +534,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             let fit = Math.min((width - 2 * FIT_PAD) / worldW, (height - FIT_PAD_TOP - FIT_PAD) / worldH) * FIT_ZOOM;
             // Only apply the fit if it's sane — never let a NaN/degenerate value nuke the whole view.
             if (Number.isFinite(fit) && fit > 0 && Number.isFinite(minX) && Number.isFinite(minY)) {
-                this.viewScale = Math.max(MIN_SCALE, Math.min(MAX_SCALE, fit));
+                this.viewScale = Math.max(MIN_SCALE, fit);
                 this.panX = -((minX + maxX) / 2 - centroidX) * this.viewScale;
                 // Center within the padded band, which sits lower than the middle because of the reserved top margin.
                 let targetCenterY = (FIT_PAD_TOP + (height - FIT_PAD)) / 2;
@@ -558,9 +573,14 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             let midX = (toScreenX(this.positionsX[edge.a]) + toScreenX(this.positionsX[edge.b])) / 2;
             let midY = (toScreenY(this.positionsY[edge.a]) + toScreenY(this.positionsY[edge.b])) / 2;
 
-            let traffic = this.pairWeight.get(Math.min(edge.a, edge.b) * this.nodes.length + Math.max(edge.a, edge.b)) || 0;
-            let trafficLabel = this.formatWeight && traffic > 0 ? this.formatWeight(traffic) : undefined;
-            let latencyY = trafficLabel ? midY - 7 : midY;
+            let pk = Math.min(edge.a, edge.b) * this.nodes.length + Math.max(edge.a, edge.b);
+            let traffic = this.pairWeight.get(pk) || 0;
+            let subLabels: LatencyGraphLabelLine[] = [];
+            let trafficLabel = this.formatWeight && traffic > 0 && this.formatWeight(traffic) || undefined;
+            if (trafficLabel) subLabels.push({ text: trafficLabel });
+            let extraLabel = this.pairExtraLabel.get(pk);
+            if (extraLabel) subLabels.push(extraLabel);
+            let latencyY = subLabels.length ? midY - 7 : midY;
 
             ctx.font = "10px sans-serif";
             let latencyLabel = formatTime(edge.latency);
@@ -570,13 +590,14 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             ctx.fillStyle = "hsl(0, 0%, 72%)";
             ctx.fillText(latencyLabel, midX, latencyY);
 
-            if (trafficLabel) {
+            for (let [si, subLabel] of subLabels.entries()) {
+                let sy = midY + 7 + si * 12;
                 ctx.font = "9px sans-serif";
-                let trafficWidth = ctx.measureText(trafficLabel).width;
+                let subWidth = ctx.measureText(subLabel.text).width;
                 ctx.fillStyle = "hsla(0, 0%, 0%, 0.5)";
-                ctx.fillRect(midX - trafficWidth / 2 - 2, midY + 7 - 6, trafficWidth + 4, 12);
-                ctx.fillStyle = "hsl(0, 0%, 58%)";
-                ctx.fillText(trafficLabel, midX, midY + 7);
+                ctx.fillRect(midX - subWidth / 2 - 2, sy - 6, subWidth + 4, 12);
+                ctx.fillStyle = subLabel.color || "hsl(0, 0%, 58%)";
+                ctx.fillText(subLabel.text, midX, sy);
             }
         }
     }
@@ -698,7 +719,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         let worldX = this.centroidX + (e.offsetX - this.viewWidth / 2 - this.panX) / this.viewScale;
         let worldY = this.centroidY + (e.offsetY - this.viewHeight / 2 - this.panY) / this.viewScale;
         let factor = e.deltaY < 0 ? ZOOM_STEP : 1 / ZOOM_STEP;
-        let newScale = Math.max(MIN_SCALE, Math.min(MAX_SCALE, this.viewScale * factor));
+        let newScale = Math.max(MIN_SCALE, this.viewScale * factor);
         this.viewScale = newScale;
         this.panX = e.offsetX - this.viewWidth / 2 - (worldX - this.centroidX) * newScale;
         this.panY = e.offsetY - this.viewHeight / 2 - (worldY - this.centroidY) * newScale;
@@ -748,8 +769,10 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         let my = e.clientY - rect.top;
         this.mouseX = mx;
         this.mouseY = my;
-        // Always hover whichever node the cursor is closest to, so moving anywhere explores the graph.
-        this.hoverNode = this.nearestNodeTo(mx, my);
+        // The move listener is on window (so panning keeps working off-canvas), so only hover while actually over the
+        // canvas — otherwise the hover would stick forever once the cursor leaves.
+        let inside = 0 <= mx && mx < rect.width && 0 <= my && my < rect.height;
+        this.hoverNode = inside ? this.nearestNodeTo(mx, my) : undefined;
         this.scheduleFrame();
     };
 
@@ -825,6 +848,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             // Same node/link counts (same order): refresh the node objects so label text that changed as data streamed
             // in — or when the traffic window toggled — shows up, without disturbing the settled layout/positions.
             this.nodes = this.props.nodes.slice();
+            this.computePairLabels();
             if (weightSig !== this.builtWeightSig) {
                 this.builtWeightSig = weightSig;
                 this.computeWeights();