querysub 0.523.0 → 0.525.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.523.0",
+  "version": "0.525.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -70,7 +70,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.12",
+    "socket-function": "^1.2.14",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-c-identity/IdentityController.ts

@@ -19,7 +19,7 @@ import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
 import { red } from "socket-function/src/formatting/logColors";
 import { isNode } from "typesafecss";
 import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
-import { timeInMinute } from "socket-function/src/misc";
+import { timeInMinute, isIpDomain } from "socket-function/src/misc";
 import { isClient, isServer } from "../config2";
 import { getDomain } from "../config";
 
@@ -242,7 +242,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
         certIssuer: issuer.cert.toString(),
         mountedPort: getNodeIdLocation(SocketFunction.mountedNodeId)?.port,
         debugEntryPoint: isServer() ? process.argv[1] : "browser",
-        clientIsNode: isServer(),
+        clientIsNode: isServer() && !isIpDomain(nodeId),
     };
     let signature = sign(threadKeyCert, payload);
     await timeoutToError(

package/src/-d-trust/NetworkTrust2.ts

@@ -111,7 +111,7 @@ export async function isNodeTrusted(nodeId: string) {
     return await isTrusted(machineId);
 }
 
-const loadServerCert = cache(async (machineId: string) => {
+export const loadServerCert = cache(async (machineId: string) => {
     // This cert isn't stored in the archives, but... this is fine?
     if (machineId === "127-0-0-1." + getDomain()) return;
     let certFile = await archives().get(machineId);
@@ -131,7 +131,7 @@ export const ensureWeAreTrusted = lazy(measureWrap(async () => {
     }
 }));
 
-async function loadTrustCerts(nodeId: string) {
+export async function loadTrustCerts(nodeId: string) {
     let location = getNodeIdLocation(nodeId);
     if (location) {
         let machineId = getMachineId(location.address, getDomain());

package/src/-f-node-discovery/LatencyTracking.ts

@@ -43,6 +43,10 @@ export function getNodeLatency(nodeId: string): number | undefined {
     return getNodeLatencyInfo(nodeId)?.averageLatency;
 }
 
+export function getCachedNodeLatencyInfoList(): Map<string, NodeLatencyInfo> {
+    return latencyByNode;
+}
+
 /** Our measured latency to every node we have reached, as a plain map (for sending over the wire). */
 export function getOwnLatencies(): { [nodeId: string]: number } {
     void startLatencyTracking();
@@ -72,9 +76,11 @@ async function pingNode(nodeId: string) {
     recordLatency(nodeId, Date.now() - start);
 }
 
+let firstCall = true;
 async function pollLatencies() {
     let nodeIds = (await getAllNodeIds()).filter(nodeId => !isOwnNodeId(nodeId));
-    await spreadCallsOverTime(nodeIds, LATENCY_POLL_INTERVAL, pingNode);
+    let spread = firstCall ? 0 : LATENCY_POLL_INTERVAL;
+    await spreadCallsOverTime(nodeIds, spread, pingNode);
 
     let now = Date.now();
     for (let [nodeId, info] of Array.from(latencyByNode)) {
@@ -82,6 +88,7 @@ async function pollLatencies() {
             latencyByNode.delete(nodeId);
         }
     }
+    firstCall = false;
 }
 
 export const startLatencyTracking = lazy(async () => {

package/src/-f-node-discovery/TrafficTracking.ts

@@ -26,14 +26,14 @@ const BUCKET_HISTORY = BUCKET_SIZE * 20;
 // not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
 const OUTSIDE_KEY = "";
 
-export type NodeDataTraffic = { sent: number; received: number; };
+export type NodeDataTraffic = { sent: number; received: number; pathValuesSent: number; pathValuesReceived: number; };
 // All numbers are PER-SECOND rates (see file header).
 export type TrafficStats = {
     functionsExecuted: number;
     pathValuesSent: number;
     pathValuesReceived: number;
     querysubCalls: number;
-    // Byte rates to/from each known network node, plus one aggregate for everything outside the node list.
+    // Byte and path value rates to/from each known network node, plus one aggregate for everything outside the node list.
     perNode: { [nodeId: string]: NodeDataTraffic };
     outside: NodeDataTraffic;
 };
@@ -48,6 +48,8 @@ let querysubCalls: BucketMap = new Map();
 // raw connection nodeId => buckets
 let bytesSent = new Map<string, BucketMap>();
 let bytesReceived = new Map<string, BucketMap>();
+let pathValuesSentPerNode = new Map<string, BucketMap>();
+let pathValuesReceivedPerNode = new Map<string, BucketMap>();
 
 function addSimple(buckets: BucketMap, amount: number) {
     let bucket = Math.floor(Date.now() / BUCKET_SIZE) * BUCKET_SIZE;
@@ -65,13 +67,15 @@ function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
 export function recordFunctionExecuted() {
     addSimple(functionsExecuted, 1);
 }
-export function recordPathValuesSent(count: number) {
-    if (count <= 0) return;
-    addSimple(pathValuesSent, count);
+export function recordPathValuesSent(config: { count: number; nodeId: string }) {
+    if (config.count <= 0) return;
+    addSimple(pathValuesSent, config.count);
+    addKeyed(pathValuesSentPerNode, trafficKey(config.nodeId), config.count);
 }
-export function recordPathValuesReceived(count: number) {
-    if (count <= 0) return;
-    addSimple(pathValuesReceived, count);
+export function recordPathValuesReceived(config: { count: number; nodeId: string }) {
+    if (config.count <= 0) return;
+    addSimple(pathValuesReceived, config.count);
+    addKeyed(pathValuesReceivedPerNode, trafficKey(config.nodeId), config.count);
 }
 export function recordQuerysubCall() {
     addSimple(querysubCalls, 1);
@@ -111,10 +115,10 @@ function bucketRate(buckets: BucketMap, windowSize: number): number {
 function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
     let known = new Set(getCachedNodeIds());
     let perNode: { [nodeId: string]: NodeDataTraffic } = {};
-    let outside: NodeDataTraffic = { sent: 0, received: 0 };
+    let outside: NodeDataTraffic = { sent: 0, received: 0, pathValuesSent: 0, pathValuesReceived: 0 };
     // At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
     // the rare server-style id that isn't already a known node.
-    let attribute = (rawNodeId: string, field: "sent" | "received", rate: number) => {
+    let attribute = (rawNodeId: string, field: keyof NodeDataTraffic, rate: number) => {
         if (rate <= 0) return;
         if (rawNodeId === OUTSIDE_KEY) {
             outside[field] += rate;
@@ -124,7 +128,7 @@ function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeD
         if (known.has(nice)) {
             let entry = perNode[nice];
             if (!entry) {
-                entry = { sent: 0, received: 0 };
+                entry = { sent: 0, received: 0, pathValuesSent: 0, pathValuesReceived: 0 };
                 perNode[nice] = entry;
             }
             entry[field] += rate;
@@ -132,14 +136,16 @@ function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeD
             outside[field] += rate;
         }
     };
-    for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
-        attribute(rawNodeId, "sent", bucketRate(buckets, windowSize));
-        if (buckets.size === 0) bytesSent.delete(rawNodeId);
-    }
-    for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
-        attribute(rawNodeId, "received", bucketRate(buckets, windowSize));
-        if (buckets.size === 0) bytesReceived.delete(rawNodeId);
-    }
+    let aggregateMap = (map: Map<string, BucketMap>, field: keyof NodeDataTraffic) => {
+        for (let [rawNodeId, buckets] of Array.from(map)) {
+            attribute(rawNodeId, field, bucketRate(buckets, windowSize));
+            if (buckets.size === 0) map.delete(rawNodeId);
+        }
+    };
+    aggregateMap(bytesSent, "sent");
+    aggregateMap(bytesReceived, "received");
+    aggregateMap(pathValuesSentPerNode, "pathValuesSent");
+    aggregateMap(pathValuesReceivedPerNode, "pathValuesReceived");
     return { perNode, outside };
 }
 

package/src/0-path-value-core/PathRouter.ts

@@ -22,6 +22,9 @@ export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
 // fine (two nearby servers just talk to each other). Replaces the old uniform-random pick between candidates.
 const LATENCY_CANDIDATE_LIMIT = 5;
 const LATENCY_WEIGHT_OFFSET = 10;
+// A candidate whose weight is below (the current max weight / this factor) is dropped to zero probability. So a node
+// weighted 10x another (10x lower effective latency) is picked over it 100% of the time.
+const LATENCY_WEIGHT_CUTOFF_FACTOR = 10;
 // Latency assumed for a node we haven't measured yet, so new/unreached nodes stay eligible but aren't preferred.
 const UNKNOWN_LATENCY_MS = 50;
 
@@ -42,14 +45,19 @@ function latencyWeightedShuffle<T extends { nodeId: string }>(arr: T[]) {
     let rest = withLatency.slice(LATENCY_CANDIDATE_LIMIT);
     let ordered: T[] = [];
     while (pool.length > 0) {
+        let weights = pool.map(e => 1 / (e.latency + LATENCY_WEIGHT_OFFSET));
+        let cutoff = Math.max(...weights) / LATENCY_WEIGHT_CUTOFF_FACTOR;
+        for (let i = 0; i < weights.length; i++) {
+            if (weights[i] < cutoff) weights[i] = 0;
+        }
         let total = 0;
-        for (let e of pool) {
-            total += 1 / (e.latency + LATENCY_WEIGHT_OFFSET);
+        for (let w of weights) {
+            total += w;
         }
         let r = Math.random() * total;
         let idx = 0;
         while (idx < pool.length - 1) {
-            r -= 1 / (pool[idx].latency + LATENCY_WEIGHT_OFFSET);
+            r -= weights[idx];
             if (r <= 0) break;
             idx++;
         }

package/src/0-path-value-core/PathValueCommitter.ts

@@ -144,7 +144,7 @@ class PathValueCommitter {
     }
 
     private broadcastValues = batchFunction(
-        { delay: 10, throttleWindow: 500, noMeasure: true },
+        { delay: 1, throttleWindow: 2000, noMeasure: true },
         async function internal_forwardWrites(valuesBatched: {
             values: Set<PathValue>;
             tryCount?: number;
@@ -327,7 +327,7 @@ class PathValueCommitter {
 
 
     public ingestRemoteValuesAndValidStates = batchFunction(
-        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
+        { delay: 1, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
         async (batched: RemoteValueAndValidState[]) => {
             const { remoteWatcher } = await import("../1-path-client/RemoteWatcher");
 

package/src/0-path-value-core/PathValueController.ts

@@ -23,11 +23,18 @@ import { encodeCborx, decodeCborx } from "../misc/cloneHelpers";
 import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
 import { delay } from "socket-function/src/batching";
 import { isDefined } from "../misc";
+import { measureBlock } from "socket-function/src/profiling/measure";
 export { pathValueCommitter };
 
 let pathValueSendCount = 0;
 export function getPathValueSendCount() { return pathValueSendCount; }
 
+// Debug hook, called with every batch of values sendData receives, before any filtering or ingest — so tests can observe exactly when values arrive, without worrying about values being ignored or batched downstream.
+let debugOnSendData: ((values: PathValue[], sourceNodeId: string) => void) | undefined;
+export function setDebugOnSendData(callback: typeof debugOnSendData) {
+    debugOnSendData = callback;
+}
+
 // ONLY returns non-canGCValues that are valid
 export type AuditSnapshotEntry = {
     path: string;
@@ -53,7 +60,7 @@ export class PathValueControllerBase {
         let { pathValues, nodeId } = config;
 
         pathValueSendCount += pathValues.length;
-        let serializedValues = await pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK });
+        let serializedValues = await measureBlock(() => pathValueSerializer.serialize(pathValues, { compress: Querysub.COMPRESS_NETWORK }), "createValues|serialize");
         if (isDebugLogEnabled()) {
             for (let value of pathValues) {
                 auditLog("SEND CREATED VALUE", {
@@ -111,11 +118,11 @@ export class PathValueControllerBase {
         let changes = config.pathValues;
         let { nodeId, initialTriggers } = config;
         pathValueSendCount += changes.length;
-        recordPathValuesSent(changes.length);
-        let buffers = await pathValueSerializer.serialize(changes, {
+        recordPathValuesSent({ count: changes.length, nodeId });
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(changes, {
             noLocks: !config.keepLocks,
             compress: getCompressNetwork(),
-        });
+        }), "sendValues|serialize");
         this.logSendValues({ nodeId, pathValues: changes, initialTriggers, reason: config.reason });
         return await PathValueController.nodes[nodeId].sendData({
             valueBuffers: buffers,
@@ -137,7 +144,7 @@ export class PathValueControllerBase {
 
     }): Promise<void | "refused"> {
         let callerId = SocketFunction.getCaller().nodeId;
-        let { initialCreation, valueBuffers } = config;
+        const { initialCreation, valueBuffers } = config;
 
         let slowdown = getSlowdown();
         if (slowdown) {
@@ -146,28 +153,34 @@ export class PathValueControllerBase {
 
         let values: PathValue[] = [];
         if (valueBuffers) {
-            values = await pathValueSerializer.deserialize(valueBuffers);
+            values = await measureBlock(() => pathValueSerializer.deserialize(valueBuffers), "sendData|deserialize");
             ActionsHistory.OnRead(values);
-            recordPathValuesReceived(values.length);
+            recordPathValuesReceived({ count: values.length, nodeId: callerId });
+        }
+        if (debugOnSendData) {
+            debugOnSendData(values, callerId);
         }
 
         if (initialCreation) {
-            let sourceNodeId = debugNodeId(callerId);
-            let rejected = false;
-            let threshold = Date.now() - MAX_CHANGE_AGE;
-            for (let value of values) {
-                let pastThreshold = threshold - value.time.time;
-                if (pastThreshold > 0) {
-                    rejected = true;
-                    console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
-                        path: value.path,
-                        timeId: value.time.time,
-                        sourceNodeId,
-                        sourceNodeThreadId: decodeNodeId(sourceNodeId, getDomain())?.threadId,
-                        totalValueCount: values.length,
-                    });
+            let rejected = measureBlock(() => {
+                let sourceNodeId = debugNodeId(callerId);
+                let anyRejected = false;
+                let threshold = Date.now() - MAX_CHANGE_AGE;
+                for (let value of values) {
+                    let pastThreshold = threshold - value.time.time;
+                    if (pastThreshold > 0) {
+                        anyRejected = true;
+                        console.error(`Rejecting values as one is too old. It likely got caught in the pipeline too long and now can't be committed without trying to undone history which can already been committed to disk.`, {
+                            path: value.path,
+                            timeId: value.time.time,
+                            sourceNodeId,
+                            sourceNodeThreadId: decodeNodeId(sourceNodeId, getDomain())?.threadId,
+                            totalValueCount: values.length,
+                        });
+                    }
                 }
-            }
+                return anyRejected;
+            }, "sendData|ageCheck");
             if (rejected) {
                 return "refused";
             }
@@ -189,7 +202,7 @@ export class PathValueControllerBase {
         }
 
         // Note which values are genuinely new to us BEFORE ingesting. We only re-share new values with the other authorities below; if we already had a value we already shared it when we first received it, so re-sharing is redundant and can clobber a newer value with an older copy.
-        let valuesNewToUs = config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [];
+        let valuesNewToUs = measureBlock(() => config.initialCreation && values.filter(value => !authorityStorage.getValueExactMaybeRejected(value.path, value.time)) || [], "sendData|newToUsCheck");
 
         try {
             let initialTriggers = config.initialTriggers || { values: new Set(), parentPaths: new Set() };
@@ -205,7 +218,8 @@ export class PathValueControllerBase {
 
         if (config.initialCreation && valuesNewToUs.length > 0) {
             // Always shared the latest. If we don't have it, maybe something with ingestion delayed, we should still share it so we don't lose the value. If we haven't even ingested it, then it is our latest version of the value.
-            valuesNewToUs = valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value);
+            valuesNewToUs = measureBlock(() => valuesNewToUs.map(value => authorityStorage.getValueExactMaybeRejected(value.path, value.time) || value), "sendData|newToUsRemap");
+            // Not measured: this waits on network sends to the other authorities.
             await PathValueControllerBase.authorityShareValues({ pathValues: valuesNewToUs });
         }
     }
@@ -242,13 +256,13 @@ export class PathValueControllerBase {
                 auditLog("WATCH PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.watchPath({
+        measureBlock(() => pathWatcher.watchPath({
             nodeId: callerId,
             paths: config.paths,
             parentPaths: config.parentPaths,
             initialTrigger: true,
             fullHistory: config.fullHistory,
-        });
+        }), "watchLatest|watchPath");
     }
     public async unwatchLatest(config: WatchConfig) {
         let callerId = SocketFunction.getCaller().nodeId;
@@ -261,7 +275,7 @@ export class PathValueControllerBase {
                 auditLog("UNWATCHING PARENT PATH", { path: value, sourceNodeId });
             }
         }
-        pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" });
+        measureBlock(() => pathWatcher.unwatchPath({ paths: config.paths, parentPaths: config.parentPaths, callback: callerId, reason: "PathValueController.unwatchLatest" }), "unwatchLatest|unwatchPath");
     }
 
     public static async getInitialValues(config: {
@@ -281,11 +295,11 @@ export class PathValueControllerBase {
         startTime: number;
         endTime: number;
     }): Promise<Buffer[]> {
-        let values = authorityStorage.getAllValues(config);
-        let buffers = await pathValueSerializer.serialize(values, {
+        let values = measureBlock(() => authorityStorage.getAllValues(config), "getInitialValues|getAllValues");
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(values, {
             noLocks: true,
             compress: getCompressNetwork(),
-        });
+        }), "getInitialValues|serialize");
         return buffers;
     }
 
@@ -304,23 +318,25 @@ export class PathValueControllerBase {
         let { spec } = config;
         let { compareTime } = await import("./pathValueCore");
 
-        let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
+        return measureBlock(() => {
+            let allValues = authorityStorage.getAllValues({ spec, startTime: 0, endTime: Number.MAX_SAFE_INTEGER });
 
-        let pathToLatest = new Map<string, Time>();
-        for (let value of allValues) {
-            if (!value.valid) continue;
-            if (value.canGCValue) continue;
+            let pathToLatest = new Map<string, Time>();
+            for (let value of allValues) {
+                if (!value.valid) continue;
+                if (value.canGCValue) continue;
 
-            let existing = pathToLatest.get(value.path);
-            if (!existing || compareTime(value.time, existing) > 0) {
-                pathToLatest.set(value.path, value.time);
+                let existing = pathToLatest.get(value.path);
+                if (!existing || compareTime(value.time, existing) > 0) {
+                    pathToLatest.set(value.path, value.time);
+                }
             }
-        }
 
-        let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
+            let entries: AuditSnapshotEntry[] = Array.from(pathToLatest.entries()).map(([path, time]) => ({ path, time }));
 
-        let encoded = encodeCborx(entries);
-        return LZ4.compress(encoded);
+            let encoded = encodeCborx(entries);
+            return LZ4.compress(encoded);
+        }, "getAuditSnapshot|local");
     }
 
 
@@ -333,19 +349,22 @@ export class PathValueControllerBase {
     }
 
     public async getValuesByPathAndTime(entries: AuditSnapshotEntry[]): Promise<Buffer[]> {
-        let values: PathValue[] = [];
-        for (let entry of entries) {
-            let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
-            if (!value) continue;
-            if (!value.valid) continue;
-            if (value.isTransparent) continue;
-            if (value.canGCValue) continue;
-            values.push(value);
-        }
-        let buffers = await pathValueSerializer.serialize(values, {
+        let values = measureBlock(() => {
+            let matched: PathValue[] = [];
+            for (let entry of entries) {
+                let value = authorityStorage.getValueExactMaybeRejected(entry.path, entry.time);
+                if (!value) continue;
+                if (!value.valid) continue;
+                if (value.isTransparent) continue;
+                if (value.canGCValue) continue;
+                matched.push(value);
+            }
+            return matched;
+        }, "getValuesByPathAndTime|gather");
+        let buffers = await measureBlock(() => pathValueSerializer.serialize(values, {
             noLocks: true,
             compress: getCompressNetwork(),
-        });
+        }), "getValuesByPathAndTime|serialize");
         return buffers;
     }
 }
@@ -367,6 +386,7 @@ export const PathValueController = SocketFunction.register(
         hooks: [requiresNetworkTrustHook],
     }),
     {
-        noFunctionMeasure: !isNode(),
+        // Most of these functions spend their time waiting on batching or remote calls, so auto-measuring them is misleading — instead the local work inside them is wrapped in individual measureBlocks.
+        noFunctionMeasure: true,
     }
 );

package/src/0-path-value-core/ValidStateComputer.ts

@@ -2,7 +2,7 @@ import { keyByArray, binarySearchIndex } from "socket-function/src/misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { isNode } from "typesafecss";
 import { isDiskAudit } from "../config";
-import { getParentPathStr, getPathFromStr } from "../path";
+import { getParentPathStr, getPathFromStr, getPathStr4 } from "../path";
 import { auditLog } from "./auditLogs";
 import { PathRouter } from "./PathRouter";
 import { PathValue, authorityStorage, compareTime, debugPathValuePath, ReadLock, byLockGroup, isCoreQuiet, debugRejections, debugTime, debugPathValue, MAX_CHANGE_AGE, createMissingEpochValue, Time, MISSING_TRANSACTION_PART_TIMEOUT, isOurPrediction, DEFER_LOCK_WINDOW } from "./pathValueCore";
@@ -21,8 +21,9 @@ class ValidStateComputer {
         parentSyncs: { parentPath: string; sourceNodeId: string }[];
         initialTriggers: { values: Set<string>; parentPaths: Set<string> };
         doNotArchive?: boolean;
+        forceValidStates?: boolean;
     }) {
-        let { pathValues, parentSyncs, doNotArchive } = config;
+        let { pathValues, parentSyncs, doNotArchive, forceValidStates } = config;
         let initialTriggers = { ...config.initialTriggers, initialTriggerNonHistoryWatchers: new Set<string>() };
 
         // TODO: We might want to add back optimizations for "no watches and no locks"?
@@ -30,6 +31,13 @@ class ValidStateComputer {
 
         let now = Date.now();
 
+        let forcedValidStates = new Map<string, boolean>();
+        if (forceValidStates) {
+            for (let value of pathValues) {
+                forcedValidStates.set(getForcedValidStateKey(value), !!value.valid);
+            }
+        }
+
         authorityStorage.addParentSyncs(parentSyncs);
 
         // Dedup by (path, time): if multiple values for the same (path, time) arrive in this batch,
@@ -216,6 +224,7 @@ class ValidStateComputer {
                 Array.from(dependenciesChanged),
                 now,
                 initialPrevValidStates,
+                forcedValidStates,
             );
             let validStateChanged = result.changed;
             for (let d of result.deferred) {
@@ -319,6 +328,7 @@ class ValidStateComputer {
         valuePaths: PathValue[],
         now: number,
         prevValidStates: Map<PathValue, boolean | undefined>,
+        forcedValidStates?: Map<string, boolean>,
     ): { changed: PathValue[]; deferred: PathValue[] } {
         let changed: PathValue[] = [];
         let deferred: PathValue[] = [];
@@ -404,6 +414,11 @@ class ValidStateComputer {
 
             for (let pathValue of valueGroup) {
                 let prevValidState = prevValidStates.get(pathValue);
+                let newValid: boolean = valid;
+                let forcedValid = forcedValidStates?.get(getForcedValidStateKey(pathValue));
+                if (forcedValid !== undefined) {
+                    newValid = forcedValid;
+                }
                 /* We used to log these values until 2026 June 28th 6:30 am
                 if (valid && prevValidState === false) {
                     console.info(`Accepting value that was previously rejected`, {
@@ -428,31 +443,32 @@ class ValidStateComputer {
                 }
                 */
                 // IMPORTANT! This means if it didn't previously exist and it's presently rejected, we count that as a change, as it this is going from undefined to false. This is actually very useful, as a lot of places want to know if a write is rejected, so they can display it in the UI for the developer.
-                if (valid === prevValidState && pathValue.valid === valid) continue;
+                if (newValid === prevValidState && pathValue.valid === newValid) continue;
 
                 // NOTE: We might remove this logging later as it's pretty heavy, but right now we still have bugs somewhere here.
                 {
-                    console.info(`Changed valid state ${JSON.stringify(prevValidState)} to ${JSON.stringify(valid)} (and path is ${JSON.stringify(pathValue.valid)})`, {
+                    console.info(`Changed valid state ${JSON.stringify(prevValidState)} to ${JSON.stringify(newValid)} (and path is ${JSON.stringify(pathValue.valid)})`, {
                         path: pathValue.path,
                         timeId: pathValue.time.time,
                         timeIdFull: pathValue.time,
-                        valid,
+                        valid: newValid,
+                        forced: forcedValid !== undefined,
                         prevValidState,
                     });
                 }
 
                 changed.push(pathValue);
 
-                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it.
+                // NOTE: This should be the only place we set it, and the read-only flag is only for us to prevent anyone from setting it. Anyone who sets the valid state anywhere else will be absolutely fired.
                 // @ts-expect-error
-                pathValue.valid = valid;
+                pathValue.valid = newValid;
 
                 // HACK: There are times when we might be evaluating the same path value multiple times at once (ex, multiple initial syncs at once). In which case we might not equal the stored path value, so we need to forcefully update the valid state of it.
                 let storedPathValue = authorityStorage.getValueExactMaybeRejected(pathValue.path, pathValue.time);
                 // I think this is fine. I think it happens if we receive it multiple times due to sharding. It's a bit inefficient, but... SHOULD be fine.
                 if (storedPathValue && pathValue !== storedPathValue) {
                     // @ts-expect-error
-                    storedPathValue.valid = valid;
+                    storedPathValue.valid = newValid;
                 }
             }
         }
@@ -541,3 +557,7 @@ class ValidStateComputer {
     }
 }
 export const validStateComputer = new ValidStateComputer();
+
+function getForcedValidStateKey(pathValue: { time: Time; path: string }): string {
+    return getPathStr4(pathValue.path, String(pathValue.time.time), String(pathValue.time.version), String(pathValue.time.creatorId));
+}

package/src/1-path-client/RemoteWatcher.ts

@@ -656,6 +656,19 @@ export class RemoteWatcher {
         return Array.from(new Set(this.remoteWatchPaths.values()));
     }
 
+    public getWatchedPathCountsPerNodeId(): Map<string, number> {
+        let counts = new Map<string, number>();
+        for (let nodeId of this.remoteWatchPaths.values()) {
+            counts.set(nodeId, (counts.get(nodeId) || 0) + 1);
+        }
+        for (let watchObj of this.remoteWatchParents2.values()) {
+            for (let range of watchObj.ranges) {
+                counts.set(range.authorityId, (counts.get(range.authorityId) || 0) + 1);
+            }
+        }
+        return counts;
+    }
+
 
 
     public async refreshAllWatches(authorityNodeId: string) {

package/src/3-path-functions/PathFunctionRunner.ts

@@ -1,5 +1,5 @@
 import { batchFunction, delay, runInfinitePoll } from "socket-function/src/batching";
-import { recordFunctionExecuted } from "../-f-node-discovery/TrafficTracking";
+import { recordFunctionExecuted, recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { cache, lazy } from "socket-function/src/caching";
 import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
 import { timeInHour, timeInMinute, timeInSecond } from "socket-function/src/misc";
@@ -56,6 +56,9 @@ setImmediate(() => {
 });
 
 export function commitCall(call: CallSpec) {
+    // Counted here (not in QuerysubController.addCall) so trusted nodes' direct writes are counted too — both the
+    // remote addCall path and the direct-write path funnel through here.
+    recordQuerysubCall();
     if (!call.network) {
         throw new Error(`Call has no network, so it would never run (no FunctionRunner would pick it up). Call: ${debugCallSpec(call)}`);
     }

package/src/4-deploy/edgeBootstrap.ts

@@ -357,7 +357,7 @@ async function edgeNodeFunction(config: {
         // Probes ALL the given nodes (even private / non-live ones), writing latencies to globalThis.EDGE_NODE_STATS (so the edge node dropdown can show them), and returns a latency-weighted random pick among `pickableHosts` (undefined if none respond).
         async function probeAndPick(probeNodes: EdgeNodeConfig[], pickableHosts: Set<string>): Promise<EdgeNodeConfig | undefined> {
             // Having this much lower latency than another node means we pick it 100% of the time (the weight scales linearly, from 1 at the best latency down to 0 at the best latency + this)
-            const LATENCY_WEIGHT_WINDOW = 1000;
+            const LATENCY_WEIGHT_WINDOW = 300;
             // How many probes are in flight at once. The first batch fires simultaneously, so anything that responds more than LATENCY_WEIGHT_WINDOW after the first response has a latency at least that much higher than it, and so would never have been picked anyway.
             const PROBE_BATCH_SIZE = 5;
 

package/src/4-querysub/QuerysubController.ts

@@ -1,6 +1,5 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { cache, lazy } from "socket-function/src/caching";
-import { recordQuerysubCall } from "../-f-node-discovery/TrafficTracking";
 import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
 import { FunctionMetadata } from "../3-path-functions/syncSchema";
 import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
@@ -535,7 +534,6 @@ export class QuerysubControllerBase {
 
     // NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
     public async addCall(call: CallSpec) {
-        recordQuerysubCall();
         if (isBootstrapOnly()) throw new Error(`Cannot add calls to bootstrap only server`);
         if (Querysub.DEBUG_CALLS) {
             console.log(`[Querysub] addCall @${debugTime(call.runAtTime)}: ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);