querysub 0.521.0 → 0.523.0

package/bin/function-public.js

@@ -2,8 +2,6 @@
 
 process.argv.push("--nobreak");
 process.argv.push("--public");
-process.argv.push("--network");
-process.argv.push("public");
 
 require("typenode");
 require("../src/3-path-functions/PathFunctionRunnerMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.521.0",
+  "version": "0.523.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/-f-node-discovery/TrafficTracking.ts

@@ -2,9 +2,14 @@
     Per-server, in-memory traffic counters (never written to the database). Everything is just an integer increment
     into a 5-minute bucket; we keep the last 60 minutes (12 buckets) and drop older ones, so memory is bounded.
 
+    Reads return PER-SECOND RATES: the window's total divided by how long data has ACTUALLY been collecting in that
+    window — from the later of the window's start and when we first saw any data, up to now (so it's dynamic, and on a
+    server that just started it divides by the real short duration, not the full window). Under a second of data falls
+    back to the window length so we never divide by ~0.
+
     Byte traffic comes from SocketFunction.trackMessageSizes (upload/download per connection). The connection nodeId is
     sometimes a raw client-connection id, so we map it to the nice node id (debugNodeId) lazily at read time — the hot
-    path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node buckets and a single
+    path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node rates and a single
     "outside the network" aggregate for anything not in our cached node list (clients, etc.).
 */
 
@@ -15,44 +20,40 @@ import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
 import { getCachedNodeIds } from "./NodeDiscovery";
 import { debugNodeId } from "../-c-identity/IdentityController";
 
-const BUCKET_MS = timeInMinute * 5;
-const BUCKET_COUNT = 12;
+const BUCKET_SIZE = timeInMinute * 5;
+const BUCKET_HISTORY = BUCKET_SIZE * 20;
 // Sentinel key for traffic that isn't a network node (client connections, etc.). Aggregated so memory stays O(nodes),
 // not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
 const OUTSIDE_KEY = "";
 
 export type NodeDataTraffic = { sent: number; received: number; };
+// All numbers are PER-SECOND rates (see file header).
 export type TrafficStats = {
     functionsExecuted: number;
     pathValuesSent: number;
     pathValuesReceived: number;
     querysubCalls: number;
-    // Bytes to/from each known network node, plus one aggregate for everything outside the node list.
+    // Byte rates to/from each known network node, plus one aggregate for everything outside the node list.
     perNode: { [nodeId: string]: NodeDataTraffic };
     outside: NodeDataTraffic;
 };
 
-function currentBucket() {
-    return Math.floor(Date.now() / BUCKET_MS);
-}
-function oldestBucket() {
-    return currentBucket() - BUCKET_COUNT + 1;
-}
+// bucket index => amount accumulated in that bucket
+type BucketMap = Map<number, number>;
 
-// bucket => count
-let functionsExecuted = new Map<number, number>();
-let pathValuesSent = new Map<number, number>();
-let pathValuesReceived = new Map<number, number>();
-let querysubCalls = new Map<number, number>();
-// raw connection nodeId => (bucket => bytes)
-let bytesSent = new Map<string, Map<number, number>>();
-let bytesReceived = new Map<string, Map<number, number>>();
+let functionsExecuted: BucketMap = new Map();
+let pathValuesSent: BucketMap = new Map();
+let pathValuesReceived: BucketMap = new Map();
+let querysubCalls: BucketMap = new Map();
+// raw connection nodeId => buckets
+let bytesSent = new Map<string, BucketMap>();
+let bytesReceived = new Map<string, BucketMap>();
 
-function addSimple(buckets: Map<number, number>, amount: number) {
-    let bucket = currentBucket();
+function addSimple(buckets: BucketMap, amount: number) {
+    let bucket = Math.floor(Date.now() / BUCKET_SIZE) * BUCKET_SIZE;
     buckets.set(bucket, (buckets.get(bucket) || 0) + amount);
 }
-function addKeyed(map: Map<string, Map<number, number>>, key: string, amount: number) {
+function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
     let buckets = map.get(key);
     if (!buckets) {
         buckets = new Map();
@@ -84,29 +85,39 @@ function trafficKey(nodeId: string): string {
 SocketFunction.trackMessageSizes.upload.push((size, nodeId) => addKeyed(bytesSent, trafficKey(nodeId), size));
 SocketFunction.trackMessageSizes.download.push((size, nodeId) => addKeyed(bytesReceived, trafficKey(nodeId), size));
 
-function sumRecent(buckets: Map<number, number>): number {
-    let oldest = oldestBucket();
+// Per-second rate: total within the window divided by the actual span the matched buckets cover (oldest bucket's start
+// time → now). Also prunes expired buckets as it goes.
+function bucketRate(buckets: BucketMap, windowSize: number): number {
+    let now = Date.now();
+    let retentionOldest = now - BUCKET_HISTORY;
+    let targetTime = now - windowSize;
     let total = 0;
+    let oldestTime = now;
     for (let [bucket, count] of Array.from(buckets)) {
-        if (bucket < oldest) {
+        if (bucket < retentionOldest) {
             buckets.delete(bucket);
             continue;
         }
+        if (bucket < targetTime) continue;
+        oldestTime = Math.min(oldestTime, bucket);
         total += count;
     }
-    return total;
+    if (oldestTime === now) {
+        oldestTime = targetTime;
+    }
+    return total / ((now - oldestTime) / 1000);
 }
 
-function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
+function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
     let known = new Set(getCachedNodeIds());
     let perNode: { [nodeId: string]: NodeDataTraffic } = {};
     let outside: NodeDataTraffic = { sent: 0, received: 0 };
     // At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
     // the rare server-style id that isn't already a known node.
-    let attribute = (rawNodeId: string, field: "sent" | "received", amount: number) => {
-        if (amount <= 0) return;
+    let attribute = (rawNodeId: string, field: "sent" | "received", rate: number) => {
+        if (rate <= 0) return;
         if (rawNodeId === OUTSIDE_KEY) {
-            outside[field] += amount;
+            outside[field] += rate;
             return;
         }
         let nice = known.has(rawNodeId) ? rawNodeId : debugNodeId(rawNodeId);
@@ -116,37 +127,37 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
                 entry = { sent: 0, received: 0 };
                 perNode[nice] = entry;
             }
-            entry[field] += amount;
+            entry[field] += rate;
         } else {
-            outside[field] += amount;
+            outside[field] += rate;
         }
     };
     for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
-        attribute(rawNodeId, "sent", sumRecent(buckets));
+        attribute(rawNodeId, "sent", bucketRate(buckets, windowSize));
         if (buckets.size === 0) bytesSent.delete(rawNodeId);
     }
     for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
-        attribute(rawNodeId, "received", sumRecent(buckets));
+        attribute(rawNodeId, "received", bucketRate(buckets, windowSize));
         if (buckets.size === 0) bytesReceived.delete(rawNodeId);
     }
     return { perNode, outside };
 }
 
-export function getTrafficStats(): TrafficStats {
-    let data = aggregateData();
+export function getTrafficStats(windowSize: number): TrafficStats {
+    let data = aggregateData(windowSize);
     return {
-        functionsExecuted: sumRecent(functionsExecuted),
-        pathValuesSent: sumRecent(pathValuesSent),
-        pathValuesReceived: sumRecent(pathValuesReceived),
-        querysubCalls: sumRecent(querysubCalls),
+        functionsExecuted: bucketRate(functionsExecuted, windowSize),
+        pathValuesSent: bucketRate(pathValuesSent, windowSize),
+        pathValuesReceived: bucketRate(pathValuesReceived, windowSize),
+        querysubCalls: bucketRate(querysubCalls, windowSize),
         perNode: data.perNode,
         outside: data.outside,
     };
 }
 
 class TrafficControllerBase {
-    public async getTrafficStats(): Promise<TrafficStats> {
-        return getTrafficStats();
+    public async getTrafficStats(windowSize: number): Promise<TrafficStats> {
+        return getTrafficStats(windowSize);
     }
 }
 export const TrafficController = SocketFunction.register(

package/src/0-path-value-core/startupAuthority.ts

@@ -34,7 +34,7 @@ export async function startupAuthority(spec: AuthoritySpec) {
             pathValues: values,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: new Set(values),
+            doNotArchive: true,
         });
         console.log(blue(`Finished ingesting values from source ${source.nodeId} (values: ${formatNumber(values.length)}`), { nodeId: source.nodeId, spec: debugSpec(source), values: values.length });
     }
@@ -62,7 +62,7 @@ export async function startupAuthority(spec: AuthoritySpec) {
         let flat = Object.values(snapshot.values).flat();
         console.log(blue(`Loaded snapshot in memory, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
         // NOTE: If it's on disk, it can't be rejected, so there's no point in passing it through our validStateComputer (and passing it through is VERY slow).
-        authorityStorage.ingestValues(flat, { doNotArchive: new Set(flat) });
+        authorityStorage.ingestValues(flat, { doNotArchive: true });
         console.log(blue(`Finished ingesting values from snapshot, found ${formatNumber(flat.length)} values`), { spec: debugSpec(spec), values: flat.length });
     })();
 

package/src/3-path-functions/PathFunctionRunner.ts

@@ -21,7 +21,7 @@ import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { getScheduledShutdownTime } from "../-g-core-values/scheduledShutdown";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
-import { getDomain, isPublic, DEFAULT_NETWORK } from "../config";
+import { getDomain, isPublic } from "../config";
 import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
 import type { DeployProgress } from "../4-deploy/deployFunctions";
 import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
@@ -363,14 +363,15 @@ export class PathFunctionRunner {
         secondaryShardRange?: { startFraction: number, endFraction: number };
         PermissionsChecker: PermissionsCheckType | undefined;
         // The networks we listen to calls on. Unset is equivalent to ["default"].
-        networks?: string[];
+        networks: string[];
     }) {
+        if (!config.networks.length) throw new Error(`networks is required`);
         PathFunctionRunner.hostControllers();
         debugFunctionRunnerShards.push({
             domainName: config.domainName,
             shardRange: config.shardRange,
             secondaryShardRange: config.secondaryShardRange,
-            networks: config.networks && config.networks.length > 0 && config.networks || [DEFAULT_NETWORK],
+            networks: config.networks,
             isPublic: isPublic(),
         });
         logErrors(this.startWatching());
@@ -401,7 +402,7 @@ export class PathFunctionRunner {
 
         let outstandingCalls = 0;
 
-        let networks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
+        let networks = this.config.networks;
 
         let watchModuleCalls = cache((moduleId: string) => {
             for (let network of networks) {
@@ -691,7 +692,7 @@ export class PathFunctionRunner {
         }
 
         let callNetwork = callSpec.network;
-        let ourNetworks = this.config.networks && this.config.networks.length > 0 && this.config.networks || [DEFAULT_NETWORK];
+        let ourNetworks = this.config.networks;
         if (!ourNetworks.includes(callNetwork)) {
             return;
         }

package/src/3-path-functions/PathFunctionRunnerMain.ts

@@ -15,7 +15,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "sliftutils/misc/https/certs";
 import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
 import { timeInMinute } from "socket-function/src/misc";
-import { getDomain, getNetworks, isLocal, isPublic, DEFAULT_NETWORK } from "../config";
+import { getDomain, getNetworks, isLocal, isPublic } from "../config";
 import { green, magenta } from "socket-function/src/formatting/logColors";
 import path from "path";
 import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
@@ -32,6 +32,9 @@ async function main() {
     // ClientWatcher.DEBUG_TRIGGERS = "heavy";
     // authorityStorage.DEBUG_UNWATCH = true;
 
+    let networks = getNetworks();
+    if (!networks.length) throw new Error(`No networks found. Use --network to specify the networks this function runner should listen on.`);
+
     PathFunctionRunner.DEBUG_CALLS = true;
     // debugCoreMode();
 
@@ -60,8 +63,6 @@ async function main() {
         console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
     }
 
-    let networks = getNetworks();
-
     new PathFunctionRunner({
         domainName: getDomain(),
         shardRange: { startFraction: shardStart, endFraction: shardEnd },
@@ -76,9 +77,5 @@ async function main() {
         let deployPath = path.resolve("./deploy.ts");
         await import(deployPath);
     }
-
-    if (networks.length !== 1 || networks[0] !== DEFAULT_NETWORK) {
-        console.log(magenta(`Only running functions on the network(s): ${networks.join(", ")}. Use --network ${networks[0]} in your http server to route calls here.`));
-    }
 }
 logErrors(main());

package/src/4-querysub/querysubPrediction.ts

@@ -308,7 +308,7 @@ function predictCallBase(config: {
             pathValues: predictions.writes,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: new Set(predictions.writes),
+            doNotArchive: true,
         });
 
         if (Querysub.DEBUG_PREDICTIONS) {
@@ -337,7 +337,7 @@ function predictCallBase(config: {
             pathValues: rejectedWrites,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
-            doNotArchive: new Set(rejectedWrites),
+            doNotArchive: true,
         });
     }
 

package/src/config.ts

@@ -44,8 +44,6 @@ if (isNode()) {
     }
 }
 
-export const DEFAULT_NETWORK = "default";
-
 export function expandHomePath(path: string): string {
     if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) {
         return os.homedir() + path.slice(1);
@@ -76,7 +74,7 @@ export function getNetworks(): string[] {
         result = [...result, ...networkFileNetworks()];
     }
     result = Array.from(new Set(result));
-    if (result.length === 0) return [DEFAULT_NETWORK];
+    if (result.length === 0) return [];
     return result;
 }
 

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -14,7 +14,7 @@ import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../managementPages";
 import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
 import { timeoutToUndefinedSilent } from "../../errors";
-import { sort } from "socket-function/src/misc";
+import { sort, timeInMinute } from "socket-function/src/misc";
 import type { AuthoritySpec } from "../../0-path-value-core/PathRouter";
 import { getFunctionRunnerIndex, FunctionRunnerNodeInfo } from "../../4-querysub/FunctionRunnerTracking";
 import { formatTime, formatNumber } from "socket-function/src/formatting/format";
@@ -34,11 +34,13 @@ const LATENCY_GRAPH_HEIGHT_PX = 720;
 
 // Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
 const selectedMachineParam = new URLParam("rtSelectedMachine", "");
+const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60);
 
 // The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
 const NODE_TABLE_COLUMNS: { key: keyof NodeRow; label: string; width: number; color?: string; perMachine?: boolean; }[] = [
     { key: "ip", label: "IP", width: 130, perMachine: true },
+    { key: "networks", label: "Networks", width: 180, color: QUERYSUB_COLOR, perMachine: true },
     { key: "machineShort", label: "Machine", width: 96, perMachine: true },
     { key: "threadPort", label: "Thread:Port", width: 130 },
     { key: "name", label: "Name", width: 160 },
@@ -52,6 +54,7 @@ type NodeRow = {
     nodeId: string;
     machineId: string;
     ip: string;
+    networks: string;
     machineShort: string;
     threadPort: string;
     name: string;
@@ -104,9 +107,10 @@ class RoutingTablePageControllerBase {
     public async getNodeLatencies(nodeId: string): Promise<{ [nodeId: string]: number } | undefined> {
         return timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, LatencyController.nodes[nodeId].getLatencies());
     }
-    // Renamed (was getNodeTraffic) because the returned shape changed — a new name busts any stale synced-call cache.
-    public async getNodeTrafficStats(nodeId: string): Promise<TrafficStats | undefined> {
-        return timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, TrafficController.nodes[nodeId].getTrafficStats());
+    // Values are now per-second rates over the window (were near-totals). Renamed so a stale synced-call cache from the
+    // old meaning is dropped instead of being served forever.
+    public async getNodeTrafficRates(nodeId: string, windowSize: number): Promise<TrafficStats | undefined> {
+        return timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, TrafficController.nodes[nodeId].getTrafficStats(windowSize));
     }
 }
 
@@ -116,7 +120,7 @@ export const RoutingTablePageController = SocketFunction.register(
     () => ({
         getAllNodeAuthoritySpecs: {},
         getNodeLatencies: {},
-        getNodeTrafficStats: {},
+        getNodeTrafficRates: {},
         getNodeIps: {},
     }),
     () => ({
@@ -207,13 +211,15 @@ class FunctionRunnersSection extends qreact.Component {
 }
 
 function nodeTotalTraffic(traffic: TrafficStats): { sent: number; received: number; outside: number; } {
-    let sent = traffic.outside.sent;
-    let received = traffic.outside.received;
-    for (let d of Object.values(traffic.perNode)) {
-        sent += d.sent;
-        received += d.received;
+    let outsideSent = traffic.outside?.sent || 0;
+    let outsideReceived = traffic.outside?.received || 0;
+    let sent = outsideSent;
+    let received = outsideReceived;
+    for (let d of Object.values(traffic.perNode || {})) {
+        sent += d?.sent || 0;
+        received += d?.received || 0;
     }
-    return { sent, received, outside: traffic.outside.sent + traffic.outside.received };
+    return { sent, received, outside: outsideSent + outsideReceived };
 }
 
 function machineIdOf(nodeId: string): string {
@@ -235,40 +241,44 @@ function sumTraffic(threads: string[], trafficMaps: Map<string, TrafficStats>) {
         let totals = nodeTotalTraffic(traffic);
         dataSent += totals.sent;
         dataReceived += totals.received;
-        valuesSent += traffic.pathValuesSent;
-        valuesReceived += traffic.pathValuesReceived;
-        calls += traffic.functionsExecuted;
-        addCalls += traffic.querysubCalls;
+        valuesSent += traffic.pathValuesSent || 0;
+        valuesReceived += traffic.pathValuesReceived || 0;
+        calls += traffic.functionsExecuted || 0;
+        addCalls += traffic.querysubCalls || 0;
     }
     return { dataSent, dataReceived, valuesSent, valuesReceived, calls, addCalls };
 }
 
-// A machine's graph label: the IP first (most important), then the same per-piece info as the table, summed across all
-// its threads and colored the same.
+// A machine's graph label: the IP first (most important), then its function-runner networks, then the same per-piece
+// info as the table, summed across all its threads and colored the same.
 function machineLabelLines(config: {
     machineId: string;
     threads: string[];
     trafficMaps: Map<string, TrafficStats>;
     ip: string | undefined;
+    networks: string[];
 }): LatencyGraphLabelLine[] {
-    let { machineId, threads, ip } = config;
+    let { machineId, threads, ip, networks } = config;
     let totals = sumTraffic(threads, config.trafficMaps);
     let lines: LatencyGraphLabelLine[] = [];
     if (ip) {
         lines.push({ text: ip });
     }
+    if (networks.length) {
+        lines.push({ text: networks.join(" | "), color: QUERYSUB_COLOR });
+    }
     lines.push({ text: `${machineId.slice(0, ID_CHARS)}  ${threads.length} threads` });
     if (totals.dataSent + totals.dataReceived > 0) {
-        lines.push({ text: `↑${formatNumber(totals.dataSent)}B ↓${formatNumber(totals.dataReceived)}B` });
+        lines.push({ text: `↑${formatNumber(totals.dataSent)}B/s ↓${formatNumber(totals.dataReceived)}B/s` });
     }
     if (totals.valuesSent || totals.valuesReceived) {
-        lines.push({ text: `↑${formatNumber(totals.valuesSent)} ↓${formatNumber(totals.valuesReceived)} values`, color: PATHVALUE_COLOR });
+        lines.push({ text: `↑${formatNumber(totals.valuesSent)}/s ↓${formatNumber(totals.valuesReceived)}/s values`, color: PATHVALUE_COLOR });
     }
     if (totals.calls) {
-        lines.push({ text: `${formatNumber(totals.calls)} calls`, color: FUNCTION_COLOR });
+        lines.push({ text: `${formatNumber(totals.calls)}/s calls`, color: FUNCTION_COLOR });
     }
     if (totals.addCalls) {
-        lines.push({ text: `${formatNumber(totals.addCalls)} addCalls`, color: QUERYSUB_COLOR });
+        lines.push({ text: `${formatNumber(totals.addCalls)}/s addCalls`, color: QUERYSUB_COLOR });
     }
     return lines;
 }
@@ -280,8 +290,9 @@ function buildNodeRow(config: {
     info: NodeAuthorityInfo | undefined;
     traffic: TrafficStats | undefined;
     machineIp: Map<string, string>;
+    machineNetworks: Map<string, string[]>;
 }): NodeRow {
-    let { nodeId, runner, info, traffic, machineIp } = config;
+    let { nodeId, runner, info, traffic, machineIp, machineNetworks } = config;
     let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
     let thread = (parts?.threadId || "?").slice(0, ID_CHARS);
     let machineId = parts?.machineId || nodeId;
@@ -294,7 +305,7 @@ function buildNodeRow(config: {
         let sum = totals.sent + totals.received;
         if (sum > 0) {
             let outsidePct = Math.round((totals.outside / sum) * 100);
-            data = `↑${formatNumber(totals.sent)}B ↓${formatNumber(totals.received)}B  ${outsidePct}%⊘`;
+            data = `↑${formatNumber(totals.sent)}B/s ↓${formatNumber(totals.received)}B/s  ${outsidePct}%⊘`;
         }
     }
 
@@ -304,7 +315,7 @@ function buildNodeRow(config: {
         for (let shard of runner.shards) {
             fnParts.push(`fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
         }
-        fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)} calls`);
+        fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)}/s calls`);
         if (!runner.isPublic) fnParts.push("PRIVATE");
     }
 
@@ -314,14 +325,15 @@ function buildNodeRow(config: {
         pvParts.push(`pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
     }
     if (traffic && (traffic.pathValuesSent || traffic.pathValuesReceived)) {
-        pvParts.push(`↑${formatNumber(traffic.pathValuesSent)} ↓${formatNumber(traffic.pathValuesReceived)} values`);
+        pvParts.push(`↑${formatNumber(traffic.pathValuesSent)}/s ↓${formatNumber(traffic.pathValuesReceived)}/s values`);
     }
 
-    let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)} addCalls` : "";
+    let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)}/s addCalls` : "";
     return {
         nodeId,
         machineId,
         ip: machineIp.get(machineId) || "",
+        networks: (machineNetworks.get(machineId) || []).join(" | "),
         machineShort: machineId.slice(0, ID_CHARS),
         threadPort: `${thread}:${parts?.port ?? "?"}`,
         name,
@@ -377,12 +389,13 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
         let infoByNode = new Map(this.props.infos.map(info => [info.nodeId, info]));
 
         // One synced call per node for each data source; each resolves independently so the graph fills in progressively.
+        let trafficWindow = trafficWindowParam.value;
         let latencyMaps = new Map<string, { [nodeId: string]: number }>();
         let trafficMaps = new Map<string, TrafficStats>();
         for (let nodeId of nodeIds) {
             let latencies = synced.getNodeLatencies(nodeId);
             if (latencies) latencyMaps.set(nodeId, latencies);
-            let traffic = synced.getNodeTrafficStats(nodeId);
+            let traffic = synced.getNodeTrafficRates(nodeId, trafficWindow);
             if (traffic) trafficMaps.set(nodeId, traffic);
         }
         // Only graph nodes that reported their latencies — ones that never respond probably don't exist.
@@ -428,14 +441,26 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             if (bestIp) machineIp.set(machineId, bestIp);
         }
 
+        // The unique set of function-runner networks running on each machine (usually one, human-readable).
+        let machineNetworks = new Map<string, string[]>();
+        for (let [machineId, threads] of machineAllThreads) {
+            let unique = new Set<string>();
+            for (let thread of threads) {
+                for (let network of runnerByNode.get(thread)?.networks || []) {
+                    unique.add(network);
+                }
+            }
+            if (unique.size) machineNetworks.set(machineId, [...unique]);
+        }
+
         // Total bytes on each thread-pair link (both directions). Both endpoints report the same total, so take the max.
         let pairKey = (a: string, b: string) => a < b ? `${a}|${b}` : `${b}|${a}`;
         let pairTraffic = new Map<string, number>();
         for (let [reporter, traffic] of trafficMaps) {
-            for (let [peer, data] of Object.entries(traffic.perNode)) {
+            for (let [peer, data] of Object.entries(traffic.perNode || {})) {
                 if (reporter === peer || !respondedSet.has(peer)) continue;
                 let key = pairKey(reporter, peer);
-                pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, data.sent + data.received));
+                pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, (data?.sent || 0) + (data?.received || 0)));
             }
         }
 
@@ -472,7 +497,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             let totals = sumTraffic(allThreads, trafficMaps);
             return {
                 id: machineId,
-                labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId) }),
+                labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId), networks: machineNetworks.get(machineId) || [] }),
                 weight: totals.dataSent + totals.dataReceived,
             };
         });
@@ -484,22 +509,35 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
 
         // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
         // Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
-        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp }));
+        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
         let selected = selectedMachineParam.value;
         let sep = String.fromCharCode(1);
         sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
 
         return <div className={css.vbox(8).fillWidth}>
-            <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
+            <div className={css.hbox(14)}>
+                <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
+                <div className={css.hbox(0).bord2(0, 0, 70)}>
+                    {([timeInMinute * 5, timeInMinute * 60]).map(windowSize => {
+                        let selected = trafficWindow === windowSize;
+                        return <div
+                            className={css.pad2(12, 6).cursor("pointer")
+                                .hsl(210, selected ? 70 : 0, selected ? 45 : 96).colorhsl(0, 0, selected ? 100 : 30)}
+                            onMouseDown={() => trafficWindowParam.value = windowSize}
+                        >{formatTime(windowSize)}</div>;
+                    })}
+                </div>
+            </div>
             <div className={css.relative.fillWidth.height(LATENCY_GRAPH_HEIGHT_PX).bord2(0, 0, 85)}>
                 <LatencyGraph
                     nodes={nodes}
                     links={links}
-                    formatWeight={weight => formatNumber(weight) + "B"}
+                    formatWeight={weight => formatNumber(weight) + "B/s"}
                     selectedId={selected || undefined}
                     onSelectNode={id => selectedMachineParam.value = selectedMachineParam.value === id ? "" : id}
                 />
             </div>
+            <div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
             <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
             <NodeInfoTable rows={rows} selectedMachine={selected} />
         </div>;

package/src/library-components/LatencyGraph.tsx

@@ -23,11 +23,12 @@ export type LatencyGraphProps = {
 const DEFAULT_CONNECTIONS = 4;
 // Nearest neighbors drawn per node (the layout still uses every measured latency; this only thins the drawn lines).
 const connectionsParam = new URLParam("lgConnections", DEFAULT_CONNECTIONS);
-const DEFAULT_LATENCY_EXPONENT = 1;
+const DEFAULT_LATENCY_EXPONENT = 0.5;
 // Exponent on (latency / reference) when mapping to layout distance, so far nodes push apart harder as it grows.
 const latencyExponentParam = new URLParam("lgLatencyExponent", DEFAULT_LATENCY_EXPONENT);
 const geoParam = new URLParam("lgGeo", false);
-const showLatenciesParam = new URLParam("lgShowLatencies", false);
+// On by default: each drawn connection shows its latency and (when a weight formatter is given) its traffic.
+const showLatenciesParam = new URLParam("lgShowLatencies", true);
 // The config panel is collapsed by default so it doesn't eat the canvas; clicking the header expands it.
 const configOpenParam = new URLParam("lgConfigOpen", false);
 // Pixels per degree of latitude/longitude in geographic mode (equirectangular projection).
@@ -45,6 +46,9 @@ const POWER_EPS = 1e-9;
 // with a floor. Normalized to the median so the overall scale is latency-magnitude independent.
 const DIST_MIN_PX = 8;
 const DIST_UNIT_PX = 90;
+// A touch of deterministic jitter on the MDS init breaks symmetry, so a handful of nodes settle into a spread (a
+// triangle for three) instead of collapsing onto a single line.
+const INIT_JITTER_PX = 15;
 
 const MIN_OPACITY = 0.05;
 const NODE_RADIUS = 6;
@@ -65,9 +69,6 @@ const FIT_PAD = 60;
 const FIT_PAD_TOP = 160;
 // Auto-fit leaves a bit of extra breathing room by not zooming all the way in to fill the padded box.
 const FIT_ZOOM = 0.7;
-// Cursor must be within this many screen pixels of a node to hover/click it.
-const HOVER_HIT_PX = 60;
-const CLICK_HIT_PX = 40;
 
 type Edge = { a: number; b: number; latency: number; };
 type SolveEdge = { a: number; b: number; target: number; weight: number; };
@@ -293,6 +294,11 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         });
         this.positionsX = pos.x;
         this.positionsY = pos.y;
+        // Deterministic per-node jitter (different frequencies for x/y so it isn't itself collinear) to break symmetry.
+        for (let i = 0; i < n; i++) {
+            this.positionsX[i] += Math.sin(i * 12.9898 + 1) * INIT_JITTER_PX;
+            this.positionsY[i] += Math.cos(i * 78.233 + 1) * INIT_JITTER_PX;
+        }
     }
 
     // Classical MDS on an m-point set given a squared-target-distance function; top-2 eigenvectors.
@@ -622,6 +628,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         let hover = this.hoverNode;
         let connections = (this.neighbors[hover] || []).map(nb => nb.edge);
 
+        // Just highlight the hovered node's connections — their latency/traffic labels are already drawn all the time.
         ctx.lineWidth = 1.5;
         for (let edge of connections) {
             ctx.strokeStyle = "hsla(40, 90%, 60%, 0.6)";
@@ -630,19 +637,6 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             ctx.lineTo(toScreenX(this.positionsX[edge.b]), toScreenY(this.positionsY[edge.b]));
             ctx.stroke();
         }
-        ctx.font = "12px sans-serif";
-        ctx.textAlign = "center";
-        ctx.textBaseline = "middle";
-        for (let edge of connections) {
-            let midX = (toScreenX(this.positionsX[edge.a]) + toScreenX(this.positionsX[edge.b])) / 2;
-            let midY = (toScreenY(this.positionsY[edge.a]) + toScreenY(this.positionsY[edge.b])) / 2;
-            let label = formatTime(edge.latency);
-            let textWidth = ctx.measureText(label).width;
-            ctx.fillStyle = "hsla(0, 0%, 0%, 0.75)";
-            ctx.fillRect(midX - textWidth / 2 - 3, midY - 8, textWidth + 6, 16);
-            ctx.fillStyle = "hsl(40, 90%, 75%)";
-            ctx.fillText(label, midX, midY);
-        }
         // Redraw the hovered node's own label last so it always sits above the connection labels.
         let hoverLines = this.nodes[hover].labelLines;
         if (hoverLines && hoverLines.length) {
@@ -720,7 +714,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         this.lastPointerY = e.clientY;
     };
 
-    // Nearest node to a screen point, with its distance in pixels.
+    // Index of the node closest to a screen point (undefined only when there are no nodes).
     nearestNodeTo(mx: number, my: number) {
         let scale = this.viewScale;
         let bestDist = Infinity;
@@ -728,13 +722,13 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         for (let i = 0; i < this.nodes.length; i++) {
             let sx = this.viewWidth / 2 + (this.positionsX[i] - this.centroidX) * scale + this.panX;
             let sy = this.viewHeight / 2 + (this.positionsY[i] - this.centroidY) * scale + this.panY;
-            let dist = Math.sqrt((sx - mx) ** 2 + (sy - my) ** 2);
+            let dist = (sx - mx) ** 2 + (sy - my) ** 2;
             if (dist < bestDist) {
                 bestDist = dist;
                 best = i;
             }
         }
-        return { node: best, dist: bestDist };
+        return best;
     }
 
     onMouseMove = (e: MouseEvent) => {
@@ -754,8 +748,8 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         let my = e.clientY - rect.top;
         this.mouseX = mx;
         this.mouseY = my;
-        let nearest = this.nearestNodeTo(mx, my);
-        this.hoverNode = nearest.dist <= HOVER_HIT_PX ? nearest.node : undefined;
+        // Always hover whichever node the cursor is closest to, so moving anywhere explores the graph.
+        this.hoverNode = this.nearestNodeTo(mx, my);
         this.scheduleFrame();
     };
 
@@ -764,8 +758,8 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         if (!canvas || !this.onSelectNode) return;
         let rect = canvas.getBoundingClientRect();
         let nearest = this.nearestNodeTo(e.clientX - rect.left, e.clientY - rect.top);
-        if (nearest.node === undefined || nearest.dist > CLICK_HIT_PX) return;
-        this.onSelectNode(this.nodes[nearest.node].id);
+        if (nearest === undefined) return;
+        this.onSelectNode(this.nodes[nearest].id);
     };
 
     onMouseLeave = () => {
@@ -827,22 +821,27 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             this.builtSig = sig;
             this.builtWeightSig = weightSig;
             this.build();
-        } else if (weightSig !== this.builtWeightSig) {
-            // Traffic weights changed but the node/link set didn't — just refresh sizing, no re-layout.
-            this.builtWeightSig = weightSig;
-            this.computeWeights();
-        } else if (latencyExponentParam.value !== this.builtLatencyExponent) {
-            this.computeSolveEdges();
-            this.applyLayout();
-        } else if (Math.max(1, Math.floor(connectionsParam.value)) !== this.builtConnections) {
-            this.computeRenderEdges();
+        } else {
+            // Same node/link counts (same order): refresh the node objects so label text that changed as data streamed
+            // in — or when the traffic window toggled — shows up, without disturbing the settled layout/positions.
+            this.nodes = this.props.nodes.slice();
+            if (weightSig !== this.builtWeightSig) {
+                this.builtWeightSig = weightSig;
+                this.computeWeights();
+            }
+            if (latencyExponentParam.value !== this.builtLatencyExponent) {
+                this.computeSolveEdges();
+                this.applyLayout();
+            } else if (Math.max(1, Math.floor(connectionsParam.value)) !== this.builtConnections) {
+                this.computeRenderEdges();
+            }
         }
         this.scheduleFrame();
 
         return <div className={css.relative.fillBoth.overflowHidden}>
             <canvas
                 ref={elem => this.mountCanvas(elem ?? undefined)}
-                className={css.absolute.pos(0, 0).fillBoth}
+                className={css.absolute.pos(0, 0).fillBoth.cursor("pointer")}
             />
             <div className={css.vbox(10).pad2(12).absolute.pos(0, 0).width(260).zIndex(2).hsla(0, 0, 8, 0.9).colorhsl(0, 0, 85)}>
                 <div