querysub 0.524.0 → 0.526.0

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -20,6 +20,7 @@ import { getFunctionRunnerIndex, FunctionRunnerNodeInfo } from "../../4-querysub
 import { formatTime, formatNumber } from "socket-function/src/formatting/format";
 import { LatencyGraph, LatencyGraphNode, LatencyGraphLink, LatencyGraphLabelLine } from "../../library-components/LatencyGraph";
 import { URLParam } from "../../library-components/URLParam";
+import { mainResets } from "../../library-components/urlResetGroups";
 
 const ID_CHARS = 8;
 // Green means querysub, blue means path value, purple means function runner.
@@ -31,10 +32,11 @@ const PROBE_TIMEOUT_MS = 5000;
 const RANGE_BAR_WIDTH_PX = 360;
 const RANGE_BAR_HEIGHT_PX = 14;
 const LATENCY_GRAPH_HEIGHT_PX = 720;
+const MACHINE_LATENCY_WIDTH_PX = 235;
 
 // Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
-const selectedMachineParam = new URLParam("rtSelectedMachine", "");
-const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60);
+const selectedMachineParam = new URLParam("rtSelectedMachine", "", { reset: [mainResets] });
+const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60, { reset: [mainResets] });
 
 // The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
@@ -227,6 +229,43 @@ function machineIdOf(nodeId: string): string {
     return parts?.machineId || nodeId;
 }
 
+function threadLabel(nodeId: string): string {
+    let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
+    return `${(parts?.threadId || "?").slice(0, ID_CHARS)}:${parts?.port ?? "?"}`;
+}
+
+// min · median · max of one source node's latencies to a machine's threads (plus a tooltip naming the farthest
+// thread), and the node's total path values sent/received to that machine (summed — the per-thread split isn't interesting).
+function machineLatencyCell(config: {
+    sourceLatencies: { [nodeId: string]: number } | undefined;
+    threads: string[];
+    traffic: TrafficStats | undefined;
+}): { text: string; tooltip: string } {
+    let { sourceLatencies, threads, traffic } = config;
+    if (!sourceLatencies) return { text: "", tooltip: "" };
+    let entries: { thread: string; ms: number }[] = [];
+    for (let thread of threads) {
+        let ms = sourceLatencies[thread];
+        if (Number.isFinite(ms)) entries.push({ thread, ms });
+    }
+    if (!entries.length) return { text: "", tooltip: "" };
+    sort(entries, entry => entry.ms);
+    let farthest = entries[entries.length - 1];
+    let text = `${formatTime(entries[0].ms)} · ${formatTime(entries[Math.floor(entries.length / 2)].ms)} · ${formatTime(farthest.ms)}`;
+    let pvSent = 0;
+    let pvReceived = 0;
+    for (let thread of threads) {
+        let data = traffic?.perNode?.[thread];
+        if (!data) continue;
+        pvSent += data.pathValuesSent || 0;
+        pvReceived += data.pathValuesReceived || 0;
+    }
+    if (pvSent || pvReceived) {
+        text += `  ↑${formatNumber(pvSent)}/s ↓${formatNumber(pvReceived)}/s values`;
+    }
+    return { text, tooltip: `${text}\nfarthest: ${threadLabel(farthest.thread)} (${formatTime(farthest.ms)})` };
+}
+
 // Summed traffic metrics across a set of nodes (a machine's threads).
 function sumTraffic(threads: string[], trafficMaps: Map<string, TrafficStats>) {
     let dataSent = 0;
@@ -313,7 +352,7 @@ function buildNodeRow(config: {
     if (runner) {
         if (runner.networks.length) fnParts.push(runner.networks.join(" "));
         for (let shard of runner.shards) {
-            fnParts.push(`fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
+            fnParts.push(`${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
         }
         fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)}/s calls`);
         if (!runner.isPublic) fnParts.push("PRIVATE");
@@ -322,7 +361,7 @@ function buildNodeRow(config: {
     let pvParts: string[] = [];
     let spec = info?.spec;
     if (spec && spec.routeStart >= 0 && spec.routeEnd >= 0) {
-        pvParts.push(`pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
+        pvParts.push(`${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
     }
     if (traffic && (traffic.pathValuesSent || traffic.pathValuesReceived)) {
         pvParts.push(`↑${formatNumber(traffic.pathValuesSent)}/s ↓${formatNumber(traffic.pathValuesReceived)}/s values`);
@@ -344,19 +383,30 @@ function buildNodeRow(config: {
     };
 }
 
-class NodeInfoTable extends qreact.Component<{ rows: NodeRow[]; selectedMachine: string }> {
+class NodeInfoTable extends qreact.Component<{
+    rows: NodeRow[];
+    selectedMachine: string;
+    machineColumns: { id: string; label: string }[];
+    nodeMachineLatency: Map<string, { [machineId: string]: { text: string; tooltip: string } }>;
+}> {
     render() {
         let rows = this.props.rows;
         let selected = this.props.selectedMachine;
+        let machineColumns = this.props.machineColumns;
+        let nodeMachineLatency = this.props.nodeMachineLatency;
         return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
             <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
                 {NODE_TABLE_COLUMNS.map(col =>
                     <div className={css.width(col.width).flexShrink0.pad2(6).ellipsis}>{col.label}</div>
                 )}
+                {machineColumns.map(col =>
+                    <div className={css.width(MACHINE_LATENCY_WIDTH_PX).flexShrink0.pad2(6).ellipsis} title={`latency and path values to ${col.label}`}>→ {col.label}</div>
+                )}
             </div>
             {rows.map((row, i) => {
                 let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
                 let isSelected = row.machineId === selected;
+                let latencies = nodeMachineLatency.get(row.nodeId) || {};
                 return <div
                     className={css.hbox(0).button.fillWidth.hsl(0, 0, isSelected ? 92 : 99)
                         .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
@@ -370,6 +420,42 @@ class NodeInfoTable extends qreact.Component<{ rows: NodeRow[]; selectedMachine:
                             title={text}
                         >{text}</div>;
                     })}
+                    {machineColumns.map(col => {
+                        let cell = latencies[col.id];
+                        return <div
+                            className={css.width(MACHINE_LATENCY_WIDTH_PX).flexShrink0.pad2(6).ellipsis.colorhsl(0, 0, 25)}
+                            title={cell?.tooltip || ""}
+                        >{cell?.text || ""}</div>;
+                    })}
+                </div>;
+            })}
+        </div>;
+    }
+}
+
+// Nodes that never reported their latencies — all we have is their nodeId (and possibly a DNS-resolved machine IP), so this is a much narrower table than NodeInfoTable, still grouped by machine.
+class UnresponsiveNodesTable extends qreact.Component<{ nodeIds: string[]; machineIp: Map<string, string> }> {
+    render() {
+        let rows = this.props.nodeIds.map(nodeId => ({ nodeId, machineId: machineIdOf(nodeId) }));
+        let sep = String.fromCharCode(1);
+        sort(rows, row => `${row.machineId}${sep}${threadLabel(row.nodeId)}`);
+        return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
+            <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
+                <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>IP</div>
+                <div className={css.width(96).flexShrink0.pad2(6).ellipsis}>Machine</div>
+                <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>Thread:Port</div>
+                <div className={css.width(520).flexShrink0.pad2(6).ellipsis}>Node Id</div>
+            </div>
+            {rows.map((row, i) => {
+                let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
+                return <div
+                    className={css.hbox(0).fillWidth.hsl(0, 0, 99)
+                        .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
+                >
+                    <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>{firstOfMachine && this.props.machineIp.get(row.machineId) || ""}</div>
+                    <div className={css.width(96).flexShrink0.pad2(6).ellipsis}>{firstOfMachine && row.machineId.slice(0, ID_CHARS) || ""}</div>
+                    <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>{threadLabel(row.nodeId)}</div>
+                    <div className={css.width(520).flexShrink0.pad2(6).ellipsis.colorhsl(0, 0, 45)} title={row.nodeId}>{row.nodeId}</div>
                 </div>;
             })}
         </div>;
@@ -453,19 +539,56 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             if (unique.size) machineNetworks.set(machineId, [...unique]);
         }
 
+        // One latency column per machine: each row (node) gets its own min · median · max latency to that machine's
+        // threads (a machine has several threads, so a cell is a range, not one number).
+        let machineColumns = [...machineAllThreads.keys()].map(id => {
+            let networks = machineNetworks.get(id) || [];
+            let label = networks.length ? `${id.slice(0, ID_CHARS)} (${networks.join(" | ")})` : id.slice(0, ID_CHARS);
+            return { id, label, threads: machineAllThreads.get(id) || [] };
+        });
+        sort(machineColumns, column => column.id);
+        let nodeMachineLatency = new Map<string, { [machineId: string]: { text: string; tooltip: string } }>();
+        for (let nodeId of nodeIds) {
+            let sourceLatencies = latencyMaps.get(nodeId);
+            let traffic = trafficMaps.get(nodeId);
+            let byMachine: { [machineId: string]: { text: string; tooltip: string } } = {};
+            for (let column of machineColumns) {
+                let cell = machineLatencyCell({ sourceLatencies, threads: column.threads, traffic });
+                if (cell.text) byMachine[column.id] = cell;
+            }
+            nodeMachineLatency.set(nodeId, byMachine);
+        }
+
         // Total bytes on each thread-pair link (both directions). Both endpoints report the same total, so take the max.
         let pairKey = (a: string, b: string) => a < b ? `${a}|${b}` : `${b}|${a}`;
         let pairTraffic = new Map<string, number>();
+        // Path values per thread pair, split by direction (fwd = lower id → higher id). Both endpoints report each
+        // direction (one as sent, one as received), so take the max of the two estimates.
+        let pairPv = new Map<string, { fwd: number; back: number }>();
         for (let [reporter, traffic] of trafficMaps) {
             for (let [peer, data] of Object.entries(traffic.perNode || {})) {
                 if (reporter === peer || !respondedSet.has(peer)) continue;
                 let key = pairKey(reporter, peer);
                 pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, (data?.sent || 0) + (data?.received || 0)));
+                let pv = pairPv.get(key);
+                if (!pv) {
+                    pv = { fwd: 0, back: 0 };
+                    pairPv.set(key, pv);
+                }
+                let sentRate = data?.pathValuesSent || 0;
+                let receivedRate = data?.pathValuesReceived || 0;
+                if (reporter < peer) {
+                    pv.fwd = Math.max(pv.fwd, sentRate);
+                    pv.back = Math.max(pv.back, receivedRate);
+                } else {
+                    pv.fwd = Math.max(pv.fwd, receivedRate);
+                    pv.back = Math.max(pv.back, sentRate);
+                }
             }
         }
 
-        // Machine-to-machine latency = mean over all cross-machine thread-pair latencies; traffic = summed pair traffic.
-        let machinePairLatency = new Map<string, { sum: number; count: number; }>();
+        // Machine-to-machine latency = minimum over all cross-machine thread-pair latencies; traffic = summed pair traffic.
+        let machinePairLatency = new Map<string, number>();
         for (let [sourceId, latencies] of latencyMaps) {
             let sourceMachine = machineIdOf(sourceId);
             for (let [destId, latencyMs] of Object.entries(latencies)) {
@@ -473,13 +596,8 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
                 let destMachine = machineIdOf(destId);
                 if (sourceMachine === destMachine) continue;
                 let key = pairKey(sourceMachine, destMachine);
-                let agg = machinePairLatency.get(key);
-                if (!agg) {
-                    agg = { sum: 0, count: 0 };
-                    machinePairLatency.set(key, agg);
-                }
-                agg.sum += latencyMs;
-                agg.count++;
+                let existing = machinePairLatency.get(key);
+                machinePairLatency.set(key, existing === undefined ? latencyMs : Math.min(existing, latencyMs));
             }
         }
         let machinePairTraffic = new Map<string, number>();
@@ -491,6 +609,28 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             let mk = pairKey(ma, mb);
             machinePairTraffic.set(mk, (machinePairTraffic.get(mk) || 0) + weight);
         }
+        // Sum thread-pair path value flows up to machine pairs, keeping direction (fwd = lower machine id → higher).
+        let machinePairPv = new Map<string, { fwd: number; back: number }>();
+        for (let [key, pv] of pairPv) {
+            let [a, b] = key.split("|");
+            let ma = machineIdOf(a);
+            let mb = machineIdOf(b);
+            if (ma === mb) continue;
+            let mk = pairKey(ma, mb);
+            let entry = machinePairPv.get(mk);
+            if (!entry) {
+                entry = { fwd: 0, back: 0 };
+                machinePairPv.set(mk, entry);
+            }
+            // The thread pair's fwd direction may be flipped relative to the machine pair's ordering.
+            if (ma < mb) {
+                entry.fwd += pv.fwd;
+                entry.back += pv.back;
+            } else {
+                entry.fwd += pv.back;
+                entry.back += pv.fwd;
+            }
+        }
 
         let nodes: LatencyGraphNode[] = [...machineRespondedThreads.keys()].map(machineId => {
             let allThreads = machineAllThreads.get(machineId) || [];
@@ -502,18 +642,25 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             };
         });
         let links: LatencyGraphLink[] = [];
-        for (let [key, agg] of machinePairLatency) {
+        for (let [key, latencyMs] of machinePairLatency) {
             let [source, destination] = key.split("|");
-            links.push({ source, destination, latencyMs: agg.sum / agg.count, weight: machinePairTraffic.get(key) || 0 });
+            let pv = machinePairPv.get(key);
+            let extraLabel: LatencyGraphLabelLine | undefined;
+            if (pv && (pv.fwd || pv.back)) {
+                extraLabel = { text: `↑${formatNumber(pv.fwd)}/s ↓${formatNumber(pv.back)}/s values`, color: PATHVALUE_COLOR };
+            }
+            links.push({ source, destination, latencyMs, weight: machinePairTraffic.get(key) || 0, extraLabel });
         }
 
         // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
         // Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
-        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
+        let rows = nodeIds.filter(nodeId => respondedSet.has(nodeId)).map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
         let selected = selectedMachineParam.value;
         let sep = String.fromCharCode(1);
         sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
 
+        let unresponsiveNodeIds = nodeIds.filter(nodeId => !respondedSet.has(nodeId));
+
         return <div className={css.vbox(8).fillWidth}>
             <div className={css.hbox(14)}>
                 <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
@@ -539,7 +686,11 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             </div>
             <div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
             <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
-            <NodeInfoTable rows={rows} selectedMachine={selected} />
+            <NodeInfoTable rows={rows} selectedMachine={selected} machineColumns={machineColumns} nodeMachineLatency={nodeMachineLatency} />
+            {unresponsiveNodeIds.length > 0 && <>
+                <h2 className={css.margin(0)}>Unresponsive Nodes ({unresponsiveNodeIds.length})</h2>
+                <UnresponsiveNodesTable nodeIds={unresponsiveNodeIds} machineIp={machineIp} />
+            </>}
         </div>;
     }
 }

package/src/diagnostics/pathAuditer.ts

@@ -204,6 +204,7 @@ async function processPendingValidityChecks(now: number) {
     let splitIndex = binarySearchBasic2(pendingValidityChecks, check => check.discoveredAt, { discoveredAt: threshold } as PendingValidityCheck);
     if (splitIndex < 0) splitIndex = ~splitIndex;
 
+    let pathsToResolve = new Set<string>();
     for (let i = 0; i < splitIndex; i++) {
         let pendingCheck = pendingValidityChecks[i];
         let currentValue = authorityStorage.getValueAtOrBeforeTime(pendingCheck.path);
@@ -215,15 +216,14 @@ async function processPendingValidityChecks(now: number) {
                 ourValid: pendingCheck.ourValid,
                 remoteValid: pendingCheck.remoteValid,
                 remoteNodeId: pendingCheck.remoteNodeId,
-                reason: "Pending validity check aged past MAX_CHANGE_AGE. Forcing sync to all nodes.",
+                reason: "Pending validity check aged past MAX_CHANGE_AGE. Resolving the valid state to valid on all authorities.",
             });
-            let allAuthorities = PathRouter.getAllAuthoritiesForValues([currentValue]);
-            for (let [authorityNodeId, _] of allAuthorities.entries()) {
-                let serialized = await pathValueSerializer.serialize([currentValue]);
-                await PathAuditerController.nodes[authorityNodeId].ingestPathValues(serialized);
-            }
+            pathsToResolve.add(pendingCheck.path);
         }
     }
+    if (pathsToResolve.size > 0) {
+        await resolveValidStateDisagreements(pathsToResolve);
+    }
 
     if (splitIndex > 0) {
         for (let i = 0; i < splitIndex; i++) {
@@ -233,6 +233,73 @@ async function processPendingValidityChecks(now: number) {
     }
 }
 
+async function resolveValidStateDisagreements(paths: Set<string>) {
+    let values: PathValue[] = [];
+    for (let path of paths) {
+        let value = authorityStorage.getValueAtOrBeforeTime(path);
+        if (value) {
+            values.push(value);
+        }
+    }
+    if (values.length === 0) return;
+
+    console.info(`Resolving valid state disagreements`, { pathCount: values.length });
+
+    let otherAuthorities = PathRouter.getAllAuthoritiesForValues(values);
+    let anyValidByPath = new Map<string, boolean>();
+    for (let value of values) {
+        anyValidByPath.set(value.path, !!value.valid);
+    }
+    for (let [authorityNodeId, authorityValues] of otherAuthorities.entries()) {
+        let requests = authorityValues.map(value => ({ path: value.path, time: value.time }));
+        let responses = await PathAuditerController.nodes[authorityNodeId].getValidStates(requests);
+        for (let response of responses) {
+            if (response.valid) {
+                anyValidByPath.set(response.path, true);
+            }
+        }
+    }
+
+    let forcedByPath = new Map<string, PathValue>();
+    for (let value of values) {
+        if (!anyValidByPath.get(value.path)) continue;
+        console.info(`Resolving valid state disagreement to valid`, {
+            path: value.path,
+            timeId: value.time.time,
+            timeIdFull: value.time,
+            ourValid: value.valid,
+        });
+        if (value.valid) {
+            forcedByPath.set(value.path, value);
+        } else {
+            forcedByPath.set(value.path, { ...value, valid: true });
+        }
+        stats.fixedCount++;
+    }
+    if (forcedByPath.size === 0) return;
+
+    for (let [authorityNodeId, authorityValues] of otherAuthorities.entries()) {
+        let toSend: PathValue[] = [];
+        for (let value of authorityValues) {
+            let forced = forcedByPath.get(value.path);
+            if (forced) {
+                toSend.push(forced);
+            }
+        }
+        if (toSend.length === 0) continue;
+        let serialized = await pathValueSerializer.serialize(toSend);
+        await PathAuditerController.nodes[authorityNodeId].ingestPathValues(serialized, true);
+    }
+
+    validStateComputer.ingestValuesAndValidStates({
+        pathValues: Array.from(forcedByPath.values()),
+        parentSyncs: [],
+        initialTriggers: { values: new Set(), parentPaths: new Set() },
+        doNotArchive: true,
+        forceValidStates: true,
+    });
+}
+
 function trackSyncAge(config: {
     path: string;
     ourTimeId: number;
@@ -364,7 +431,7 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
                         ourValid: ourExact.valid,
                         remoteValid: response.valid,
                         remoteNodeId: nodeId,
-                        reason: "Remote says our value is invalid, but we think it's valid. Telling all nodes about this value to ensure it's in sync everywhere.",
+                        reason: "Remote says our value is invalid, but we think it's valid. Resolving the valid state to valid on all authorities.",
                     });
                 } else {
                     if (!pendingValidityCheckPaths.has(request.path)) {
@@ -439,16 +506,7 @@ async function auditAuthority(nodeId: string, pathsToAudit: { path: string }[],
     }
 
     if (pathsToForceSync.size > 0) {
-        for (let path of pathsToForceSync) {
-            let valueToForce = authorityStorage.getValueAtOrBeforeTime(path);
-            if (valueToForce) {
-                let allAuthorities = PathRouter.getAllAuthoritiesForValues([valueToForce]);
-                for (let [authorityNodeId, _] of allAuthorities.entries()) {
-                    let serialized = await pathValueSerializer.serialize([valueToForce]);
-                    await PathAuditerController.nodes[authorityNodeId].ingestPathValues(serialized);
-                }
-            }
-        }
+        await resolveValidStateDisagreements(pathsToForceSync);
     }
 
     stats.totalAudited += pathsToAudit.length;
@@ -505,13 +563,14 @@ class PathAuditerService {
         return await pathValueSerializer.serialize(results);
     }
 
-    public async ingestPathValues(serializedValues: Buffer[]): Promise<void> {
+    public async ingestPathValues(serializedValues: Buffer[], forceValidStates?: boolean): Promise<void> {
         let values = await pathValueSerializer.deserialize(serializedValues);
         validStateComputer.ingestValuesAndValidStates({
             pathValues: values,
             parentSyncs: [],
             initialTriggers: { values: new Set(), parentPaths: new Set() },
             doNotArchive: true,
+            forceValidStates,
         });
     }
 

package/src/diagnostics/periodic.ts

@@ -78,7 +78,8 @@ if (isNode()) {
         }
 
         if (data.toString().includes("\r")) {
-            logAll();
+            import("./watchdog").then(m => m.logUnfiltered());
+            //logAll();
         }
     });
 

package/src/diagnostics/watchdog.ts

@@ -116,6 +116,17 @@ function logProfileMeasuresTimingsNow() {
         thresholdInTable: 0
     });
 };
+export function logUnfiltered(depth = 2) {
+    let profile = measureObj.finish();
+    measureObj = startMeasure();
+    logMeasureTable(profile, {
+        name: `all logs at ${new Date().toLocaleString()}`,
+        mergeDepth: depth,
+        minTimeToLog: 0,
+        maxTableEntries: 10000000,
+        thresholdInTable: 0
+    });
+};
 
 
 registerPeriodic(logProfileMeasuresTimingsNow);

package/src/library-components/LatencyGraph.tsx

@@ -1,6 +1,7 @@
 import { qreact } from "../4-dom/qreact";
 import { Button } from "./Button";
 import { URLParam } from "./URLParam";
+import { mainResets } from "./urlResetGroups";
 import { InputLabelURL } from "./InputLabel";
 import { css } from "typesafecss";
 import { sort } from "socket-function/src/misc";
@@ -8,7 +9,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 
 export type LatencyGraphLabelLine = { text: string; color?: string; };
 export type LatencyGraphNode = { id: string; label?: string; latitude?: number; longitude?: number; labelLines?: LatencyGraphLabelLine[]; weight?: number; };
-export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; };
+export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; extraLabel?: LatencyGraphLabelLine; };
 export type LatencyGraphProps = {
     nodes: LatencyGraphNode[];
     links: LatencyGraphLink[];
@@ -22,15 +23,15 @@ export type LatencyGraphProps = {
 
 const DEFAULT_CONNECTIONS = 4;
 // Nearest neighbors drawn per node (the layout still uses every measured latency; this only thins the drawn lines).
-const connectionsParam = new URLParam("lgConnections", DEFAULT_CONNECTIONS);
+const connectionsParam = new URLParam("lgConnections", DEFAULT_CONNECTIONS, { reset: [mainResets] });
 const DEFAULT_LATENCY_EXPONENT = 0.5;
 // Exponent on (latency / reference) when mapping to layout distance, so far nodes push apart harder as it grows.
-const latencyExponentParam = new URLParam("lgLatencyExponent", DEFAULT_LATENCY_EXPONENT);
-const geoParam = new URLParam("lgGeo", false);
+const latencyExponentParam = new URLParam("lgLatencyExponent", DEFAULT_LATENCY_EXPONENT, { reset: [mainResets] });
+const geoParam = new URLParam("lgGeo", false, { reset: [mainResets] });
 // On by default: each drawn connection shows its latency and (when a weight formatter is given) its traffic.
-const showLatenciesParam = new URLParam("lgShowLatencies", true);
+const showLatenciesParam = new URLParam("lgShowLatencies", true, { reset: [mainResets] });
 // The config panel is collapsed by default so it doesn't eat the canvas; clicking the header expands it.
-const configOpenParam = new URLParam("lgConfigOpen", false);
+const configOpenParam = new URLParam("lgConfigOpen", false, { reset: [mainResets] });
 // Pixels per degree of latitude/longitude in geographic mode (equirectangular projection).
 const GEO_SCALE = 6;
 
@@ -62,7 +63,6 @@ const LINE_MAX_WIDTH = 7;
 const LABEL_LINE_HEIGHT = 13;
 const ZOOM_STEP = 1.1;
 const MIN_SCALE = 0.05;
-const MAX_SCALE = 12;
 // Screen-space margins left around the content when auto-fitting. The top gets much more room because node labels
 // stack upward above the nodes (and are drawn at a fixed screen size regardless of zoom).
 const FIT_PAD = 60;
@@ -84,6 +84,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
     maxNodeWeight = 0;
     pairWeight = new Map<number, number>();
     maxPairWeight = 0;
+    pairExtraLabel = new Map<number, LatencyGraphLabelLine>();
     // Cached from props in render() so draw() (a rAF callback, not synced) never reads this.props.
     formatWeight: ((weight: number) => string) | undefined = undefined;
     selectedId: string | undefined = undefined;
@@ -135,6 +136,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         this.positionsY = new Float64Array(n);
         this.ingestLinks();
         this.computeWeights();
+        this.computePairLabels();
         this.computeSolveEdges();
         this.computeRenderEdges();
         this.applyLayout();
@@ -166,6 +168,19 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         }
     }
 
+    computePairLabels() {
+        let n = this.nodes.length;
+        let index = this.nodeIndex();
+        this.pairExtraLabel = new Map();
+        for (let link of this.props.links) {
+            if (!link.extraLabel) continue;
+            let a = index.get(link.source);
+            let b = index.get(link.destination);
+            if (a === undefined || b === undefined || a === b) continue;
+            this.pairExtraLabel.set(Math.min(a, b) * n + Math.max(a, b), link.extraLabel);
+        }
+    }
+
     nodeIndex() {
         let index = new Map<string, number>();
         for (let [i, node] of this.nodes.entries()) {
@@ -519,7 +534,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             let fit = Math.min((width - 2 * FIT_PAD) / worldW, (height - FIT_PAD_TOP - FIT_PAD) / worldH) * FIT_ZOOM;
             // Only apply the fit if it's sane — never let a NaN/degenerate value nuke the whole view.
             if (Number.isFinite(fit) && fit > 0 && Number.isFinite(minX) && Number.isFinite(minY)) {
-                this.viewScale = Math.max(MIN_SCALE, Math.min(MAX_SCALE, fit));
+                this.viewScale = Math.max(MIN_SCALE, fit);
                 this.panX = -((minX + maxX) / 2 - centroidX) * this.viewScale;
                 // Center within the padded band, which sits lower than the middle because of the reserved top margin.
                 let targetCenterY = (FIT_PAD_TOP + (height - FIT_PAD)) / 2;
@@ -558,9 +573,14 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             let midX = (toScreenX(this.positionsX[edge.a]) + toScreenX(this.positionsX[edge.b])) / 2;
             let midY = (toScreenY(this.positionsY[edge.a]) + toScreenY(this.positionsY[edge.b])) / 2;
 
-            let traffic = this.pairWeight.get(Math.min(edge.a, edge.b) * this.nodes.length + Math.max(edge.a, edge.b)) || 0;
-            let trafficLabel = this.formatWeight && traffic > 0 ? this.formatWeight(traffic) : undefined;
-            let latencyY = trafficLabel ? midY - 7 : midY;
+            let pk = Math.min(edge.a, edge.b) * this.nodes.length + Math.max(edge.a, edge.b);
+            let traffic = this.pairWeight.get(pk) || 0;
+            let subLabels: LatencyGraphLabelLine[] = [];
+            let trafficLabel = this.formatWeight && traffic > 0 && this.formatWeight(traffic) || undefined;
+            if (trafficLabel) subLabels.push({ text: trafficLabel });
+            let extraLabel = this.pairExtraLabel.get(pk);
+            if (extraLabel) subLabels.push(extraLabel);
+            let latencyY = subLabels.length ? midY - 7 : midY;
 
             ctx.font = "10px sans-serif";
             let latencyLabel = formatTime(edge.latency);
@@ -570,13 +590,14 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             ctx.fillStyle = "hsl(0, 0%, 72%)";
             ctx.fillText(latencyLabel, midX, latencyY);
 
-            if (trafficLabel) {
+            for (let [si, subLabel] of subLabels.entries()) {
+                let sy = midY + 7 + si * 12;
                 ctx.font = "9px sans-serif";
-                let trafficWidth = ctx.measureText(trafficLabel).width;
+                let subWidth = ctx.measureText(subLabel.text).width;
                 ctx.fillStyle = "hsla(0, 0%, 0%, 0.5)";
-                ctx.fillRect(midX - trafficWidth / 2 - 2, midY + 7 - 6, trafficWidth + 4, 12);
-                ctx.fillStyle = "hsl(0, 0%, 58%)";
-                ctx.fillText(trafficLabel, midX, midY + 7);
+                ctx.fillRect(midX - subWidth / 2 - 2, sy - 6, subWidth + 4, 12);
+                ctx.fillStyle = subLabel.color || "hsl(0, 0%, 58%)";
+                ctx.fillText(subLabel.text, midX, sy);
             }
         }
     }
@@ -698,7 +719,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         let worldX = this.centroidX + (e.offsetX - this.viewWidth / 2 - this.panX) / this.viewScale;
         let worldY = this.centroidY + (e.offsetY - this.viewHeight / 2 - this.panY) / this.viewScale;
         let factor = e.deltaY < 0 ? ZOOM_STEP : 1 / ZOOM_STEP;
-        let newScale = Math.max(MIN_SCALE, Math.min(MAX_SCALE, this.viewScale * factor));
+        let newScale = Math.max(MIN_SCALE, this.viewScale * factor);
         this.viewScale = newScale;
         this.panX = e.offsetX - this.viewWidth / 2 - (worldX - this.centroidX) * newScale;
         this.panY = e.offsetY - this.viewHeight / 2 - (worldY - this.centroidY) * newScale;
@@ -748,8 +769,10 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         let my = e.clientY - rect.top;
         this.mouseX = mx;
         this.mouseY = my;
-        // Always hover whichever node the cursor is closest to, so moving anywhere explores the graph.
-        this.hoverNode = this.nearestNodeTo(mx, my);
+        // The move listener is on window (so panning keeps working off-canvas), so only hover while actually over the
+        // canvas — otherwise the hover would stick forever once the cursor leaves.
+        let inside = 0 <= mx && mx < rect.width && 0 <= my && my < rect.height;
+        this.hoverNode = inside ? this.nearestNodeTo(mx, my) : undefined;
         this.scheduleFrame();
     };
 
@@ -825,6 +848,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             // Same node/link counts (same order): refresh the node objects so label text that changed as data streamed
             // in — or when the traffic window toggled — shows up, without disturbing the settled layout/positions.
             this.nodes = this.props.nodes.slice();
+            this.computePairLabels();
             if (weightSig !== this.builtWeightSig) {
                 this.builtWeightSig = weightSig;
                 this.computeWeights();

package/src/server.ts

@@ -1,3 +1,10 @@
+import { disableMeasurements } from "socket-function/src/profiling/measure";
+// NOTE: Profiling seems to make us use about twice as much memory, at least in the bad case where we have many tiny watchers.
+if (typeof document === "undefined" && (process.argv.includes("--noprofile") || process.argv.includes("--nprofile"))) {
+    disableMeasurements();
+}
+
+
 import "./forceProduction";
 import "./inject";