querysub 0.519.0 → 0.521.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.519.0",
+  "version": "0.521.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -70,7 +70,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.5",
-    "socket-function": "^1.2.11",
+    "socket-function": "^1.2.12",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-f-node-discovery/TrafficTracking.ts

@@ -25,6 +25,7 @@ export type NodeDataTraffic = { sent: number; received: number; };
 export type TrafficStats = {
     functionsExecuted: number;
     pathValuesSent: number;
+    pathValuesReceived: number;
     querysubCalls: number;
     // Bytes to/from each known network node, plus one aggregate for everything outside the node list.
     perNode: { [nodeId: string]: NodeDataTraffic };
@@ -41,6 +42,7 @@ function oldestBucket() {
 // bucket => count
 let functionsExecuted = new Map<number, number>();
 let pathValuesSent = new Map<number, number>();
+let pathValuesReceived = new Map<number, number>();
 let querysubCalls = new Map<number, number>();
 // raw connection nodeId => (bucket => bytes)
 let bytesSent = new Map<string, Map<number, number>>();
@@ -66,6 +68,10 @@ export function recordPathValuesSent(count: number) {
     if (count <= 0) return;
     addSimple(pathValuesSent, count);
 }
+export function recordPathValuesReceived(count: number) {
+    if (count <= 0) return;
+    addSimple(pathValuesReceived, count);
+}
 export function recordQuerysubCall() {
     addSimple(querysubCalls, 1);
 }
@@ -131,6 +137,7 @@ export function getTrafficStats(): TrafficStats {
     return {
         functionsExecuted: sumRecent(functionsExecuted),
         pathValuesSent: sumRecent(pathValuesSent),
+        pathValuesReceived: sumRecent(pathValuesReceived),
         querysubCalls: sumRecent(querysubCalls),
         perNode: data.perNode,
         outside: data.outside,

package/src/0-path-value-core/PathValueController.ts

@@ -12,7 +12,7 @@ import { debugNodeId, debugNodeThread } from "../-c-identity/IdentityController"
 import { getSlowdown, isDiskAudit, getDomain } from "../config";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { areNodeIdsEqual, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { recordPathValuesSent } from "../-f-node-discovery/TrafficTracking";
+import { recordPathValuesSent, recordPathValuesReceived } from "../-f-node-discovery/TrafficTracking";
 import { getNodeIdIP } from "socket-function/src/nodeCache";
 import { authorityLookup } from "./AuthorityLookup";
 import { timeoutToError } from "../errors";
@@ -148,6 +148,7 @@ export class PathValueControllerBase {
         if (valueBuffers) {
             values = await pathValueSerializer.deserialize(valueBuffers);
             ActionsHistory.OnRead(values);
+            recordPathValuesReceived(values.length);
         }
 
         if (initialCreation) {

package/src/deployManager/components/ServiceDetailPage.tsx

@@ -169,20 +169,24 @@ export class ServiceDetailPage extends qreact.Component {
     private renderTemplateVariables(config: ServiceConfig) {
         let templateVariables = getCommandTemplateVariables(config.parameters.command);
         let targets = getMachineTargets(config.parameters);
-        let anyVariablesSet = targets.some(target => Object.keys(target.variables).length > 0);
-        if (templateVariables.length === 0 && !anyVariablesSet) return undefined;
+        if (targets.length === 0) return undefined;
 
-        const setVariable = (entryIndex: number, name: string, value: string) => {
+        const updateTargets = (change: (targets: ReturnType<typeof getMachineTargets>) => void) => {
             let updated = deepCloneJSON(config);
             let updatedTargets = getMachineTargets(updated.parameters);
-            if (value) {
-                updatedTargets[entryIndex].variables[name] = value;
-            } else {
-                delete updatedTargets[entryIndex].variables[name];
-            }
+            change(updatedTargets);
             setMachineTargets(updated.parameters, updatedTargets);
             this.updateEditorState(updated);
         };
+        const setVariable = (entryIndex: number, name: string, value: string) => {
+            updateTargets(targets => {
+                if (value) {
+                    targets[entryIndex].variables[name] = value;
+                } else {
+                    delete targets[entryIndex].variables[name];
+                }
+            });
+        };
 
         let dupIndexes = new Map<string, number>();
         return <div className={css.vbox(10).fillWidth.pad2(12).bord2(0, 0, 20)}>
@@ -201,6 +205,26 @@ export class ServiceDetailPage extends qreact.Component {
                             onChangeValue={value => setVariable(entryIndex, name, value)}
                         />;
                     })}
+                    <button
+                        className={css.pad2(10, 4).button.bord2(0, 0, 20).hsl(120, 70, 90)}
+                        title="Duplicate this entry (same machine and variables)"
+                        onClick={() => {
+                            updateTargets(targets => {
+                                targets.splice(entryIndex + 1, 0, deepCloneJSON(targets[entryIndex]));
+                            });
+                        }}>
+                        +
+                    </button>
+                    <button
+                        className={css.pad2(10, 4).button.bord2(0, 0, 20).hsl(0, 70, 90)}
+                        title="Remove this entry"
+                        onClick={() => {
+                            updateTargets(targets => {
+                                targets.splice(entryIndex, 1);
+                            });
+                        }}>
+                        −
+                    </button>
                 </div>;
             })}
         </div>;

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -8,6 +8,8 @@ import { LatencyController } from "../../-f-node-discovery/LatencyTracking";
 import { TrafficController, TrafficStats } from "../../-f-node-discovery/TrafficTracking";
 import { getDomain } from "../../config";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
+import { getNodeIdDomainMaybeUndefined } from "socket-function/src/nodeCache";
+import dns from "dns";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../managementPages";
 import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
@@ -17,6 +19,7 @@ import type { AuthoritySpec } from "../../0-path-value-core/PathRouter";
 import { getFunctionRunnerIndex, FunctionRunnerNodeInfo } from "../../4-querysub/FunctionRunnerTracking";
 import { formatTime, formatNumber } from "socket-function/src/formatting/format";
 import { LatencyGraph, LatencyGraphNode, LatencyGraphLink, LatencyGraphLabelLine } from "../../library-components/LatencyGraph";
+import { URLParam } from "../../library-components/URLParam";
 
 const ID_CHARS = 8;
 // Green means querysub, blue means path value, purple means function runner.
@@ -29,13 +32,66 @@ const RANGE_BAR_WIDTH_PX = 360;
 const RANGE_BAR_HEIGHT_PX = 14;
 const LATENCY_GRAPH_HEIGHT_PX = 720;
 
+// Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
+const selectedMachineParam = new URLParam("rtSelectedMachine", "");
+
+// The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
+// `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
+const NODE_TABLE_COLUMNS: { key: keyof NodeRow; label: string; width: number; color?: string; perMachine?: boolean; }[] = [
+    { key: "ip", label: "IP", width: 130, perMachine: true },
+    { key: "machineShort", label: "Machine", width: 96, perMachine: true },
+    { key: "threadPort", label: "Thread:Port", width: 130 },
+    { key: "name", label: "Name", width: 160 },
+    { key: "data", label: "Data", width: 210 },
+    { key: "fn", label: "Function", width: 320, color: FUNCTION_COLOR },
+    { key: "pv", label: "Path Values", width: 250, color: PATHVALUE_COLOR },
+    { key: "qs", label: "Querysub", width: 120, color: QUERYSUB_COLOR },
+];
+
+type NodeRow = {
+    nodeId: string;
+    machineId: string;
+    ip: string;
+    machineShort: string;
+    threadPort: string;
+    name: string;
+    data: string;
+    fn: string;
+    pv: string;
+    qs: string;
+};
+
 type NodeAuthorityInfo = {
     nodeId: string;
     entryPoint?: string;
     spec?: AuthoritySpec;
 };
 
+// A machine's IP never changes, so we cache each hostname's resolved IP forever (keyed by hostname, shared across all
+// the machine's threads). DNS resolution only runs on the server node this controller executes on.
+const nodeIpCache = new Map<string, string>();
+
+async function resolveNodeIp(nodeId: string): Promise<string | undefined> {
+    let hostname = getNodeIdDomainMaybeUndefined(nodeId);
+    if (!hostname) return undefined;
+    let cached = nodeIpCache.get(hostname);
+    if (cached) return cached;
+    let result = await timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, dns.promises.lookup(hostname));
+    if (!result) return undefined;
+    nodeIpCache.set(hostname, result.address);
+    return result.address;
+}
+
 class RoutingTablePageControllerBase {
+    // Resolves every node's IP in parallel (cached). A machine's IP is then the most common IP across its threads.
+    public async getNodeIps(nodeIds: string[]): Promise<{ [nodeId: string]: string }> {
+        let entries = await Promise.all(nodeIds.map(async nodeId => [nodeId, await resolveNodeIp(nodeId)] as const));
+        let result: { [nodeId: string]: string } = {};
+        for (let [nodeId, ip] of entries) {
+            if (ip) result[nodeId] = ip;
+        }
+        return result;
+    }
     public async getAllNodeAuthoritySpecs(): Promise<NodeAuthorityInfo[]> {
         let nodes = await getAllNodeIds();
         return Promise.all(nodes.map(async (nodeId): Promise<NodeAuthorityInfo> => {
@@ -61,6 +117,7 @@ export const RoutingTablePageController = SocketFunction.register(
         getAllNodeAuthoritySpecs: {},
         getNodeLatencies: {},
         getNodeTrafficStats: {},
+        getNodeIps: {},
     }),
     () => ({
         hooks: [assertIsManagementUser],
@@ -149,69 +206,162 @@ class FunctionRunnersSection extends qreact.Component {
     }
 }
 
-function nodeLabelLines(config: {
+function nodeTotalTraffic(traffic: TrafficStats): { sent: number; received: number; outside: number; } {
+    let sent = traffic.outside.sent;
+    let received = traffic.outside.received;
+    for (let d of Object.values(traffic.perNode)) {
+        sent += d.sent;
+        received += d.received;
+    }
+    return { sent, received, outside: traffic.outside.sent + traffic.outside.received };
+}
+
+function machineIdOf(nodeId: string): string {
+    let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
+    return parts?.machineId || nodeId;
+}
+
+// Summed traffic metrics across a set of nodes (a machine's threads).
+function sumTraffic(threads: string[], trafficMaps: Map<string, TrafficStats>) {
+    let dataSent = 0;
+    let dataReceived = 0;
+    let valuesSent = 0;
+    let valuesReceived = 0;
+    let calls = 0;
+    let addCalls = 0;
+    for (let id of threads) {
+        let traffic = trafficMaps.get(id);
+        if (!traffic) continue;
+        let totals = nodeTotalTraffic(traffic);
+        dataSent += totals.sent;
+        dataReceived += totals.received;
+        valuesSent += traffic.pathValuesSent;
+        valuesReceived += traffic.pathValuesReceived;
+        calls += traffic.functionsExecuted;
+        addCalls += traffic.querysubCalls;
+    }
+    return { dataSent, dataReceived, valuesSent, valuesReceived, calls, addCalls };
+}
+
+// A machine's graph label: the IP first (most important), then the same per-piece info as the table, summed across all
+// its threads and colored the same.
+function machineLabelLines(config: {
+    machineId: string;
+    threads: string[];
+    trafficMaps: Map<string, TrafficStats>;
+    ip: string | undefined;
+}): LatencyGraphLabelLine[] {
+    let { machineId, threads, ip } = config;
+    let totals = sumTraffic(threads, config.trafficMaps);
+    let lines: LatencyGraphLabelLine[] = [];
+    if (ip) {
+        lines.push({ text: ip });
+    }
+    lines.push({ text: `${machineId.slice(0, ID_CHARS)}  ${threads.length} threads` });
+    if (totals.dataSent + totals.dataReceived > 0) {
+        lines.push({ text: `↑${formatNumber(totals.dataSent)}B ↓${formatNumber(totals.dataReceived)}B` });
+    }
+    if (totals.valuesSent || totals.valuesReceived) {
+        lines.push({ text: `↑${formatNumber(totals.valuesSent)} ↓${formatNumber(totals.valuesReceived)} values`, color: PATHVALUE_COLOR });
+    }
+    if (totals.calls) {
+        lines.push({ text: `${formatNumber(totals.calls)} calls`, color: FUNCTION_COLOR });
+    }
+    if (totals.addCalls) {
+        lines.push({ text: `${formatNumber(totals.addCalls)} addCalls`, color: QUERYSUB_COLOR });
+    }
+    return lines;
+}
+
+// One table row per thread node: each piece of info that used to live on the graph label becomes its own column.
+function buildNodeRow(config: {
     nodeId: string;
     runner: FunctionRunnerNodeInfo | undefined;
     info: NodeAuthorityInfo | undefined;
     traffic: TrafficStats | undefined;
-}): LatencyGraphLabelLine[] {
-    let { nodeId, runner, info, traffic } = config;
+    machineIp: Map<string, string>;
+}): NodeRow {
+    let { nodeId, runner, info, traffic, machineIp } = config;
     let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
     let thread = (parts?.threadId || "?").slice(0, ID_CHARS);
-    let machine = (parts?.machineId || nodeId).slice(0, ID_CHARS);
-    let lines: LatencyGraphLabelLine[] = [{ text: `${thread}:${parts?.port ?? "?"}` }];
-    // The last segment of the entry-point path is a good human name, and more useful than the machine id.
+    let machineId = parts?.machineId || nodeId;
     let entryPoint = info?.entryPoint || runner?.entryPoint;
-    let name = entryPoint ? entryPoint.split(/[\\/]/).filter(Boolean).at(-1) : undefined;
-    if (name) {
-        lines.push({ text: name });
-    }
-    lines.push({ text: machine });
-    // Total data this node sent / received, and the percent of it that went outside our known node list.
+    let name = entryPoint && entryPoint.split(/[\\/]/).filter(Boolean).at(-1) || "";
+
+    let data = "";
     if (traffic) {
         let totals = nodeTotalTraffic(traffic);
         let sum = totals.sent + totals.received;
         if (sum > 0) {
             let outsidePct = Math.round((totals.outside / sum) * 100);
-            lines.push({ text: `↑${formatNumber(totals.sent)}B ↓${formatNumber(totals.received)}B ${outsidePct}%⊘` });
+            data = `↑${formatNumber(totals.sent)}B ↓${formatNumber(totals.received)}B  ${outsidePct}%⊘`;
         }
     }
+
+    let fnParts: string[] = [];
     if (runner) {
-        // Network names are human-readable, so we show them in full.
-        for (let network of runner.networks) {
-            lines.push({ text: network, color: FUNCTION_COLOR });
-        }
+        if (runner.networks.length) fnParts.push(runner.networks.join(" "));
         for (let shard of runner.shards) {
-            lines.push({ text: `fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`, color: FUNCTION_COLOR });
-        }
-        lines.push({ text: `${formatNumber(traffic?.functionsExecuted ?? 0)} calls`, color: FUNCTION_COLOR });
-        if (!runner.isPublic) {
-            lines.push({ text: "PRIVATE" });
+            fnParts.push(`fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
         }
+        fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)} calls`);
+        if (!runner.isPublic) fnParts.push("PRIVATE");
     }
-    // routeStart/routeEnd of -1 means "no path-value sharding", so only show the range when it's real.
+
+    let pvParts: string[] = [];
     let spec = info?.spec;
     if (spec && spec.routeStart >= 0 && spec.routeEnd >= 0) {
-        lines.push({ text: `pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`, color: PATHVALUE_COLOR });
+        pvParts.push(`pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
     }
-    if (traffic?.pathValuesSent) {
-        lines.push({ text: `${formatNumber(traffic.pathValuesSent)} sent`, color: PATHVALUE_COLOR });
+    if (traffic && (traffic.pathValuesSent || traffic.pathValuesReceived)) {
+        pvParts.push(`↑${formatNumber(traffic.pathValuesSent)} ↓${formatNumber(traffic.pathValuesReceived)} values`);
     }
-    // Querysub activity: how many function calls were submitted to this node's QuerysubController.
-    if (traffic?.querysubCalls) {
-        lines.push({ text: `${formatNumber(traffic.querysubCalls)} addCalls`, color: QUERYSUB_COLOR });
-    }
-    return lines;
+
+    let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)} addCalls` : "";
+    return {
+        nodeId,
+        machineId,
+        ip: machineIp.get(machineId) || "",
+        machineShort: machineId.slice(0, ID_CHARS),
+        threadPort: `${thread}:${parts?.port ?? "?"}`,
+        name,
+        data,
+        fn: fnParts.join("  "),
+        pv: pvParts.join("  "),
+        qs,
+    };
 }
 
-function nodeTotalTraffic(traffic: TrafficStats): { sent: number; received: number; outside: number; } {
-    let sent = traffic.outside.sent;
-    let received = traffic.outside.received;
-    for (let d of Object.values(traffic.perNode)) {
-        sent += d.sent;
-        received += d.received;
+class NodeInfoTable extends qreact.Component<{ rows: NodeRow[]; selectedMachine: string }> {
+    render() {
+        let rows = this.props.rows;
+        let selected = this.props.selectedMachine;
+        return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
+            <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
+                {NODE_TABLE_COLUMNS.map(col =>
+                    <div className={css.width(col.width).flexShrink0.pad2(6).ellipsis}>{col.label}</div>
+                )}
+            </div>
+            {rows.map((row, i) => {
+                let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
+                let isSelected = row.machineId === selected;
+                return <div
+                    className={css.hbox(0).button.fillWidth.hsl(0, 0, isSelected ? 92 : 99)
+                        .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
+                    onClick={() => selectedMachineParam.value = selected === row.machineId ? "" : row.machineId}
+                >
+                    {NODE_TABLE_COLUMNS.map(col => {
+                        // Per-machine columns (IP, machine id) only print on the first row of each group, so it reads clearly.
+                        let text = col.perMachine && !firstOfMachine ? "" : row[col.key];
+                        return <div
+                            className={css.width(col.width).flexShrink0.pad2(6).ellipsis.color(col.color || "hsl(0, 0%, 25%)")}
+                            title={text}
+                        >{text}</div>;
+                    })}
+                </div>;
+            })}
+        </div>;
     }
-    return { sent, received, outside: traffic.outside.sent + traffic.outside.received };
 }
 
 class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: NodeAuthorityInfo[] }> {
@@ -235,11 +385,50 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             let traffic = synced.getNodeTrafficStats(nodeId);
             if (traffic) trafficMaps.set(nodeId, traffic);
         }
-        // Only render nodes that reported their latencies — ones that never respond probably don't exist.
+        // Only graph nodes that reported their latencies — ones that never respond probably don't exist.
         let respondedSet = new Set(latencyMaps.keys());
 
-        // Total bytes on each node-pair link (both directions). Both endpoints report the same link total, so we take
-        // the max instead of summing. Circle size = a node's total traffic.
+        // Group every thread by its machine; the graph shows one node per machine that has a responding thread.
+        let machineAllThreads = new Map<string, string[]>();
+        let machineRespondedThreads = new Map<string, string[]>();
+        let pushInto = (map: Map<string, string[]>, key: string, value: string) => {
+            let list = map.get(key);
+            if (!list) {
+                list = [];
+                map.set(key, list);
+            }
+            list.push(value);
+        };
+        for (let nodeId of nodeIds) {
+            let machineId = machineIdOf(nodeId);
+            pushInto(machineAllThreads, machineId, nodeId);
+            if (respondedSet.has(nodeId)) {
+                pushInto(machineRespondedThreads, machineId, nodeId);
+            }
+        }
+
+        // A machine's IP is the most common IP across its threads (they should all match, but resolve robustly).
+        let nodeIps = synced.getNodeIps(nodeIds);
+        let machineIp = new Map<string, string>();
+        for (let [machineId, threads] of machineAllThreads) {
+            let counts = new Map<string, number>();
+            for (let thread of threads) {
+                let ip = nodeIps?.[thread];
+                if (!ip) continue;
+                counts.set(ip, (counts.get(ip) || 0) + 1);
+            }
+            let bestIp = "";
+            let bestCount = 0;
+            for (let [ip, count] of counts) {
+                if (count > bestCount) {
+                    bestCount = count;
+                    bestIp = ip;
+                }
+            }
+            if (bestIp) machineIp.set(machineId, bestIp);
+        }
+
+        // Total bytes on each thread-pair link (both directions). Both endpoints report the same total, so take the max.
         let pairKey = (a: string, b: string) => a < b ? `${a}|${b}` : `${b}|${a}`;
         let pairTraffic = new Map<string, number>();
         for (let [reporter, traffic] of trafficMaps) {
@@ -250,28 +439,69 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             }
         }
 
-        let nodes: LatencyGraphNode[] = [...respondedSet].map(nodeId => {
-            let traffic = trafficMaps.get(nodeId);
-            let totals = traffic ? nodeTotalTraffic(traffic) : undefined;
+        // Machine-to-machine latency = mean over all cross-machine thread-pair latencies; traffic = summed pair traffic.
+        let machinePairLatency = new Map<string, { sum: number; count: number; }>();
+        for (let [sourceId, latencies] of latencyMaps) {
+            let sourceMachine = machineIdOf(sourceId);
+            for (let [destId, latencyMs] of Object.entries(latencies)) {
+                if (!respondedSet.has(destId) || !Number.isFinite(latencyMs)) continue;
+                let destMachine = machineIdOf(destId);
+                if (sourceMachine === destMachine) continue;
+                let key = pairKey(sourceMachine, destMachine);
+                let agg = machinePairLatency.get(key);
+                if (!agg) {
+                    agg = { sum: 0, count: 0 };
+                    machinePairLatency.set(key, agg);
+                }
+                agg.sum += latencyMs;
+                agg.count++;
+            }
+        }
+        let machinePairTraffic = new Map<string, number>();
+        for (let [key, weight] of pairTraffic) {
+            let [a, b] = key.split("|");
+            let ma = machineIdOf(a);
+            let mb = machineIdOf(b);
+            if (ma === mb) continue;
+            let mk = pairKey(ma, mb);
+            machinePairTraffic.set(mk, (machinePairTraffic.get(mk) || 0) + weight);
+        }
+
+        let nodes: LatencyGraphNode[] = [...machineRespondedThreads.keys()].map(machineId => {
+            let allThreads = machineAllThreads.get(machineId) || [];
+            let totals = sumTraffic(allThreads, trafficMaps);
             return {
-                id: nodeId,
-                labelLines: nodeLabelLines({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic }),
-                weight: totals ? totals.sent + totals.received : 0,
+                id: machineId,
+                labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId) }),
+                weight: totals.dataSent + totals.dataReceived,
             };
         });
         let links: LatencyGraphLink[] = [];
-        for (let [sourceId, latencies] of latencyMaps) {
-            for (let [destId, latencyMs] of Object.entries(latencies)) {
-                if (!respondedSet.has(destId)) continue;
-                links.push({ source: sourceId, destination: destId, latencyMs, weight: pairTraffic.get(pairKey(sourceId, destId)) || 0 });
-            }
+        for (let [key, agg] of machinePairLatency) {
+            let [source, destination] = key.split("|");
+            links.push({ source, destination, latencyMs: agg.sum / agg.count, weight: machinePairTraffic.get(key) || 0 });
         }
 
+        // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
+        // Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
+        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp }));
+        let selected = selectedMachineParam.value;
+        let sep = String.fromCharCode(1);
+        sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
+
         return <div className={css.vbox(8).fillWidth}>
-            <h2 className={css.margin(0)}>Latency Graph ({respondedSet.size}/{nodeIds.length} nodes reported)</h2>
+            <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
             <div className={css.relative.fillWidth.height(LATENCY_GRAPH_HEIGHT_PX).bord2(0, 0, 85)}>
-                <LatencyGraph nodes={nodes} links={links} formatWeight={weight => formatNumber(weight) + "B"} />
+                <LatencyGraph
+                    nodes={nodes}
+                    links={links}
+                    formatWeight={weight => formatNumber(weight) + "B"}
+                    selectedId={selected || undefined}
+                    onSelectNode={id => selectedMachineParam.value = selectedMachineParam.value === id ? "" : id}
+                />
             </div>
+            <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
+            <NodeInfoTable rows={rows} selectedMachine={selected} />
         </div>;
     }
 }