querysub 0.520.0 → 0.522.0

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -8,6 +8,8 @@ import { LatencyController } from "../../-f-node-discovery/LatencyTracking";
 import { TrafficController, TrafficStats } from "../../-f-node-discovery/TrafficTracking";
 import { getDomain } from "../../config";
 import { decodeNodeId } from "sliftutils/misc/https/certs";
+import { getNodeIdDomainMaybeUndefined } from "socket-function/src/nodeCache";
+import dns from "dns";
 import { getSyncedController } from "../../library-components/SyncedController";
 import { assertIsManagementUser } from "../managementPages";
 import { NodeCapabilitiesController } from "../../-g-core-values/NodeCapabilities";
@@ -17,6 +19,7 @@ import type { AuthoritySpec } from "../../0-path-value-core/PathRouter";
 import { getFunctionRunnerIndex, FunctionRunnerNodeInfo } from "../../4-querysub/FunctionRunnerTracking";
 import { formatTime, formatNumber } from "socket-function/src/formatting/format";
 import { LatencyGraph, LatencyGraphNode, LatencyGraphLink, LatencyGraphLabelLine } from "../../library-components/LatencyGraph";
+import { URLParam } from "../../library-components/URLParam";
 
 const ID_CHARS = 8;
 // Green means querysub, blue means path value, purple means function runner.
@@ -29,13 +32,68 @@ const RANGE_BAR_WIDTH_PX = 360;
 const RANGE_BAR_HEIGHT_PX = 14;
 const LATENCY_GRAPH_HEIGHT_PX = 720;
 
+// Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
+const selectedMachineParam = new URLParam("rtSelectedMachine", "");
+
+// The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
+// `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
+const NODE_TABLE_COLUMNS: { key: keyof NodeRow; label: string; width: number; color?: string; perMachine?: boolean; }[] = [
+    { key: "ip", label: "IP", width: 130, perMachine: true },
+    { key: "networks", label: "Networks", width: 180, color: QUERYSUB_COLOR, perMachine: true },
+    { key: "machineShort", label: "Machine", width: 96, perMachine: true },
+    { key: "threadPort", label: "Thread:Port", width: 130 },
+    { key: "name", label: "Name", width: 160 },
+    { key: "data", label: "Data", width: 210 },
+    { key: "fn", label: "Function", width: 320, color: FUNCTION_COLOR },
+    { key: "pv", label: "Path Values", width: 250, color: PATHVALUE_COLOR },
+    { key: "qs", label: "Querysub", width: 120, color: QUERYSUB_COLOR },
+];
+
+type NodeRow = {
+    nodeId: string;
+    machineId: string;
+    ip: string;
+    networks: string;
+    machineShort: string;
+    threadPort: string;
+    name: string;
+    data: string;
+    fn: string;
+    pv: string;
+    qs: string;
+};
+
 type NodeAuthorityInfo = {
     nodeId: string;
     entryPoint?: string;
     spec?: AuthoritySpec;
 };
 
+// A machine's IP never changes, so we cache each hostname's resolved IP forever (keyed by hostname, shared across all
+// the machine's threads). DNS resolution only runs on the server node this controller executes on.
+const nodeIpCache = new Map<string, string>();
+
+async function resolveNodeIp(nodeId: string): Promise<string | undefined> {
+    let hostname = getNodeIdDomainMaybeUndefined(nodeId);
+    if (!hostname) return undefined;
+    let cached = nodeIpCache.get(hostname);
+    if (cached) return cached;
+    let result = await timeoutToUndefinedSilent(PROBE_TIMEOUT_MS, dns.promises.lookup(hostname));
+    if (!result) return undefined;
+    nodeIpCache.set(hostname, result.address);
+    return result.address;
+}
+
 class RoutingTablePageControllerBase {
+    // Resolves every node's IP in parallel (cached). A machine's IP is then the most common IP across its threads.
+    public async getNodeIps(nodeIds: string[]): Promise<{ [nodeId: string]: string }> {
+        let entries = await Promise.all(nodeIds.map(async nodeId => [nodeId, await resolveNodeIp(nodeId)] as const));
+        let result: { [nodeId: string]: string } = {};
+        for (let [nodeId, ip] of entries) {
+            if (ip) result[nodeId] = ip;
+        }
+        return result;
+    }
     public async getAllNodeAuthoritySpecs(): Promise<NodeAuthorityInfo[]> {
         let nodes = await getAllNodeIds();
         return Promise.all(nodes.map(async (nodeId): Promise<NodeAuthorityInfo> => {
@@ -61,6 +119,7 @@ export const RoutingTablePageController = SocketFunction.register(
         getAllNodeAuthoritySpecs: {},
         getNodeLatencies: {},
         getNodeTrafficStats: {},
+        getNodeIps: {},
     }),
     () => ({
         hooks: [assertIsManagementUser],
@@ -149,68 +208,170 @@ class FunctionRunnersSection extends qreact.Component {
     }
 }
 
-function nodeLabelLines(config: {
+function nodeTotalTraffic(traffic: TrafficStats): { sent: number; received: number; outside: number; } {
+    let outsideSent = traffic.outside?.sent || 0;
+    let outsideReceived = traffic.outside?.received || 0;
+    let sent = outsideSent;
+    let received = outsideReceived;
+    for (let d of Object.values(traffic.perNode || {})) {
+        sent += d?.sent || 0;
+        received += d?.received || 0;
+    }
+    return { sent, received, outside: outsideSent + outsideReceived };
+}
+
+function machineIdOf(nodeId: string): string {
+    let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
+    return parts?.machineId || nodeId;
+}
+
+// Summed traffic metrics across a set of nodes (a machine's threads).
+function sumTraffic(threads: string[], trafficMaps: Map<string, TrafficStats>) {
+    let dataSent = 0;
+    let dataReceived = 0;
+    let valuesSent = 0;
+    let valuesReceived = 0;
+    let calls = 0;
+    let addCalls = 0;
+    for (let id of threads) {
+        let traffic = trafficMaps.get(id);
+        if (!traffic) continue;
+        let totals = nodeTotalTraffic(traffic);
+        dataSent += totals.sent;
+        dataReceived += totals.received;
+        valuesSent += traffic.pathValuesSent || 0;
+        valuesReceived += traffic.pathValuesReceived || 0;
+        calls += traffic.functionsExecuted || 0;
+        addCalls += traffic.querysubCalls || 0;
+    }
+    return { dataSent, dataReceived, valuesSent, valuesReceived, calls, addCalls };
+}
+
+// A machine's graph label: the IP first (most important), then its function-runner networks, then the same per-piece
+// info as the table, summed across all its threads and colored the same.
+function machineLabelLines(config: {
+    machineId: string;
+    threads: string[];
+    trafficMaps: Map<string, TrafficStats>;
+    ip: string | undefined;
+    networks: string[];
+}): LatencyGraphLabelLine[] {
+    let { machineId, threads, ip, networks } = config;
+    let totals = sumTraffic(threads, config.trafficMaps);
+    let lines: LatencyGraphLabelLine[] = [];
+    if (ip) {
+        lines.push({ text: ip });
+    }
+    if (networks.length) {
+        lines.push({ text: networks.join(" | "), color: QUERYSUB_COLOR });
+    }
+    lines.push({ text: `${machineId.slice(0, ID_CHARS)}  ${threads.length} threads` });
+    if (totals.dataSent + totals.dataReceived > 0) {
+        lines.push({ text: `↑${formatNumber(totals.dataSent)}B/s ↓${formatNumber(totals.dataReceived)}B/s` });
+    }
+    if (totals.valuesSent || totals.valuesReceived) {
+        lines.push({ text: `↑${formatNumber(totals.valuesSent)}/s ↓${formatNumber(totals.valuesReceived)}/s values`, color: PATHVALUE_COLOR });
+    }
+    if (totals.calls) {
+        lines.push({ text: `${formatNumber(totals.calls)}/s calls`, color: FUNCTION_COLOR });
+    }
+    if (totals.addCalls) {
+        lines.push({ text: `${formatNumber(totals.addCalls)}/s addCalls`, color: QUERYSUB_COLOR });
+    }
+    return lines;
+}
+
+// One table row per thread node: each piece of info that used to live on the graph label becomes its own column.
+function buildNodeRow(config: {
     nodeId: string;
     runner: FunctionRunnerNodeInfo | undefined;
     info: NodeAuthorityInfo | undefined;
     traffic: TrafficStats | undefined;
-}): LatencyGraphLabelLine[] {
-    let { nodeId, runner, info, traffic } = config;
+    machineIp: Map<string, string>;
+    machineNetworks: Map<string, string[]>;
+}): NodeRow {
+    let { nodeId, runner, info, traffic, machineIp, machineNetworks } = config;
     let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
     let thread = (parts?.threadId || "?").slice(0, ID_CHARS);
-    let machine = (parts?.machineId || nodeId).slice(0, ID_CHARS);
-    let lines: LatencyGraphLabelLine[] = [{ text: `${thread}:${parts?.port ?? "?"}` }];
-    // The last segment of the entry-point path is a good human name; shown on one line with the server (machine) id.
+    let machineId = parts?.machineId || nodeId;
     let entryPoint = info?.entryPoint || runner?.entryPoint;
-    let name = entryPoint ? entryPoint.split(/[\\/]/).filter(Boolean).at(-1) : undefined;
-    lines.push({ text: name ? `${name}  ${machine}` : machine });
-    // Total data this node sent / received, and the percent of it that went outside our known node list.
+    let name = entryPoint && entryPoint.split(/[\\/]/).filter(Boolean).at(-1) || "";
+
+    let data = "";
     if (traffic) {
         let totals = nodeTotalTraffic(traffic);
         let sum = totals.sent + totals.received;
         if (sum > 0) {
             let outsidePct = Math.round((totals.outside / sum) * 100);
-            lines.push({ text: `↑${formatNumber(totals.sent)}B ↓${formatNumber(totals.received)}B ${outsidePct}%⊘` });
+            data = `↑${formatNumber(totals.sent)}B/s ↓${formatNumber(totals.received)}B/s  ${outsidePct}%⊘`;
         }
     }
-    // All function-runner info on one purple line: networks, shard ranges, calls, private.
+
+    let fnParts: string[] = [];
     if (runner) {
-        let fnParts: string[] = [];
         if (runner.networks.length) fnParts.push(runner.networks.join(" "));
         for (let shard of runner.shards) {
             fnParts.push(`fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
         }
-        fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)} calls`);
+        fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)}/s calls`);
         if (!runner.isPublic) fnParts.push("PRIVATE");
-        lines.push({ text: fnParts.join("  "), color: FUNCTION_COLOR });
     }
-    // All path-value info on one blue line: shard range (only if real) and values sent.
-    let spec = info?.spec;
+
     let pvParts: string[] = [];
+    let spec = info?.spec;
     if (spec && spec.routeStart >= 0 && spec.routeEnd >= 0) {
         pvParts.push(`pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
     }
     if (traffic && (traffic.pathValuesSent || traffic.pathValuesReceived)) {
-        pvParts.push(`↑${formatNumber(traffic.pathValuesSent)} ↓${formatNumber(traffic.pathValuesReceived)} values`);
-    }
-    if (pvParts.length) {
-        lines.push({ text: pvParts.join("  "), color: PATHVALUE_COLOR });
+        pvParts.push(`↑${formatNumber(traffic.pathValuesSent)}/s ↓${formatNumber(traffic.pathValuesReceived)}/s values`);
     }
-    // Querysub activity: how many function calls were submitted to this node's QuerysubController.
-    if (traffic?.querysubCalls) {
-        lines.push({ text: `${formatNumber(traffic.querysubCalls)} addCalls`, color: QUERYSUB_COLOR });
-    }
-    return lines;
+
+    let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)}/s addCalls` : "";
+    return {
+        nodeId,
+        machineId,
+        ip: machineIp.get(machineId) || "",
+        networks: (machineNetworks.get(machineId) || []).join(" | "),
+        machineShort: machineId.slice(0, ID_CHARS),
+        threadPort: `${thread}:${parts?.port ?? "?"}`,
+        name,
+        data,
+        fn: fnParts.join("  "),
+        pv: pvParts.join("  "),
+        qs,
+    };
 }
 
-function nodeTotalTraffic(traffic: TrafficStats): { sent: number; received: number; outside: number; } {
-    let sent = traffic.outside.sent;
-    let received = traffic.outside.received;
-    for (let d of Object.values(traffic.perNode)) {
-        sent += d.sent;
-        received += d.received;
+class NodeInfoTable extends qreact.Component<{ rows: NodeRow[]; selectedMachine: string }> {
+    render() {
+        let rows = this.props.rows;
+        let selected = this.props.selectedMachine;
+        return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
+            <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
+                {NODE_TABLE_COLUMNS.map(col =>
+                    <div className={css.width(col.width).flexShrink0.pad2(6).ellipsis}>{col.label}</div>
+                )}
+            </div>
+            {rows.map((row, i) => {
+                let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
+                let isSelected = row.machineId === selected;
+                return <div
+                    className={css.hbox(0).button.fillWidth.hsl(0, 0, isSelected ? 92 : 99)
+                        .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
+                    onClick={() => selectedMachineParam.value = selected === row.machineId ? "" : row.machineId}
+                >
+                    {NODE_TABLE_COLUMNS.map(col => {
+                        // Per-machine columns (IP, machine id) only print on the first row of each group, so it reads clearly.
+                        let text = col.perMachine && !firstOfMachine ? "" : row[col.key];
+                        return <div
+                            className={css.width(col.width).flexShrink0.pad2(6).ellipsis.color(col.color || "hsl(0, 0%, 25%)")}
+                            title={text}
+                        >{text}</div>;
+                    })}
+                </div>;
+            })}
+        </div>;
     }
-    return { sent, received, outside: traffic.outside.sent + traffic.outside.received };
 }
 
 class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: NodeAuthorityInfo[] }> {
@@ -234,70 +395,136 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             let traffic = synced.getNodeTrafficStats(nodeId);
             if (traffic) trafficMaps.set(nodeId, traffic);
         }
-        // Only render nodes that reported their latencies — ones that never respond probably don't exist.
+        // Only graph nodes that reported their latencies — ones that never respond probably don't exist.
         let respondedSet = new Set(latencyMaps.keys());
 
-        // Total bytes on each node-pair link (both directions). Both endpoints report the same link total, so we take
-        // the max instead of summing. Circle size = a node's total traffic.
+        // Group every thread by its machine; the graph shows one node per machine that has a responding thread.
+        let machineAllThreads = new Map<string, string[]>();
+        let machineRespondedThreads = new Map<string, string[]>();
+        let pushInto = (map: Map<string, string[]>, key: string, value: string) => {
+            let list = map.get(key);
+            if (!list) {
+                list = [];
+                map.set(key, list);
+            }
+            list.push(value);
+        };
+        for (let nodeId of nodeIds) {
+            let machineId = machineIdOf(nodeId);
+            pushInto(machineAllThreads, machineId, nodeId);
+            if (respondedSet.has(nodeId)) {
+                pushInto(machineRespondedThreads, machineId, nodeId);
+            }
+        }
+
+        // A machine's IP is the most common IP across its threads (they should all match, but resolve robustly).
+        let nodeIps = synced.getNodeIps(nodeIds);
+        let machineIp = new Map<string, string>();
+        for (let [machineId, threads] of machineAllThreads) {
+            let counts = new Map<string, number>();
+            for (let thread of threads) {
+                let ip = nodeIps?.[thread];
+                if (!ip) continue;
+                counts.set(ip, (counts.get(ip) || 0) + 1);
+            }
+            let bestIp = "";
+            let bestCount = 0;
+            for (let [ip, count] of counts) {
+                if (count > bestCount) {
+                    bestCount = count;
+                    bestIp = ip;
+                }
+            }
+            if (bestIp) machineIp.set(machineId, bestIp);
+        }
+
+        // The unique set of function-runner networks running on each machine (usually one, human-readable).
+        let machineNetworks = new Map<string, string[]>();
+        for (let [machineId, threads] of machineAllThreads) {
+            let unique = new Set<string>();
+            for (let thread of threads) {
+                for (let network of runnerByNode.get(thread)?.networks || []) {
+                    unique.add(network);
+                }
+            }
+            if (unique.size) machineNetworks.set(machineId, [...unique]);
+        }
+
+        // Total bytes on each thread-pair link (both directions). Both endpoints report the same total, so take the max.
         let pairKey = (a: string, b: string) => a < b ? `${a}|${b}` : `${b}|${a}`;
         let pairTraffic = new Map<string, number>();
         for (let [reporter, traffic] of trafficMaps) {
-            for (let [peer, data] of Object.entries(traffic.perNode)) {
+            for (let [peer, data] of Object.entries(traffic.perNode || {})) {
                 if (reporter === peer || !respondedSet.has(peer)) continue;
                 let key = pairKey(reporter, peer);
-                pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, data.sent + data.received));
+                pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, (data?.sent || 0) + (data?.received || 0)));
             }
         }
 
-        let nodes: LatencyGraphNode[] = [...respondedSet].map(nodeId => {
-            let traffic = trafficMaps.get(nodeId);
-            let totals = traffic ? nodeTotalTraffic(traffic) : undefined;
+        // Machine-to-machine latency = mean over all cross-machine thread-pair latencies; traffic = summed pair traffic.
+        let machinePairLatency = new Map<string, { sum: number; count: number; }>();
+        for (let [sourceId, latencies] of latencyMaps) {
+            let sourceMachine = machineIdOf(sourceId);
+            for (let [destId, latencyMs] of Object.entries(latencies)) {
+                if (!respondedSet.has(destId) || !Number.isFinite(latencyMs)) continue;
+                let destMachine = machineIdOf(destId);
+                if (sourceMachine === destMachine) continue;
+                let key = pairKey(sourceMachine, destMachine);
+                let agg = machinePairLatency.get(key);
+                if (!agg) {
+                    agg = { sum: 0, count: 0 };
+                    machinePairLatency.set(key, agg);
+                }
+                agg.sum += latencyMs;
+                agg.count++;
+            }
+        }
+        let machinePairTraffic = new Map<string, number>();
+        for (let [key, weight] of pairTraffic) {
+            let [a, b] = key.split("|");
+            let ma = machineIdOf(a);
+            let mb = machineIdOf(b);
+            if (ma === mb) continue;
+            let mk = pairKey(ma, mb);
+            machinePairTraffic.set(mk, (machinePairTraffic.get(mk) || 0) + weight);
+        }
+
+        let nodes: LatencyGraphNode[] = [...machineRespondedThreads.keys()].map(machineId => {
+            let allThreads = machineAllThreads.get(machineId) || [];
+            let totals = sumTraffic(allThreads, trafficMaps);
             return {
-                id: nodeId,
-                labelLines: nodeLabelLines({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic }),
-                weight: totals ? totals.sent + totals.received : 0,
+                id: machineId,
+                labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId), networks: machineNetworks.get(machineId) || [] }),
+                weight: totals.dataSent + totals.dataReceived,
             };
         });
         let links: LatencyGraphLink[] = [];
-        for (let [sourceId, latencies] of latencyMaps) {
-            for (let [destId, latencyMs] of Object.entries(latencies)) {
-                if (!respondedSet.has(destId)) continue;
-                links.push({ source: sourceId, destination: destId, latencyMs, weight: pairTraffic.get(pairKey(sourceId, destId)) || 0 });
-            }
+        for (let [key, agg] of machinePairLatency) {
+            let [source, destination] = key.split("|");
+            links.push({ source, destination, latencyMs: agg.sum / agg.count, weight: machinePairTraffic.get(key) || 0 });
         }
 
-        // Per-cluster summary: sum the same per-node metrics across the cluster's members.
-        let clusterSummary = (memberIds: string[]): LatencyGraphLabelLine[] => {
-            let sent = 0;
-            let received = 0;
-            let calls = 0;
-            let addCalls = 0;
-            let dataSent = 0;
-            let dataReceived = 0;
-            for (let id of memberIds) {
-                let traffic = trafficMaps.get(id);
-                if (!traffic) continue;
-                sent += traffic.pathValuesSent;
-                received += traffic.pathValuesReceived;
-                calls += traffic.functionsExecuted;
-                addCalls += traffic.querysubCalls;
-                let totals = nodeTotalTraffic(traffic);
-                dataSent += totals.sent;
-                dataReceived += totals.received;
-            }
-            let summary: LatencyGraphLabelLine[] = [];
-            summary.push({ text: `↑${formatNumber(dataSent)}B ↓${formatNumber(dataReceived)}B` });
-            summary.push({ text: `↑${formatNumber(sent)} ↓${formatNumber(received)} values`, color: PATHVALUE_COLOR });
-            summary.push({ text: `${formatNumber(calls)} calls`, color: FUNCTION_COLOR });
-            summary.push({ text: `${formatNumber(addCalls)} addCalls`, color: QUERYSUB_COLOR });
-            return summary;
-        };
+        // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
+        // Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
+        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
+        let selected = selectedMachineParam.value;
+        let sep = String.fromCharCode(1);
+        sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
 
         return <div className={css.vbox(8).fillWidth}>
-            <h2 className={css.margin(0)}>Latency Graph ({respondedSet.size}/{nodeIds.length} nodes reported)</h2>
+            <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
             <div className={css.relative.fillWidth.height(LATENCY_GRAPH_HEIGHT_PX).bord2(0, 0, 85)}>
-                <LatencyGraph nodes={nodes} links={links} formatWeight={weight => formatNumber(weight) + "B"} clusterSummary={clusterSummary} />
+                <LatencyGraph
+                    nodes={nodes}
+                    links={links}
+                    formatWeight={weight => formatNumber(weight) + "B/s"}
+                    selectedId={selected || undefined}
+                    onSelectNode={id => selectedMachineParam.value = selectedMachineParam.value === id ? "" : id}
+                />
             </div>
+            <div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
+            <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
+            <NodeInfoTable rows={rows} selectedMachine={selected} />
         </div>;
     }
 }