querysub 0.525.0 → 0.527.0

package/src/0-path-value-core/PathRouter.ts

@@ -5,7 +5,7 @@ import { PathValue } from "./pathValueCore";
 import { shuffle } from "../misc/random";
 import { fastHash } from "../misc/hash";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
-import { getNodeLatency } from "../-f-node-discovery/LatencyTracking";
+import { getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
 import { unique } from "../misc";
 import { measureFnc } from "socket-function/src/profiling/measure";
 import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
@@ -22,15 +22,18 @@ export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
 // fine (two nearby servers just talk to each other). Replaces the old uniform-random pick between candidates.
 const LATENCY_CANDIDATE_LIMIT = 5;
 const LATENCY_WEIGHT_OFFSET = 10;
-// A candidate whose weight is below (the current max weight / this factor) is dropped to zero probability. So a node
-// weighted 10x another (10x lower effective latency) is picked over it 100% of the time.
-const LATENCY_WEIGHT_CUTOFF_FACTOR = 10;
+// How many latency samples we use per node (median of the last N). The more we actually got, the more we trust the number, and the more aggressively we cut slow candidates.
+const LATENCY_HISTORY_COUNT = 10;
+// A candidate whose weight is below (the current max weight / (this - its historyUsed)) is dropped to zero probability. So with full history the factor is 2 (2x the effective latency of the best is never picked), and with a single sample it is 11 (only extreme outliers are cut).
+const LATENCY_CUTOFF_BASE_FACTOR = 12;
 // Latency assumed for a node we haven't measured yet, so new/unreached nodes stay eligible but aren't preferred.
 const UNKNOWN_LATENCY_MS = 50;
 
-function getRoutingLatency(nodeId: string): number {
-    if (isOwnNodeId(nodeId)) return 0;
-    return getNodeLatency(nodeId) ?? UNKNOWN_LATENCY_MS;
+function getRoutingLatency(nodeId: string): { latency: number; historyUsed: number } {
+    if (isOwnNodeId(nodeId)) return { latency: 0, historyUsed: LATENCY_HISTORY_COUNT };
+    let median = getNodeLatencyMedian({ nodeId, historyCount: LATENCY_HISTORY_COUNT });
+    if (!median) return { latency: UNKNOWN_LATENCY_MS, historyUsed: 0 };
+    return median;
 }
 
 // In-place reorder of candidates by a latency-weighted random draw over the lowest LATENCY_CANDIDATE_LIMIT
@@ -39,16 +42,22 @@ function getRoutingLatency(nodeId: string): number {
 //       since an overloaded server's latency spikes, which already makes it less likely to be chosen here.
 function latencyWeightedShuffle<T extends { nodeId: string }>(arr: T[]) {
     if (arr.length <= 1) return;
-    let withLatency = arr.map(x => ({ x, latency: getRoutingLatency(x.nodeId) }));
+    let withLatency = arr.map(x => {
+        let routingLatency = getRoutingLatency(x.nodeId);
+        return { x, latency: routingLatency.latency, historyUsed: routingLatency.historyUsed };
+    });
     sort(withLatency, e => e.latency);
     let pool = withLatency.slice(0, LATENCY_CANDIDATE_LIMIT);
     let rest = withLatency.slice(LATENCY_CANDIDATE_LIMIT);
     let ordered: T[] = [];
     while (pool.length > 0) {
         let weights = pool.map(e => 1 / (e.latency + LATENCY_WEIGHT_OFFSET));
-        let cutoff = Math.max(...weights) / LATENCY_WEIGHT_CUTOFF_FACTOR;
+        let maxWeight = Math.max(...weights);
         for (let i = 0; i < weights.length; i++) {
-            if (weights[i] < cutoff) weights[i] = 0;
+            let cutoffFactor = LATENCY_CUTOFF_BASE_FACTOR - Math.min(pool[i].historyUsed, LATENCY_HISTORY_COUNT);
+            if (weights[i] < maxWeight / cutoffFactor) {
+                weights[i] = 0;
+            }
         }
         let total = 0;
         for (let w of weights) {
@@ -90,6 +99,7 @@ export type AuthoritySpec = {
     routeEnd: number;
     // If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
     //  - For now, let's just never add overlapping prefixes.
+    //  - Sorted by originalPrefix (authorityLookup.setOurSpec sorts before publishing), so specs can be compared element-wise.
     prefixes: PrefixMatcher[];
     //  - Make sure to set this if you just want the prefix values, otherwise you will get all that that prefixes don't match (the prefixes exclude prefix match but where route does not match).
     excludeDefault?: boolean;
@@ -162,6 +172,18 @@ export function parsePrefixMatcher(prefixPath: string): PrefixMatcher {
     };
 }
 
+// Full-overlap equivalence: same range, same default matching, and the same prefixes (prefixes are sorted, so element-wise comparison works).
+export function areAuthoritySpecsEquivalent(a: AuthoritySpec, b: AuthoritySpec): boolean {
+    if (a.routeStart !== b.routeStart) return false;
+    if (a.routeEnd !== b.routeEnd) return false;
+    if (!a.excludeDefault !== !b.excludeDefault) return false;
+    if (a.prefixes.length !== b.prefixes.length) return false;
+    for (let [i, prefix] of a.prefixes.entries()) {
+        if (prefix.originalPrefix !== b.prefixes[i].originalPrefix) return false;
+    }
+    return true;
+}
+
 export function debugSpec(spec: AuthoritySpec) {
     return {
         info: `${spec.routeStart}-${spec.routeEnd} (${spec.prefixes.length} prefixes${spec.excludeDefault ? " excluding default" : ""})`,

package/src/1-path-client/RemoteWatcher.ts

@@ -6,7 +6,9 @@ import { isNode, sort, timeInMinute, timeInSecond } from "socket-function/src/mi
 import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
 import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
 import { PathValueController } from "../0-path-value-core/PathValueController";
-import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
+import { AuthoritySpec, PathRouter, areAuthoritySpecsEquivalent } from "../0-path-value-core/PathRouter";
+import { getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
+import { formatTime } from "socket-function/src/formatting/format";
 import { WatchConfig, authorityStorage } from "../0-path-value-core/pathValueCore";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { ActionsHistory } from "../diagnostics/ActionsHistory";
@@ -29,9 +31,15 @@ setImmediate(() => import("../4-querysub/Querysub"));
 const STOP_KEYS_DOUBLE_SENDS = true;
 
 // How close to an authority's scheduled shutdown we start treating it like a disconnect (rehoming all the paths we watch on it).
-const SHUTDOWN_REHOME_WINDOW = timeInMinute * 5;
+const SHUTDOWN_REHOME_WINDOW = timeInMinute * 3;
 const SHUTDOWN_REHOME_POLL_INTERVAL = timeInSecond * 30;
 
+// Rehome the watches on an authority when an equivalent authority has this factor lower latency. At 2 the router's full-confidence cutoff factor (also 2) then reliably routes everything to the better node on the rewatch.
+const LATENCY_REHOME_FACTOR = 2;
+// Both latencies must be medians over this many samples — with fewer we don't trust either number enough to move watches over it.
+const LATENCY_REHOME_HISTORY_COUNT = 10;
+const LATENCY_REHOME_POLL_INTERVAL = timeInMinute;
+
 // NOTE: If a parent watch is broken up between multiple nodes, we generally watch everything on all those nodes and then filter when we receive the data. This isn't efficient, and we should probably change it. However, in practice, it's probably fine, as it's unlikely for the watches to be broken up to a much finer granularity than the network (it would require very strange partially overlapping sharding cases which no reasonable sharding setup would ever satisfy).
 export class RemoteWatcher {
     public static DEBUG = false;
@@ -152,6 +160,40 @@ export class RemoteWatcher {
             }
         }
     }
+
+    private latencyRehomeLoop = runInfinitePoll(LATENCY_REHOME_POLL_INTERVAL, () => this.rehomeHighLatencyAuthorities());
+    private rehomeHighLatencyAuthorities() {
+        let authorityIds = new Set<string>(this.remoteWatchPaths.values());
+        for (let watchObj of this.remoteWatchParents2.values()) {
+            for (let range of watchObj.ranges) {
+                authorityIds.add(range.authorityId);
+            }
+        }
+        // Also guards getTopologySync — if we watch anything remotely, the authority lookup has synced.
+        if (authorityIds.size === 0) return;
+        let topology = authorityLookup.getTopologySync();
+        for (let authorityId of authorityIds) {
+            if (isOwnNodeId(authorityId)) continue;
+            let entry = topology.find(candidate => candidate.nodeId === authorityId);
+            if (!entry) continue;
+            let current = getNodeLatencyMedian({ nodeId: authorityId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
+            if (!current || current.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
+            for (let candidate of topology) {
+                if (candidate.nodeId === authorityId) continue;
+                if (isOwnNodeId(candidate.nodeId)) continue;
+                if (!candidate.isReady) continue;
+                if (authorityLookup.nodeIsShuttingDownSoon(candidate.nodeId)) continue;
+                if (!areAuthoritySpecsEquivalent(entry.authoritySpec, candidate.authoritySpec)) continue;
+                let candidateLatency = getNodeLatencyMedian({ nodeId: candidate.nodeId, historyCount: LATENCY_REHOME_HISTORY_COUNT });
+                if (!candidateLatency || candidateLatency.historyUsed < LATENCY_REHOME_HISTORY_COUNT) continue;
+                if (candidateLatency.latency * LATENCY_REHOME_FACTOR > current.latency) continue;
+                console.log(yellow(`Authority ${authorityId} has high latency (${formatTime(current.latency)}), and the equivalent authority ${candidate.nodeId} is much faster (${formatTime(candidateLatency.latency)}), so we are rehoming all paths watched on it`));
+                logErrors(this.refreshAllWatches(authorityId));
+                break;
+            }
+        }
+    }
+
     private async tryToReconnectNow() {
         if (!this.disconnectedPaths.size && !this.disconnectedParents.size) return;
 
@@ -709,6 +751,8 @@ export class RemoteWatcher {
             }
             let paths = Array.from(pathsToWatch);
 
+            console.log(yellow(`Refreshing all watches on ${authorityNodeId}: ${pathsToWatch.size} paths, ${parentPathsToRewatch.size} parent paths (${parentRemotePathsToUnwatch.size} remote parent ranges)`));
+
             logErrors(RemoteWatcher.REMOTE_UNWATCH_FUNCTION({
                 paths,
                 parentPaths: Array.from(parentRemotePathsToUnwatch)

package/src/4-querysub/FunctionRunnerTracking.ts

@@ -7,13 +7,13 @@
 import { SocketFunction } from "socket-function/SocketFunction";
 import { timeInMinute, timeInSecond, sort } from "socket-function/src/misc";
 import { lazy } from "socket-function/src/caching";
-import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isClient } from "../config2";
 import { t } from "../2-proxy/schema2";
 import { createLocalSchema } from "./schemaHelpers";
 import { Querysub } from "./Querysub";
 import { getAllNodeIds, getOwnNodeId, watchDeltaNodeIds } from "../-f-node-discovery/NodeDiscovery";
-import { getNodeLatencyInfo } from "../-f-node-discovery/LatencyTracking";
+import { getNodeLatencyInfo, getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
 import { FunctionRunnerInfoController, FunctionStatsSummary } from "../3-path-functions/PathFunctionRunner";
 import { URLParam } from "../library-components/URLParam";
 import { logErrors, timeoutToUndefined, timeoutToUndefinedSilent } from "../errors";
@@ -37,6 +37,12 @@ const CALL_TIME_WEIGHT_BASE = 1000;
 // If every runner on a network will be shut down within this window, the network is dying and we switch off of it.
 const NETWORK_DYING_WINDOW = timeInMinute * 2;
 
+// Once selected we stick with a network — only switching when another network's best node has this factor lower latency than our network's best node (comparing medians, both with full history, so we never switch on noisy data).
+const NETWORK_SWITCH_LATENCY_FACTOR = 2;
+const NETWORK_SWITCH_HISTORY_COUNT = 10;
+// The latency data is already collected by LatencyTracking, so checking is cheap and can run often.
+const NETWORK_SWITCH_POLL_INTERVAL = timeInMinute;
+
 // The forced network is a URL parameter, so it survives reloads and is shareable / obvious in the URL.
 export const forcedNetworkURL = new URLParam("network", "");
 
@@ -306,11 +312,75 @@ export function getAutoSelectedNetwork(): string | undefined {
     return candidates[0]?.network;
 }
 
+// The lowest full-history median latency among the network's runners (the best node is what we compare networks by).
+function getNetworkBestNodeLatency(network: string): number | undefined {
+    let index = getFunctionRunnerIndex();
+    if (!index) return undefined;
+    let best: number | undefined;
+    for (let node of index.nodes) {
+        if (!node.networks.includes(network)) continue;
+        let median = getNodeLatencyMedian({ nodeId: node.nodeId, historyCount: NETWORK_SWITCH_HISTORY_COUNT });
+        if (!median || median.historyUsed < NETWORK_SWITCH_HISTORY_COUNT) continue;
+        if (best === undefined || median.latency < best) {
+            best = median.latency;
+        }
+    }
+    return best;
+}
+
+let stickyNetwork: string | undefined;
+function checkNetworkSwitch() {
+    if (!stickyNetwork) return;
+    let currentLatency = getNetworkBestNodeLatency(stickyNetwork);
+    if (currentLatency === undefined) return;
+
+    let candidates = getNetworkSelectionInfos().filter(x => !x.dying && x.network !== stickyNetwork);
+    if (isPublic()) {
+        candidates = candidates.filter(x => x.isPublic);
+    }
+    let upLongEnough = candidates.filter(x => x.upLongEnough);
+    if (upLongEnough.length > 0) {
+        candidates = upLongEnough;
+    }
+
+    let bestNetwork: string | undefined;
+    let bestLatency: number | undefined;
+    for (let candidate of candidates) {
+        let latency = getNetworkBestNodeLatency(candidate.network);
+        if (latency === undefined) continue;
+        if (bestLatency === undefined || latency < bestLatency) {
+            bestLatency = latency;
+            bestNetwork = candidate.network;
+        }
+    }
+    if (bestNetwork === undefined || bestLatency === undefined) return;
+    if (bestLatency * NETWORK_SWITCH_LATENCY_FACTOR > currentLatency) return;
+
+    console.log(yellow(`Switching the selected function network from ${stickyNetwork} (best node latency ${formatTime(currentLatency)}) to ${bestNetwork} (best node latency ${formatTime(bestLatency)}), so all new function calls use the faster network`));
+    stickyNetwork = bestNetwork;
+}
+const startNetworkSwitchPoll = lazy(() => {
+    logErrors(runInfinitePoll(NETWORK_SWITCH_POLL_INTERVAL, checkNetworkSwitch));
+});
+
+// Sticky: the score-based pick only initializes (or replaces a dead/dying selection); after that only checkNetworkSwitch changes it, so calls don't bounce between networks on small score changes.
+function getStickyNetwork(): string | undefined {
+    void startNetworkSwitchPoll();
+    if (stickyNetwork) {
+        let info = getNetworkSelectionInfos().find(x => x.network === stickyNetwork);
+        if (info && !info.dying && (!isPublic() || info.isPublic)) {
+            return stickyNetwork;
+        }
+    }
+    stickyNetwork = getAutoSelectedNetwork();
+    return stickyNetwork;
+}
+
 /** Safe to call from any context. Returns the network new function calls should be put on. Throws if no network is available (calls without a network can never run), unless noThrow is set. */
 export function getSelectedNetwork(): string;
 export function getSelectedNetwork(config: { noThrow: boolean }): string | undefined;
 export function getSelectedNetwork(config?: { noThrow?: boolean }): string | undefined {
-    let network = forcedNetworkURL.value || getAutoSelectedNetwork();
+    let network = forcedNetworkURL.value || getStickyNetwork();
     if (!network && !config?.noThrow) {
         throw new Error(`No function runner discovered during discovery, so this call cannot be given a network and would never run (we keep polling for function runners, every ${formatTime(POLL_INTERVAL)} when tracking, every ${formatTime(INDEX_POLL_INTERVAL)} for the client index)`);
     }

package/src/config.ts

@@ -35,6 +35,7 @@ let yargObj = parseArgsFactory()
     .option("slowdown", { type: "number", desc: "Delay all input data values by this amount of time, pretending like we didn't even receive it until this time is up." })
     .option("network", { type: "array", desc: `The networks this node is on (pass multiple arguments to be on multiple, ex: --network test --network default). Authorities only satisfy paths on their networks ("default" if unset). Function calls are put on the first network in the list. If no FunctionRunner is on a call's network, the call will fail to run.` })
     .option("networkfile", { type: "string", desc: `The same as --network, except the networks are read from the given file (one per line). Supports "~/" for the home directory. If the file doesn't exist, the process exits with an error.` })
+    .option("port", { type: "number", desc: "The storage port for `yarn storageserve`" })
     .argv
     ;
 
@@ -51,6 +52,10 @@ export function expandHomePath(path: string): string {
     return path;
 }
 
+export function getPort(): number | undefined {
+    return yargObj.port;
+}
+
 let networkFileNetworks = lazy((): string[] => {
     if (!yargObj.networkfile) return [];
     let path = expandHomePath(String(yargObj.networkfile));

package/src/diagnostics/managementPages.tsx

@@ -128,12 +128,6 @@ export async function registerManagementPages2(config: {
         controllerName: "SnapshotViewerController",
         getModule: () => import("./misc-pages/SnapshotViewer"),
     });
-    inputPages.push({
-        title: "DNS",
-        componentName: "DNSPage",
-        controllerName: "DNSPageController",
-        getModule: () => import("./misc-pages/DNSPage"),
-    });
     inputPages.push({
         title: "Fnc Capture",
         componentName: "FunctionCapturePage",

package/src/diagnostics/misc-pages/RoutingTablePage.tsx

@@ -36,7 +36,7 @@ const MACHINE_LATENCY_WIDTH_PX = 235;
 
 // Clicking a machine node in the graph (or a row) sorts that machine's nodes to the top of the table.
 const selectedMachineParam = new URLParam("rtSelectedMachine", "", { reset: [mainResets] });
-const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 60, { reset: [mainResets] });
+const trafficWindowParam = new URLParam<number>("rtTrafficWindow", timeInMinute * 5, { reset: [mainResets] });
 
 // The per-node table columns, in order. `color` tints the function/path-value/querysub columns to match the graph;
 // `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
@@ -433,6 +433,35 @@ class NodeInfoTable extends qreact.Component<{
     }
 }
 
+// Nodes that never reported their latencies — all we have is their nodeId (and possibly a DNS-resolved machine IP), so this is a much narrower table than NodeInfoTable, still grouped by machine.
+class UnresponsiveNodesTable extends qreact.Component<{ nodeIds: string[]; machineIp: Map<string, string> }> {
+    render() {
+        let rows = this.props.nodeIds.map(nodeId => ({ nodeId, machineId: machineIdOf(nodeId) }));
+        let sep = String.fromCharCode(1);
+        sort(rows, row => `${row.machineId}${sep}${threadLabel(row.nodeId)}`);
+        return <div className={css.vbox(0).fillWidth.overflowAuto.bord2(0, 0, 85)}>
+            <div className={css.hbox(0).hsl(0, 0, 96).colorhsl(0, 0, 20).boldStyle}>
+                <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>IP</div>
+                <div className={css.width(96).flexShrink0.pad2(6).ellipsis}>Machine</div>
+                <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>Thread:Port</div>
+                <div className={css.width(520).flexShrink0.pad2(6).ellipsis}>Node Id</div>
+            </div>
+            {rows.map((row, i) => {
+                let firstOfMachine = i === 0 || rows[i - 1].machineId !== row.machineId;
+                return <div
+                    className={css.hbox(0).fillWidth.hsl(0, 0, 99)
+                        .borderTop(firstOfMachine ? "1px solid hsl(0, 0%, 80%)" : "1px solid hsl(0, 0%, 93%)")}
+                >
+                    <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>{firstOfMachine && this.props.machineIp.get(row.machineId) || ""}</div>
+                    <div className={css.width(96).flexShrink0.pad2(6).ellipsis}>{firstOfMachine && row.machineId.slice(0, ID_CHARS) || ""}</div>
+                    <div className={css.width(130).flexShrink0.pad2(6).ellipsis}>{threadLabel(row.nodeId)}</div>
+                    <div className={css.width(520).flexShrink0.pad2(6).ellipsis.colorhsl(0, 0, 45)} title={row.nodeId}>{row.nodeId}</div>
+                </div>;
+            })}
+        </div>;
+    }
+}
+
 class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: NodeAuthorityInfo[] }> {
     render() {
         let nodeIds = this.props.nodeIds;
@@ -609,7 +638,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             return {
                 id: machineId,
                 labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId), networks: machineNetworks.get(machineId) || [] }),
-                weight: totals.dataSent + totals.dataReceived,
+                weight: totals.valuesSent + totals.valuesReceived,
             };
         });
         let links: LatencyGraphLink[] = [];
@@ -620,16 +649,18 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             if (pv && (pv.fwd || pv.back)) {
                 extraLabel = { text: `↑${formatNumber(pv.fwd)}/s ↓${formatNumber(pv.back)}/s values`, color: PATHVALUE_COLOR };
             }
-            links.push({ source, destination, latencyMs, weight: machinePairTraffic.get(key) || 0, extraLabel });
+            links.push({ source, destination, latencyMs, weight: machinePairTraffic.get(key) || 0, valueWeight: pv && pv.fwd + pv.back || 0, extraLabel });
         }
 
         // One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
         // Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
-        let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
+        let rows = nodeIds.filter(nodeId => respondedSet.has(nodeId)).map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
         let selected = selectedMachineParam.value;
         let sep = String.fromCharCode(1);
         sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
 
+        let unresponsiveNodeIds = nodeIds.filter(nodeId => !respondedSet.has(nodeId));
+
         return <div className={css.vbox(8).fillWidth}>
             <div className={css.hbox(14)}>
                 <h2 className={css.margin(0)}>Latency Graph ({machineRespondedThreads.size} machines · {respondedSet.size}/{nodeIds.length} nodes reported)</h2>
@@ -656,6 +687,10 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
             <div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
             <h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
             <NodeInfoTable rows={rows} selectedMachine={selected} machineColumns={machineColumns} nodeMachineLatency={nodeMachineLatency} />
+            {unresponsiveNodeIds.length > 0 && <>
+                <h2 className={css.margin(0)}>Unresponsive Nodes ({unresponsiveNodeIds.length})</h2>
+                <UnresponsiveNodesTable nodeIds={unresponsiveNodeIds} machineIp={machineIp} />
+            </>}
         </div>;
     }
 }

package/src/diagnostics/periodic.ts

@@ -78,7 +78,8 @@ if (isNode()) {
         }
 
         if (data.toString().includes("\r")) {
-            logAll();
+            import("./watchdog").then(m => m.logUnfiltered());
+            //logAll();
         }
     });
 

package/src/diagnostics/watchdog.ts

@@ -116,6 +116,17 @@ function logProfileMeasuresTimingsNow() {
         thresholdInTable: 0
     });
 };
+export function logUnfiltered(depth = 2) {
+    let profile = measureObj.finish();
+    measureObj = startMeasure();
+    logMeasureTable(profile, {
+        name: `all logs at ${new Date().toLocaleString()}`,
+        mergeDepth: depth,
+        minTimeToLog: 0,
+        maxTableEntries: 10000000,
+        thresholdInTable: 0
+    });
+};
 
 
 registerPeriodic(logProfileMeasuresTimingsNow);

package/src/library-components/EdgeNodeSelector.tsx

@@ -1,7 +1,7 @@
 import { css } from "typesafecss";
 import { qreact } from "../4-dom/qreact";
 import { Querysub } from "../4-querysub/Querysub";
-import { timeInSecond } from "socket-function/src/misc";
+import { sort, timeInSecond } from "socket-function/src/misc";
 import { formatTime } from "socket-function/src/formatting/format";
 import { isCurrentUserSuperUser } from "../user-implementation/userData";
 import type { EdgeNodeConfig, EdgeNodeStat } from "../4-deploy/edgeNodes";
@@ -43,6 +43,7 @@ export class EdgeNodeSelector extends qreact.Component<{}> {
             nodes = [...nodes, { host: booted.host, nodeId: booted.nodeId, public: booted.public, live: true, finished: false }];
         }
         let autoStat = nodes.find(x => x.host === stats?.autoPickedHost);
+        sort(nodes, x => x.latency);
 
         return <div className={css.hbox(6)}>
             <span className={css.opacity(0.7)}>Edge</span>

package/src/library-components/LatencyGraph.tsx

@@ -9,7 +9,7 @@ import { formatTime } from "socket-function/src/formatting/format";
 
 export type LatencyGraphLabelLine = { text: string; color?: string; };
 export type LatencyGraphNode = { id: string; label?: string; latitude?: number; longitude?: number; labelLines?: LatencyGraphLabelLine[]; weight?: number; };
-export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; extraLabel?: LatencyGraphLabelLine; };
+export type LatencyGraphLink = { source: string; destination: string; latencyMs: number; weight?: number; valueWeight?: number; extraLabel?: LatencyGraphLabelLine; };
 export type LatencyGraphProps = {
     nodes: LatencyGraphNode[];
     links: LatencyGraphLink[];
@@ -59,7 +59,7 @@ const HIGHLIGHT_COLOR = "hsl(40, 90%, 60%)";
 const NODE_WEIGHT_MULT = 2.5;
 const LINE_BASE_WIDTH = 1.5;
 const LINE_MIN_WIDTH = 1;
-const LINE_MAX_WIDTH = 7;
+const LINE_MAX_WIDTH = 35;
 const LABEL_LINE_HEIGHT = 13;
 const ZOOM_STEP = 1.1;
 const MIN_SCALE = 0.05;
@@ -84,6 +84,8 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
     maxNodeWeight = 0;
     pairWeight = new Map<number, number>();
     maxPairWeight = 0;
+    pairValueWeight = new Map<number, number>();
+    maxPairValueWeight = 0;
     pairExtraLabel = new Map<number, LatencyGraphLabelLine>();
     // Cached from props in render() so draw() (a rAF callback, not synced) never reads this.props.
     formatWeight: ((weight: number) => string) | undefined = undefined;
@@ -156,15 +158,23 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         }
         this.pairWeight = new Map();
         this.maxPairWeight = 0;
+        this.pairValueWeight = new Map();
+        this.maxPairValueWeight = 0;
         for (let link of this.props.links) {
-            if (!link.weight) continue;
             let a = index.get(link.source);
             let b = index.get(link.destination);
             if (a === undefined || b === undefined || a === b) continue;
             let pk = Math.min(a, b) * n + Math.max(a, b);
-            let combined = (this.pairWeight.get(pk) || 0) + link.weight;
-            this.pairWeight.set(pk, combined);
-            this.maxPairWeight = Math.max(this.maxPairWeight, combined);
+            if (link.weight) {
+                let combined = (this.pairWeight.get(pk) || 0) + link.weight;
+                this.pairWeight.set(pk, combined);
+                this.maxPairWeight = Math.max(this.maxPairWeight, combined);
+            }
+            if (link.valueWeight) {
+                let combined = (this.pairValueWeight.get(pk) || 0) + link.valueWeight;
+                this.pairValueWeight.set(pk, combined);
+                this.maxPairValueWeight = Math.max(this.maxPairValueWeight, combined);
+            }
         }
     }
 
@@ -554,10 +564,19 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
         this.drawHover(ctx, toScreenX, toScreenY);
     }
 
+    // Opacity encodes the values sent back and forth on the connection (normalized to the busiest pair). Latency-based opacity is only the fallback when no link has value data.
+    edgeOpacity(a: number, b: number, latency: number) {
+        if (this.maxPairValueWeight <= 0) {
+            return this.opacityFor(latency, this.renderMin, this.renderMax);
+        }
+        let valueWeight = this.pairValueWeight.get(Math.min(a, b) * this.nodes.length + Math.max(a, b)) || 0;
+        return Math.max(MIN_OPACITY, valueWeight / this.maxPairValueWeight);
+    }
+
     drawEdges(ctx: CanvasRenderingContext2D, toScreenX: (wx: number) => number, toScreenY: (wy: number) => number) {
         for (let edge of this.renderEdges) {
             ctx.lineWidth = this.lineWidthFor(edge.a, edge.b);
-            ctx.strokeStyle = `hsla(${NODE_HUE}, 70%, 62%, ${this.opacityFor(edge.latency, this.renderMin, this.renderMax)})`;
+            ctx.strokeStyle = `hsla(${NODE_HUE}, 70%, 62%, ${this.edgeOpacity(edge.a, edge.b, edge.latency)})`;
             ctx.beginPath();
             ctx.moveTo(toScreenX(this.positionsX[edge.a]), toScreenY(this.positionsY[edge.a]));
             ctx.lineTo(toScreenX(this.positionsX[edge.b]), toScreenY(this.positionsY[edge.b]));
@@ -836,7 +855,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
             weightSig += node.weight || 0;
         }
         for (let link of this.props.links) {
-            weightSig += link.weight || 0;
+            weightSig += (link.weight || 0) + (link.valueWeight || 0);
         }
         if (sig !== this.builtSig || geoParam.value !== this.builtGeo) {
             // Switching layout mode (geographic vs solved) changes the whole coordinate space, so re-fit the view.

package/src/server.ts

@@ -1,3 +1,10 @@
+import { disableMeasurements } from "socket-function/src/profiling/measure";
+// NOTE: Profiling seems to make us use about twice as much memory, at least in the bad case where we have many tiny watchers.
+if (typeof document === "undefined" && (process.argv.includes("--noprofile") || process.argv.includes("--nprofile"))) {
+    disableMeasurements();
+}
+
+
 import "./forceProduction";
 import "./inject";