querysub 0.521.0 → 0.522.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/function-public.js +0 -2
- package/package.json +1 -1
- package/src/-f-node-discovery/TrafficTracking.ts +49 -28
- package/src/3-path-functions/PathFunctionRunner.ts +6 -5
- package/src/3-path-functions/PathFunctionRunnerMain.ts +4 -7
- package/src/config.ts +1 -3
- package/src/diagnostics/misc-pages/RoutingTablePage.tsx +50 -27
- package/src/library-components/LatencyGraph.tsx +20 -26
package/bin/function-public.js
CHANGED
package/package.json
CHANGED
|
@@ -2,9 +2,13 @@
|
|
|
2
2
|
Per-server, in-memory traffic counters (never written to the database). Everything is just an integer increment
|
|
3
3
|
into a 5-minute bucket; we keep the last 60 minutes (12 buckets) and drop older ones, so memory is bounded.
|
|
4
4
|
|
|
5
|
+
Reads return PER-SECOND RATES, not totals. Each bucket also records the timestamp it first received data, so the
|
|
6
|
+
rate window runs from the oldest retained bucket's first data up to now — which stays accurate right after startup
|
|
7
|
+
(when only a partial window of data exists) instead of dividing a small burst by the full 60-minute window.
|
|
8
|
+
|
|
5
9
|
Byte traffic comes from SocketFunction.trackMessageSizes (upload/download per connection). The connection nodeId is
|
|
6
10
|
sometimes a raw client-connection id, so we map it to the nice node id (debugNodeId) lazily at read time — the hot
|
|
7
|
-
path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node
|
|
11
|
+
path is just a Map increment keyed by the raw id. At read we split traffic into per-known-node rates and a single
|
|
8
12
|
"outside the network" aggregate for anything not in our cached node list (clients, etc.).
|
|
9
13
|
*/
|
|
10
14
|
|
|
@@ -17,21 +21,28 @@ import { debugNodeId } from "../-c-identity/IdentityController";
|
|
|
17
21
|
|
|
18
22
|
const BUCKET_MS = timeInMinute * 5;
|
|
19
23
|
const BUCKET_COUNT = 12;
|
|
24
|
+
// Floor on the rate window so a burst in the first fraction of a second after startup can't divide out to an absurd rate.
|
|
25
|
+
const MIN_WINDOW_SECONDS = 1;
|
|
20
26
|
// Sentinel key for traffic that isn't a network node (client connections, etc.). Aggregated so memory stays O(nodes),
|
|
21
27
|
// not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
|
|
22
28
|
const OUTSIDE_KEY = "";
|
|
23
29
|
|
|
24
30
|
export type NodeDataTraffic = { sent: number; received: number; };
|
|
31
|
+
// All numbers are PER-SECOND rates (see file header).
|
|
25
32
|
export type TrafficStats = {
|
|
26
33
|
functionsExecuted: number;
|
|
27
34
|
pathValuesSent: number;
|
|
28
35
|
pathValuesReceived: number;
|
|
29
36
|
querysubCalls: number;
|
|
30
|
-
//
|
|
37
|
+
// Byte rates to/from each known network node, plus one aggregate for everything outside the node list.
|
|
31
38
|
perNode: { [nodeId: string]: NodeDataTraffic };
|
|
32
39
|
outside: NodeDataTraffic;
|
|
33
40
|
};
|
|
34
41
|
|
|
42
|
+
// count = amount accumulated in the bucket; firstMs = when the bucket first received data (for the rate window).
|
|
43
|
+
type Bucket = { count: number; firstMs: number; };
|
|
44
|
+
type BucketMap = Map<number, Bucket>;
|
|
45
|
+
|
|
35
46
|
function currentBucket() {
|
|
36
47
|
return Math.floor(Date.now() / BUCKET_MS);
|
|
37
48
|
}
|
|
@@ -39,20 +50,24 @@ function oldestBucket() {
|
|
|
39
50
|
return currentBucket() - BUCKET_COUNT + 1;
|
|
40
51
|
}
|
|
41
52
|
|
|
42
|
-
|
|
43
|
-
let
|
|
44
|
-
let
|
|
45
|
-
let
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
let
|
|
49
|
-
let bytesReceived = new Map<string, Map<number, number>>();
|
|
53
|
+
let functionsExecuted: BucketMap = new Map();
|
|
54
|
+
let pathValuesSent: BucketMap = new Map();
|
|
55
|
+
let pathValuesReceived: BucketMap = new Map();
|
|
56
|
+
let querysubCalls: BucketMap = new Map();
|
|
57
|
+
// raw connection nodeId => buckets
|
|
58
|
+
let bytesSent = new Map<string, BucketMap>();
|
|
59
|
+
let bytesReceived = new Map<string, BucketMap>();
|
|
50
60
|
|
|
51
|
-
function addSimple(buckets:
|
|
61
|
+
function addSimple(buckets: BucketMap, amount: number) {
|
|
52
62
|
let bucket = currentBucket();
|
|
53
|
-
|
|
63
|
+
let entry = buckets.get(bucket);
|
|
64
|
+
if (!entry) {
|
|
65
|
+
entry = { count: 0, firstMs: Date.now() };
|
|
66
|
+
buckets.set(bucket, entry);
|
|
67
|
+
}
|
|
68
|
+
entry.count += amount;
|
|
54
69
|
}
|
|
55
|
-
function addKeyed(map: Map<string,
|
|
70
|
+
function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
|
|
56
71
|
let buckets = map.get(key);
|
|
57
72
|
if (!buckets) {
|
|
58
73
|
buckets = new Map();
|
|
@@ -84,17 +99,23 @@ function trafficKey(nodeId: string): string {
|
|
|
84
99
|
SocketFunction.trackMessageSizes.upload.push((size, nodeId) => addKeyed(bytesSent, trafficKey(nodeId), size));
|
|
85
100
|
SocketFunction.trackMessageSizes.download.push((size, nodeId) => addKeyed(bytesReceived, trafficKey(nodeId), size));
|
|
86
101
|
|
|
87
|
-
|
|
102
|
+
// Per-second rate over the retained window: total accumulated / (now - oldest retained bucket's first-data time).
|
|
103
|
+
// Also prunes expired buckets as it goes.
|
|
104
|
+
function bucketRate(buckets: BucketMap): number {
|
|
88
105
|
let oldest = oldestBucket();
|
|
89
106
|
let total = 0;
|
|
90
|
-
|
|
107
|
+
let firstMs = Infinity;
|
|
108
|
+
for (let [bucket, entry] of Array.from(buckets)) {
|
|
91
109
|
if (bucket < oldest) {
|
|
92
110
|
buckets.delete(bucket);
|
|
93
111
|
continue;
|
|
94
112
|
}
|
|
95
|
-
total += count;
|
|
113
|
+
total += entry.count;
|
|
114
|
+
if (entry.firstMs < firstMs) firstMs = entry.firstMs;
|
|
96
115
|
}
|
|
97
|
-
return
|
|
116
|
+
if (total <= 0 || !Number.isFinite(firstMs)) return 0;
|
|
117
|
+
let windowSeconds = Math.max(MIN_WINDOW_SECONDS, (Date.now() - firstMs) / 1000);
|
|
118
|
+
return total / windowSeconds;
|
|
98
119
|
}
|
|
99
120
|
|
|
100
121
|
function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
|
|
@@ -103,10 +124,10 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
|
|
|
103
124
|
let outside: NodeDataTraffic = { sent: 0, received: 0 };
|
|
104
125
|
// At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
|
|
105
126
|
// the rare server-style id that isn't already a known node.
|
|
106
|
-
let attribute = (rawNodeId: string, field: "sent" | "received",
|
|
107
|
-
if (
|
|
127
|
+
let attribute = (rawNodeId: string, field: "sent" | "received", rate: number) => {
|
|
128
|
+
if (rate <= 0) return;
|
|
108
129
|
if (rawNodeId === OUTSIDE_KEY) {
|
|
109
|
-
outside[field] +=
|
|
130
|
+
outside[field] += rate;
|
|
110
131
|
return;
|
|
111
132
|
}
|
|
112
133
|
let nice = known.has(rawNodeId) ? rawNodeId : debugNodeId(rawNodeId);
|
|
@@ -116,17 +137,17 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
|
|
|
116
137
|
entry = { sent: 0, received: 0 };
|
|
117
138
|
perNode[nice] = entry;
|
|
118
139
|
}
|
|
119
|
-
entry[field] +=
|
|
140
|
+
entry[field] += rate;
|
|
120
141
|
} else {
|
|
121
|
-
outside[field] +=
|
|
142
|
+
outside[field] += rate;
|
|
122
143
|
}
|
|
123
144
|
};
|
|
124
145
|
for (let [rawNodeId, buckets] of Array.from(bytesSent)) {
|
|
125
|
-
attribute(rawNodeId, "sent",
|
|
146
|
+
attribute(rawNodeId, "sent", bucketRate(buckets));
|
|
126
147
|
if (buckets.size === 0) bytesSent.delete(rawNodeId);
|
|
127
148
|
}
|
|
128
149
|
for (let [rawNodeId, buckets] of Array.from(bytesReceived)) {
|
|
129
|
-
attribute(rawNodeId, "received",
|
|
150
|
+
attribute(rawNodeId, "received", bucketRate(buckets));
|
|
130
151
|
if (buckets.size === 0) bytesReceived.delete(rawNodeId);
|
|
131
152
|
}
|
|
132
153
|
return { perNode, outside };
|
|
@@ -135,10 +156,10 @@ function aggregateData(): { perNode: { [nodeId: string]: NodeDataTraffic }; outs
|
|
|
135
156
|
export function getTrafficStats(): TrafficStats {
|
|
136
157
|
let data = aggregateData();
|
|
137
158
|
return {
|
|
138
|
-
functionsExecuted:
|
|
139
|
-
pathValuesSent:
|
|
140
|
-
pathValuesReceived:
|
|
141
|
-
querysubCalls:
|
|
159
|
+
functionsExecuted: bucketRate(functionsExecuted),
|
|
160
|
+
pathValuesSent: bucketRate(pathValuesSent),
|
|
161
|
+
pathValuesReceived: bucketRate(pathValuesReceived),
|
|
162
|
+
querysubCalls: bucketRate(querysubCalls),
|
|
142
163
|
perNode: data.perNode,
|
|
143
164
|
outside: data.outside,
|
|
144
165
|
};
|
|
@@ -21,7 +21,7 @@ import { getOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
|
|
|
21
21
|
import { getScheduledShutdownTime } from "../-g-core-values/scheduledShutdown";
|
|
22
22
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
23
23
|
import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
|
|
24
|
-
import { getDomain, isPublic
|
|
24
|
+
import { getDomain, isPublic } from "../config";
|
|
25
25
|
import { getGitRefSync, getGitURLSync } from "../4-deploy/git";
|
|
26
26
|
import type { DeployProgress } from "../4-deploy/deployFunctions";
|
|
27
27
|
import { getRoutingOverride, getRoutingOverridePart } from "../0-path-value-core/PathRouterRouteOverride";
|
|
@@ -363,14 +363,15 @@ export class PathFunctionRunner {
|
|
|
363
363
|
secondaryShardRange?: { startFraction: number, endFraction: number };
|
|
364
364
|
PermissionsChecker: PermissionsCheckType | undefined;
|
|
365
365
|
// The networks we listen to calls on. Unset is equivalent to ["default"].
|
|
366
|
-
networks
|
|
366
|
+
networks: string[];
|
|
367
367
|
}) {
|
|
368
|
+
if (!config.networks.length) throw new Error(`networks is required`);
|
|
368
369
|
PathFunctionRunner.hostControllers();
|
|
369
370
|
debugFunctionRunnerShards.push({
|
|
370
371
|
domainName: config.domainName,
|
|
371
372
|
shardRange: config.shardRange,
|
|
372
373
|
secondaryShardRange: config.secondaryShardRange,
|
|
373
|
-
networks: config.networks
|
|
374
|
+
networks: config.networks,
|
|
374
375
|
isPublic: isPublic(),
|
|
375
376
|
});
|
|
376
377
|
logErrors(this.startWatching());
|
|
@@ -401,7 +402,7 @@ export class PathFunctionRunner {
|
|
|
401
402
|
|
|
402
403
|
let outstandingCalls = 0;
|
|
403
404
|
|
|
404
|
-
let networks = this.config.networks
|
|
405
|
+
let networks = this.config.networks;
|
|
405
406
|
|
|
406
407
|
let watchModuleCalls = cache((moduleId: string) => {
|
|
407
408
|
for (let network of networks) {
|
|
@@ -691,7 +692,7 @@ export class PathFunctionRunner {
|
|
|
691
692
|
}
|
|
692
693
|
|
|
693
694
|
let callNetwork = callSpec.network;
|
|
694
|
-
let ourNetworks = this.config.networks
|
|
695
|
+
let ourNetworks = this.config.networks;
|
|
695
696
|
if (!ourNetworks.includes(callNetwork)) {
|
|
696
697
|
return;
|
|
697
698
|
}
|
|
@@ -15,7 +15,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
|
|
|
15
15
|
import { getThreadKeyCert } from "sliftutils/misc/https/certs";
|
|
16
16
|
import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
|
|
17
17
|
import { timeInMinute } from "socket-function/src/misc";
|
|
18
|
-
import { getDomain, getNetworks, isLocal, isPublic
|
|
18
|
+
import { getDomain, getNetworks, isLocal, isPublic } from "../config";
|
|
19
19
|
import { green, magenta } from "socket-function/src/formatting/logColors";
|
|
20
20
|
import path from "path";
|
|
21
21
|
import { IndexedLogs } from "../diagnostics/logs/IndexedLogs/IndexedLogs";
|
|
@@ -32,6 +32,9 @@ async function main() {
|
|
|
32
32
|
// ClientWatcher.DEBUG_TRIGGERS = "heavy";
|
|
33
33
|
// authorityStorage.DEBUG_UNWATCH = true;
|
|
34
34
|
|
|
35
|
+
let networks = getNetworks();
|
|
36
|
+
if (!networks.length) throw new Error(`No networks found. Use --network to specify the networks this function runner should listen on.`);
|
|
37
|
+
|
|
35
38
|
PathFunctionRunner.DEBUG_CALLS = true;
|
|
36
39
|
// debugCoreMode();
|
|
37
40
|
|
|
@@ -60,8 +63,6 @@ async function main() {
|
|
|
60
63
|
console.log(green(`Sharding from ${shardStart} to ${shardEnd}`));
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
let networks = getNetworks();
|
|
64
|
-
|
|
65
66
|
new PathFunctionRunner({
|
|
66
67
|
domainName: getDomain(),
|
|
67
68
|
shardRange: { startFraction: shardStart, endFraction: shardEnd },
|
|
@@ -76,9 +77,5 @@ async function main() {
|
|
|
76
77
|
let deployPath = path.resolve("./deploy.ts");
|
|
77
78
|
await import(deployPath);
|
|
78
79
|
}
|
|
79
|
-
|
|
80
|
-
if (networks.length !== 1 || networks[0] !== DEFAULT_NETWORK) {
|
|
81
|
-
console.log(magenta(`Only running functions on the network(s): ${networks.join(", ")}. Use --network ${networks[0]} in your http server to route calls here.`));
|
|
82
|
-
}
|
|
83
80
|
}
|
|
84
81
|
logErrors(main());
|
package/src/config.ts
CHANGED
|
@@ -44,8 +44,6 @@ if (isNode()) {
|
|
|
44
44
|
}
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
-
export const DEFAULT_NETWORK = "default";
|
|
48
|
-
|
|
49
47
|
export function expandHomePath(path: string): string {
|
|
50
48
|
if (path === "~" || path.startsWith("~/") || path.startsWith("~\\")) {
|
|
51
49
|
return os.homedir() + path.slice(1);
|
|
@@ -76,7 +74,7 @@ export function getNetworks(): string[] {
|
|
|
76
74
|
result = [...result, ...networkFileNetworks()];
|
|
77
75
|
}
|
|
78
76
|
result = Array.from(new Set(result));
|
|
79
|
-
if (result.length === 0) return [
|
|
77
|
+
if (result.length === 0) return [];
|
|
80
78
|
return result;
|
|
81
79
|
}
|
|
82
80
|
|
|
@@ -39,6 +39,7 @@ const selectedMachineParam = new URLParam("rtSelectedMachine", "");
|
|
|
39
39
|
// `perMachine` columns (the IP and machine id) only print on the first row of each machine group.
|
|
40
40
|
const NODE_TABLE_COLUMNS: { key: keyof NodeRow; label: string; width: number; color?: string; perMachine?: boolean; }[] = [
|
|
41
41
|
{ key: "ip", label: "IP", width: 130, perMachine: true },
|
|
42
|
+
{ key: "networks", label: "Networks", width: 180, color: QUERYSUB_COLOR, perMachine: true },
|
|
42
43
|
{ key: "machineShort", label: "Machine", width: 96, perMachine: true },
|
|
43
44
|
{ key: "threadPort", label: "Thread:Port", width: 130 },
|
|
44
45
|
{ key: "name", label: "Name", width: 160 },
|
|
@@ -52,6 +53,7 @@ type NodeRow = {
|
|
|
52
53
|
nodeId: string;
|
|
53
54
|
machineId: string;
|
|
54
55
|
ip: string;
|
|
56
|
+
networks: string;
|
|
55
57
|
machineShort: string;
|
|
56
58
|
threadPort: string;
|
|
57
59
|
name: string;
|
|
@@ -207,13 +209,15 @@ class FunctionRunnersSection extends qreact.Component {
|
|
|
207
209
|
}
|
|
208
210
|
|
|
209
211
|
function nodeTotalTraffic(traffic: TrafficStats): { sent: number; received: number; outside: number; } {
|
|
210
|
-
let
|
|
211
|
-
let
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
212
|
+
let outsideSent = traffic.outside?.sent || 0;
|
|
213
|
+
let outsideReceived = traffic.outside?.received || 0;
|
|
214
|
+
let sent = outsideSent;
|
|
215
|
+
let received = outsideReceived;
|
|
216
|
+
for (let d of Object.values(traffic.perNode || {})) {
|
|
217
|
+
sent += d?.sent || 0;
|
|
218
|
+
received += d?.received || 0;
|
|
215
219
|
}
|
|
216
|
-
return { sent, received, outside:
|
|
220
|
+
return { sent, received, outside: outsideSent + outsideReceived };
|
|
217
221
|
}
|
|
218
222
|
|
|
219
223
|
function machineIdOf(nodeId: string): string {
|
|
@@ -235,40 +239,44 @@ function sumTraffic(threads: string[], trafficMaps: Map<string, TrafficStats>) {
|
|
|
235
239
|
let totals = nodeTotalTraffic(traffic);
|
|
236
240
|
dataSent += totals.sent;
|
|
237
241
|
dataReceived += totals.received;
|
|
238
|
-
valuesSent += traffic.pathValuesSent;
|
|
239
|
-
valuesReceived += traffic.pathValuesReceived;
|
|
240
|
-
calls += traffic.functionsExecuted;
|
|
241
|
-
addCalls += traffic.querysubCalls;
|
|
242
|
+
valuesSent += traffic.pathValuesSent || 0;
|
|
243
|
+
valuesReceived += traffic.pathValuesReceived || 0;
|
|
244
|
+
calls += traffic.functionsExecuted || 0;
|
|
245
|
+
addCalls += traffic.querysubCalls || 0;
|
|
242
246
|
}
|
|
243
247
|
return { dataSent, dataReceived, valuesSent, valuesReceived, calls, addCalls };
|
|
244
248
|
}
|
|
245
249
|
|
|
246
|
-
// A machine's graph label: the IP first (most important), then
|
|
247
|
-
// its threads and colored the same.
|
|
250
|
+
// A machine's graph label: the IP first (most important), then its function-runner networks, then the same per-piece
|
|
251
|
+
// info as the table, summed across all its threads and colored the same.
|
|
248
252
|
function machineLabelLines(config: {
|
|
249
253
|
machineId: string;
|
|
250
254
|
threads: string[];
|
|
251
255
|
trafficMaps: Map<string, TrafficStats>;
|
|
252
256
|
ip: string | undefined;
|
|
257
|
+
networks: string[];
|
|
253
258
|
}): LatencyGraphLabelLine[] {
|
|
254
|
-
let { machineId, threads, ip } = config;
|
|
259
|
+
let { machineId, threads, ip, networks } = config;
|
|
255
260
|
let totals = sumTraffic(threads, config.trafficMaps);
|
|
256
261
|
let lines: LatencyGraphLabelLine[] = [];
|
|
257
262
|
if (ip) {
|
|
258
263
|
lines.push({ text: ip });
|
|
259
264
|
}
|
|
265
|
+
if (networks.length) {
|
|
266
|
+
lines.push({ text: networks.join(" | "), color: QUERYSUB_COLOR });
|
|
267
|
+
}
|
|
260
268
|
lines.push({ text: `${machineId.slice(0, ID_CHARS)} ${threads.length} threads` });
|
|
261
269
|
if (totals.dataSent + totals.dataReceived > 0) {
|
|
262
|
-
lines.push({ text: `↑${formatNumber(totals.dataSent)}B ↓${formatNumber(totals.dataReceived)}B` });
|
|
270
|
+
lines.push({ text: `↑${formatNumber(totals.dataSent)}B/s ↓${formatNumber(totals.dataReceived)}B/s` });
|
|
263
271
|
}
|
|
264
272
|
if (totals.valuesSent || totals.valuesReceived) {
|
|
265
|
-
lines.push({ text: `↑${formatNumber(totals.valuesSent)} ↓${formatNumber(totals.valuesReceived)} values`, color: PATHVALUE_COLOR });
|
|
273
|
+
lines.push({ text: `↑${formatNumber(totals.valuesSent)}/s ↓${formatNumber(totals.valuesReceived)}/s values`, color: PATHVALUE_COLOR });
|
|
266
274
|
}
|
|
267
275
|
if (totals.calls) {
|
|
268
|
-
lines.push({ text: `${formatNumber(totals.calls)} calls`, color: FUNCTION_COLOR });
|
|
276
|
+
lines.push({ text: `${formatNumber(totals.calls)}/s calls`, color: FUNCTION_COLOR });
|
|
269
277
|
}
|
|
270
278
|
if (totals.addCalls) {
|
|
271
|
-
lines.push({ text: `${formatNumber(totals.addCalls)} addCalls`, color: QUERYSUB_COLOR });
|
|
279
|
+
lines.push({ text: `${formatNumber(totals.addCalls)}/s addCalls`, color: QUERYSUB_COLOR });
|
|
272
280
|
}
|
|
273
281
|
return lines;
|
|
274
282
|
}
|
|
@@ -280,8 +288,9 @@ function buildNodeRow(config: {
|
|
|
280
288
|
info: NodeAuthorityInfo | undefined;
|
|
281
289
|
traffic: TrafficStats | undefined;
|
|
282
290
|
machineIp: Map<string, string>;
|
|
291
|
+
machineNetworks: Map<string, string[]>;
|
|
283
292
|
}): NodeRow {
|
|
284
|
-
let { nodeId, runner, info, traffic, machineIp } = config;
|
|
293
|
+
let { nodeId, runner, info, traffic, machineIp, machineNetworks } = config;
|
|
285
294
|
let parts = decodeNodeId(nodeId, getDomain(), "allowMissingThreadId");
|
|
286
295
|
let thread = (parts?.threadId || "?").slice(0, ID_CHARS);
|
|
287
296
|
let machineId = parts?.machineId || nodeId;
|
|
@@ -294,7 +303,7 @@ function buildNodeRow(config: {
|
|
|
294
303
|
let sum = totals.sent + totals.received;
|
|
295
304
|
if (sum > 0) {
|
|
296
305
|
let outsidePct = Math.round((totals.outside / sum) * 100);
|
|
297
|
-
data = `↑${formatNumber(totals.sent)}B ↓${formatNumber(totals.received)}B ${outsidePct}%⊘`;
|
|
306
|
+
data = `↑${formatNumber(totals.sent)}B/s ↓${formatNumber(totals.received)}B/s ${outsidePct}%⊘`;
|
|
298
307
|
}
|
|
299
308
|
}
|
|
300
309
|
|
|
@@ -304,7 +313,7 @@ function buildNodeRow(config: {
|
|
|
304
313
|
for (let shard of runner.shards) {
|
|
305
314
|
fnParts.push(`fn ${shard.shardRange.startFraction.toFixed(3)}-${shard.shardRange.endFraction.toFixed(3)}`);
|
|
306
315
|
}
|
|
307
|
-
fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)} calls`);
|
|
316
|
+
fnParts.push(`${formatNumber(traffic?.functionsExecuted ?? 0)}/s calls`);
|
|
308
317
|
if (!runner.isPublic) fnParts.push("PRIVATE");
|
|
309
318
|
}
|
|
310
319
|
|
|
@@ -314,14 +323,15 @@ function buildNodeRow(config: {
|
|
|
314
323
|
pvParts.push(`pv ${spec.routeStart.toFixed(3)}-${spec.routeEnd.toFixed(3)}`);
|
|
315
324
|
}
|
|
316
325
|
if (traffic && (traffic.pathValuesSent || traffic.pathValuesReceived)) {
|
|
317
|
-
pvParts.push(`↑${formatNumber(traffic.pathValuesSent)} ↓${formatNumber(traffic.pathValuesReceived)} values`);
|
|
326
|
+
pvParts.push(`↑${formatNumber(traffic.pathValuesSent)}/s ↓${formatNumber(traffic.pathValuesReceived)}/s values`);
|
|
318
327
|
}
|
|
319
328
|
|
|
320
|
-
let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)} addCalls` : "";
|
|
329
|
+
let qs = traffic?.querysubCalls ? `${formatNumber(traffic.querysubCalls)}/s addCalls` : "";
|
|
321
330
|
return {
|
|
322
331
|
nodeId,
|
|
323
332
|
machineId,
|
|
324
333
|
ip: machineIp.get(machineId) || "",
|
|
334
|
+
networks: (machineNetworks.get(machineId) || []).join(" | "),
|
|
325
335
|
machineShort: machineId.slice(0, ID_CHARS),
|
|
326
336
|
threadPort: `${thread}:${parts?.port ?? "?"}`,
|
|
327
337
|
name,
|
|
@@ -428,14 +438,26 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
|
|
|
428
438
|
if (bestIp) machineIp.set(machineId, bestIp);
|
|
429
439
|
}
|
|
430
440
|
|
|
441
|
+
// The unique set of function-runner networks running on each machine (usually one, human-readable).
|
|
442
|
+
let machineNetworks = new Map<string, string[]>();
|
|
443
|
+
for (let [machineId, threads] of machineAllThreads) {
|
|
444
|
+
let unique = new Set<string>();
|
|
445
|
+
for (let thread of threads) {
|
|
446
|
+
for (let network of runnerByNode.get(thread)?.networks || []) {
|
|
447
|
+
unique.add(network);
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
if (unique.size) machineNetworks.set(machineId, [...unique]);
|
|
451
|
+
}
|
|
452
|
+
|
|
431
453
|
// Total bytes on each thread-pair link (both directions). Both endpoints report the same total, so take the max.
|
|
432
454
|
let pairKey = (a: string, b: string) => a < b ? `${a}|${b}` : `${b}|${a}`;
|
|
433
455
|
let pairTraffic = new Map<string, number>();
|
|
434
456
|
for (let [reporter, traffic] of trafficMaps) {
|
|
435
|
-
for (let [peer, data] of Object.entries(traffic.perNode)) {
|
|
457
|
+
for (let [peer, data] of Object.entries(traffic.perNode || {})) {
|
|
436
458
|
if (reporter === peer || !respondedSet.has(peer)) continue;
|
|
437
459
|
let key = pairKey(reporter, peer);
|
|
438
|
-
pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, data
|
|
460
|
+
pairTraffic.set(key, Math.max(pairTraffic.get(key) || 0, (data?.sent || 0) + (data?.received || 0)));
|
|
439
461
|
}
|
|
440
462
|
}
|
|
441
463
|
|
|
@@ -472,7 +494,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
|
|
|
472
494
|
let totals = sumTraffic(allThreads, trafficMaps);
|
|
473
495
|
return {
|
|
474
496
|
id: machineId,
|
|
475
|
-
labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId) }),
|
|
497
|
+
labelLines: machineLabelLines({ machineId, threads: allThreads, trafficMaps, ip: machineIp.get(machineId), networks: machineNetworks.get(machineId) || [] }),
|
|
476
498
|
weight: totals.dataSent + totals.dataReceived,
|
|
477
499
|
};
|
|
478
500
|
});
|
|
@@ -484,7 +506,7 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
|
|
|
484
506
|
|
|
485
507
|
// One row per thread node, grouped so a machine's threads sit together; the selected machine sorts to the top.
|
|
486
508
|
// Single composite key: selected-first, then machine, then thread — the low separator keeps segments ordered.
|
|
487
|
-
let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp }));
|
|
509
|
+
let rows = nodeIds.map(nodeId => buildNodeRow({ nodeId, runner: runnerByNode.get(nodeId), info: infoByNode.get(nodeId), traffic: trafficMaps.get(nodeId), machineIp, machineNetworks }));
|
|
488
510
|
let selected = selectedMachineParam.value;
|
|
489
511
|
let sep = String.fromCharCode(1);
|
|
490
512
|
sort(rows, row => `${row.machineId === selected ? 0 : 1}${sep}${row.machineId}${sep}${row.threadPort}`);
|
|
@@ -495,11 +517,12 @@ class LatencyGraphSection extends qreact.Component<{ nodeIds: string[]; infos: N
|
|
|
495
517
|
<LatencyGraph
|
|
496
518
|
nodes={nodes}
|
|
497
519
|
links={links}
|
|
498
|
-
formatWeight={weight => formatNumber(weight) + "B"}
|
|
520
|
+
formatWeight={weight => formatNumber(weight) + "B/s"}
|
|
499
521
|
selectedId={selected || undefined}
|
|
500
522
|
onSelectNode={id => selectedMachineParam.value = selectedMachineParam.value === id ? "" : id}
|
|
501
523
|
/>
|
|
502
524
|
</div>
|
|
525
|
+
<div className={css.colorhsl(0, 0, 50)}>Click a node to sort its information to the top of the table below.</div>
|
|
503
526
|
<h2 className={css.margin(0)}>Nodes ({rows.length})</h2>
|
|
504
527
|
<NodeInfoTable rows={rows} selectedMachine={selected} />
|
|
505
528
|
</div>;
|
|
@@ -23,11 +23,12 @@ export type LatencyGraphProps = {
|
|
|
23
23
|
const DEFAULT_CONNECTIONS = 4;
|
|
24
24
|
// Nearest neighbors drawn per node (the layout still uses every measured latency; this only thins the drawn lines).
|
|
25
25
|
const connectionsParam = new URLParam("lgConnections", DEFAULT_CONNECTIONS);
|
|
26
|
-
const DEFAULT_LATENCY_EXPONENT =
|
|
26
|
+
const DEFAULT_LATENCY_EXPONENT = 0.5;
|
|
27
27
|
// Exponent on (latency / reference) when mapping to layout distance, so far nodes push apart harder as it grows.
|
|
28
28
|
const latencyExponentParam = new URLParam("lgLatencyExponent", DEFAULT_LATENCY_EXPONENT);
|
|
29
29
|
const geoParam = new URLParam("lgGeo", false);
|
|
30
|
-
|
|
30
|
+
// On by default: each drawn connection shows its latency and (when a weight formatter is given) its traffic.
|
|
31
|
+
const showLatenciesParam = new URLParam("lgShowLatencies", true);
|
|
31
32
|
// The config panel is collapsed by default so it doesn't eat the canvas; clicking the header expands it.
|
|
32
33
|
const configOpenParam = new URLParam("lgConfigOpen", false);
|
|
33
34
|
// Pixels per degree of latitude/longitude in geographic mode (equirectangular projection).
|
|
@@ -45,6 +46,9 @@ const POWER_EPS = 1e-9;
|
|
|
45
46
|
// with a floor. Normalized to the median so the overall scale is latency-magnitude independent.
|
|
46
47
|
const DIST_MIN_PX = 8;
|
|
47
48
|
const DIST_UNIT_PX = 90;
|
|
49
|
+
// A touch of deterministic jitter on the MDS init breaks symmetry, so a handful of nodes settle into a spread (a
|
|
50
|
+
// triangle for three) instead of collapsing onto a single line.
|
|
51
|
+
const INIT_JITTER_PX = 15;
|
|
48
52
|
|
|
49
53
|
const MIN_OPACITY = 0.05;
|
|
50
54
|
const NODE_RADIUS = 6;
|
|
@@ -65,9 +69,6 @@ const FIT_PAD = 60;
|
|
|
65
69
|
const FIT_PAD_TOP = 160;
|
|
66
70
|
// Auto-fit leaves a bit of extra breathing room by not zooming all the way in to fill the padded box.
|
|
67
71
|
const FIT_ZOOM = 0.7;
|
|
68
|
-
// Cursor must be within this many screen pixels of a node to hover/click it.
|
|
69
|
-
const HOVER_HIT_PX = 60;
|
|
70
|
-
const CLICK_HIT_PX = 40;
|
|
71
72
|
|
|
72
73
|
type Edge = { a: number; b: number; latency: number; };
|
|
73
74
|
type SolveEdge = { a: number; b: number; target: number; weight: number; };
|
|
@@ -293,6 +294,11 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
293
294
|
});
|
|
294
295
|
this.positionsX = pos.x;
|
|
295
296
|
this.positionsY = pos.y;
|
|
297
|
+
// Deterministic per-node jitter (different frequencies for x/y so it isn't itself collinear) to break symmetry.
|
|
298
|
+
for (let i = 0; i < n; i++) {
|
|
299
|
+
this.positionsX[i] += Math.sin(i * 12.9898 + 1) * INIT_JITTER_PX;
|
|
300
|
+
this.positionsY[i] += Math.cos(i * 78.233 + 1) * INIT_JITTER_PX;
|
|
301
|
+
}
|
|
296
302
|
}
|
|
297
303
|
|
|
298
304
|
// Classical MDS on an m-point set given a squared-target-distance function; top-2 eigenvectors.
|
|
@@ -622,6 +628,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
622
628
|
let hover = this.hoverNode;
|
|
623
629
|
let connections = (this.neighbors[hover] || []).map(nb => nb.edge);
|
|
624
630
|
|
|
631
|
+
// Just highlight the hovered node's connections — their latency/traffic labels are already drawn all the time.
|
|
625
632
|
ctx.lineWidth = 1.5;
|
|
626
633
|
for (let edge of connections) {
|
|
627
634
|
ctx.strokeStyle = "hsla(40, 90%, 60%, 0.6)";
|
|
@@ -630,19 +637,6 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
630
637
|
ctx.lineTo(toScreenX(this.positionsX[edge.b]), toScreenY(this.positionsY[edge.b]));
|
|
631
638
|
ctx.stroke();
|
|
632
639
|
}
|
|
633
|
-
ctx.font = "12px sans-serif";
|
|
634
|
-
ctx.textAlign = "center";
|
|
635
|
-
ctx.textBaseline = "middle";
|
|
636
|
-
for (let edge of connections) {
|
|
637
|
-
let midX = (toScreenX(this.positionsX[edge.a]) + toScreenX(this.positionsX[edge.b])) / 2;
|
|
638
|
-
let midY = (toScreenY(this.positionsY[edge.a]) + toScreenY(this.positionsY[edge.b])) / 2;
|
|
639
|
-
let label = formatTime(edge.latency);
|
|
640
|
-
let textWidth = ctx.measureText(label).width;
|
|
641
|
-
ctx.fillStyle = "hsla(0, 0%, 0%, 0.75)";
|
|
642
|
-
ctx.fillRect(midX - textWidth / 2 - 3, midY - 8, textWidth + 6, 16);
|
|
643
|
-
ctx.fillStyle = "hsl(40, 90%, 75%)";
|
|
644
|
-
ctx.fillText(label, midX, midY);
|
|
645
|
-
}
|
|
646
640
|
// Redraw the hovered node's own label last so it always sits above the connection labels.
|
|
647
641
|
let hoverLines = this.nodes[hover].labelLines;
|
|
648
642
|
if (hoverLines && hoverLines.length) {
|
|
@@ -720,7 +714,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
720
714
|
this.lastPointerY = e.clientY;
|
|
721
715
|
};
|
|
722
716
|
|
|
723
|
-
//
|
|
717
|
+
// Index of the node closest to a screen point (undefined only when there are no nodes).
|
|
724
718
|
nearestNodeTo(mx: number, my: number) {
|
|
725
719
|
let scale = this.viewScale;
|
|
726
720
|
let bestDist = Infinity;
|
|
@@ -728,13 +722,13 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
728
722
|
for (let i = 0; i < this.nodes.length; i++) {
|
|
729
723
|
let sx = this.viewWidth / 2 + (this.positionsX[i] - this.centroidX) * scale + this.panX;
|
|
730
724
|
let sy = this.viewHeight / 2 + (this.positionsY[i] - this.centroidY) * scale + this.panY;
|
|
731
|
-
let dist =
|
|
725
|
+
let dist = (sx - mx) ** 2 + (sy - my) ** 2;
|
|
732
726
|
if (dist < bestDist) {
|
|
733
727
|
bestDist = dist;
|
|
734
728
|
best = i;
|
|
735
729
|
}
|
|
736
730
|
}
|
|
737
|
-
return
|
|
731
|
+
return best;
|
|
738
732
|
}
|
|
739
733
|
|
|
740
734
|
onMouseMove = (e: MouseEvent) => {
|
|
@@ -754,8 +748,8 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
754
748
|
let my = e.clientY - rect.top;
|
|
755
749
|
this.mouseX = mx;
|
|
756
750
|
this.mouseY = my;
|
|
757
|
-
|
|
758
|
-
this.hoverNode =
|
|
751
|
+
// Always hover whichever node the cursor is closest to, so moving anywhere explores the graph.
|
|
752
|
+
this.hoverNode = this.nearestNodeTo(mx, my);
|
|
759
753
|
this.scheduleFrame();
|
|
760
754
|
};
|
|
761
755
|
|
|
@@ -764,8 +758,8 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
764
758
|
if (!canvas || !this.onSelectNode) return;
|
|
765
759
|
let rect = canvas.getBoundingClientRect();
|
|
766
760
|
let nearest = this.nearestNodeTo(e.clientX - rect.left, e.clientY - rect.top);
|
|
767
|
-
if (nearest
|
|
768
|
-
this.onSelectNode(this.nodes[nearest
|
|
761
|
+
if (nearest === undefined) return;
|
|
762
|
+
this.onSelectNode(this.nodes[nearest].id);
|
|
769
763
|
};
|
|
770
764
|
|
|
771
765
|
onMouseLeave = () => {
|
|
@@ -842,7 +836,7 @@ export class LatencyGraph extends qreact.Component<LatencyGraphProps> {
|
|
|
842
836
|
return <div className={css.relative.fillBoth.overflowHidden}>
|
|
843
837
|
<canvas
|
|
844
838
|
ref={elem => this.mountCanvas(elem ?? undefined)}
|
|
845
|
-
className={css.absolute.pos(0, 0).fillBoth}
|
|
839
|
+
className={css.absolute.pos(0, 0).fillBoth.cursor("pointer")}
|
|
846
840
|
/>
|
|
847
841
|
<div className={css.vbox(10).pad2(12).absolute.pos(0, 0).width(260).zIndex(2).hsla(0, 0, 8, 0.9).colorhsl(0, 0, 85)}>
|
|
848
842
|
<div
|