querysub 0.524.0 → 0.526.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/-c-identity/IdentityController.ts +2 -2
- package/src/-d-trust/NetworkTrust2.ts +2 -2
- package/src/-f-node-discovery/LatencyTracking.ts +26 -2
- package/src/-f-node-discovery/TrafficTracking.ts +25 -19
- package/src/-h-path-value-serialize/PathValueSerializer.ts +35 -18
- package/src/0-path-value-core/AuthorityLookup.ts +4 -2
- package/src/0-path-value-core/PathRouter.ts +38 -8
- package/src/0-path-value-core/PathValueCommitter.ts +2 -2
- package/src/0-path-value-core/PathValueController.ts +74 -54
- package/src/0-path-value-core/ValidStateComputer.ts +28 -8
- package/src/1-path-client/RemoteWatcher.ts +59 -2
- package/src/3-path-functions/PathFunctionRunner.ts +4 -1
- package/src/4-deploy/edgeBootstrap.ts +1 -1
- package/src/4-querysub/FunctionRunnerTracking.ts +73 -3
- package/src/4-querysub/QuerysubController.ts +0 -2
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +1 -1
- package/src/diagnostics/misc-pages/RoutingTablePage.tsx +169 -18
- package/src/diagnostics/pathAuditer.ts +77 -18
- package/src/diagnostics/periodic.ts +2 -1
- package/src/diagnostics/watchdog.ts +11 -0
- package/src/library-components/LatencyGraph.tsx +43 -19
- package/src/server.ts +7 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.526.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -70,7 +70,7 @@
|
|
|
70
70
|
"pako": "^2.1.0",
|
|
71
71
|
"peggy": "^5.0.6",
|
|
72
72
|
"sliftutils": "^1.7.5",
|
|
73
|
-
"socket-function": "^1.2.
|
|
73
|
+
"socket-function": "^1.2.14",
|
|
74
74
|
"terser": "^5.31.0",
|
|
75
75
|
"typenode": "^6.6.1",
|
|
76
76
|
"typesafecss": "^0.32.0",
|
|
@@ -19,7 +19,7 @@ import { waitForFirstTimeSync } from "socket-function/time/trueTimeShim";
|
|
|
19
19
|
import { red } from "socket-function/src/formatting/logColors";
|
|
20
20
|
import { isNode } from "typesafecss";
|
|
21
21
|
import { areNodeIdsEqual, getOwnNodeId, getOwnThreadId } from "../-f-node-discovery/NodeDiscovery";
|
|
22
|
-
import { timeInMinute } from "socket-function/src/misc";
|
|
22
|
+
import { timeInMinute, isIpDomain } from "socket-function/src/misc";
|
|
23
23
|
import { isClient, isServer } from "../config2";
|
|
24
24
|
import { getDomain } from "../config";
|
|
25
25
|
|
|
@@ -242,7 +242,7 @@ const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectio
|
|
|
242
242
|
certIssuer: issuer.cert.toString(),
|
|
243
243
|
mountedPort: getNodeIdLocation(SocketFunction.mountedNodeId)?.port,
|
|
244
244
|
debugEntryPoint: isServer() ? process.argv[1] : "browser",
|
|
245
|
-
clientIsNode: isServer(),
|
|
245
|
+
clientIsNode: isServer() && !isIpDomain(nodeId),
|
|
246
246
|
};
|
|
247
247
|
let signature = sign(threadKeyCert, payload);
|
|
248
248
|
await timeoutToError(
|
|
@@ -111,7 +111,7 @@ export async function isNodeTrusted(nodeId: string) {
|
|
|
111
111
|
return await isTrusted(machineId);
|
|
112
112
|
}
|
|
113
113
|
|
|
114
|
-
const loadServerCert = cache(async (machineId: string) => {
|
|
114
|
+
export const loadServerCert = cache(async (machineId: string) => {
|
|
115
115
|
// This cert isn't stored in the archives, but... this is fine?
|
|
116
116
|
if (machineId === "127-0-0-1." + getDomain()) return;
|
|
117
117
|
let certFile = await archives().get(machineId);
|
|
@@ -131,7 +131,7 @@ export const ensureWeAreTrusted = lazy(measureWrap(async () => {
|
|
|
131
131
|
}
|
|
132
132
|
}));
|
|
133
133
|
|
|
134
|
-
async function loadTrustCerts(nodeId: string) {
|
|
134
|
+
export async function loadTrustCerts(nodeId: string) {
|
|
135
135
|
let location = getNodeIdLocation(nodeId);
|
|
136
136
|
if (location) {
|
|
137
137
|
let machineId = getMachineId(location.address, getDomain());
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
11
|
-
import { timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
11
|
+
import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
12
12
|
import { lazy } from "socket-function/src/caching";
|
|
13
13
|
import { delay, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
14
14
|
import { isServer } from "../config2";
|
|
@@ -23,6 +23,8 @@ const LATENCY_POLL_INTERVAL = timeInMinute;
|
|
|
23
23
|
const PING_TIMEOUT = timeInSecond * 5;
|
|
24
24
|
// Rolling average size. Small, so latency changes show up quickly.
|
|
25
25
|
const LATENCY_SAMPLE_LIMIT = 20;
|
|
26
|
+
// Raw sample history kept per node, so callers can ask for a median over the last N with a known confidence.
|
|
27
|
+
const LATENCY_HISTORY_LIMIT = 100;
|
|
26
28
|
// Forget nodes we haven't been able to reach for this long.
|
|
27
29
|
const NODE_EXPIRY_TIME = LATENCY_POLL_INTERVAL * 5;
|
|
28
30
|
|
|
@@ -30,6 +32,7 @@ export type NodeLatencyInfo = {
|
|
|
30
32
|
averageLatency: number;
|
|
31
33
|
sampleCount: number;
|
|
32
34
|
lastSeen: number;
|
|
35
|
+
history: number[];
|
|
33
36
|
};
|
|
34
37
|
|
|
35
38
|
// otherNodeId => our rolling latency to it
|
|
@@ -42,6 +45,18 @@ export function getNodeLatencyInfo(nodeId: string): NodeLatencyInfo | undefined
|
|
|
42
45
|
export function getNodeLatency(nodeId: string): number | undefined {
|
|
43
46
|
return getNodeLatencyInfo(nodeId)?.averageLatency;
|
|
44
47
|
}
|
|
48
|
+
// Median of the last historyCount raw samples, plus how many samples were actually available — so callers can scale how much they trust the number.
|
|
49
|
+
export function getNodeLatencyMedian(config: { nodeId: string; historyCount: number }): { latency: number; historyUsed: number } | undefined {
|
|
50
|
+
let history = getNodeLatencyInfo(config.nodeId)?.history;
|
|
51
|
+
if (!history || history.length === 0) return undefined;
|
|
52
|
+
let samples = history.slice(-config.historyCount);
|
|
53
|
+
sort(samples, x => x);
|
|
54
|
+
return { latency: samples[Math.floor(samples.length / 2)], historyUsed: samples.length };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export function getCachedNodeLatencyInfoList(): Map<string, NodeLatencyInfo> {
|
|
58
|
+
return latencyByNode;
|
|
59
|
+
}
|
|
45
60
|
|
|
46
61
|
/** Our measured latency to every node we have reached, as a plain map (for sending over the wire). */
|
|
47
62
|
export function getOwnLatencies(): { [nodeId: string]: number } {
|
|
@@ -56,10 +71,16 @@ export function getOwnLatencies(): { [nodeId: string]: number } {
|
|
|
56
71
|
function recordLatency(nodeId: string, latency: number) {
|
|
57
72
|
let prev = latencyByNode.get(nodeId);
|
|
58
73
|
let sampleCount = Math.min(prev?.sampleCount || 0, LATENCY_SAMPLE_LIMIT);
|
|
74
|
+
let history = prev?.history || [];
|
|
75
|
+
history.push(latency);
|
|
76
|
+
if (history.length > LATENCY_HISTORY_LIMIT) {
|
|
77
|
+
history.shift();
|
|
78
|
+
}
|
|
59
79
|
latencyByNode.set(nodeId, {
|
|
60
80
|
averageLatency: ((prev?.averageLatency || 0) * sampleCount + latency) / (sampleCount + 1),
|
|
61
81
|
sampleCount: sampleCount + 1,
|
|
62
82
|
lastSeen: Date.now(),
|
|
83
|
+
history,
|
|
63
84
|
});
|
|
64
85
|
}
|
|
65
86
|
|
|
@@ -72,9 +93,11 @@ async function pingNode(nodeId: string) {
|
|
|
72
93
|
recordLatency(nodeId, Date.now() - start);
|
|
73
94
|
}
|
|
74
95
|
|
|
96
|
+
let firstCall = true;
|
|
75
97
|
async function pollLatencies() {
|
|
76
98
|
let nodeIds = (await getAllNodeIds()).filter(nodeId => !isOwnNodeId(nodeId));
|
|
77
|
-
|
|
99
|
+
let spread = firstCall ? 0 : LATENCY_POLL_INTERVAL;
|
|
100
|
+
await spreadCallsOverTime(nodeIds, spread, pingNode);
|
|
78
101
|
|
|
79
102
|
let now = Date.now();
|
|
80
103
|
for (let [nodeId, info] of Array.from(latencyByNode)) {
|
|
@@ -82,6 +105,7 @@ async function pollLatencies() {
|
|
|
82
105
|
latencyByNode.delete(nodeId);
|
|
83
106
|
}
|
|
84
107
|
}
|
|
108
|
+
firstCall = false;
|
|
85
109
|
}
|
|
86
110
|
|
|
87
111
|
export const startLatencyTracking = lazy(async () => {
|
|
@@ -26,14 +26,14 @@ const BUCKET_HISTORY = BUCKET_SIZE * 20;
|
|
|
26
26
|
// not O(client connections) — clients churn through many raw ids per hour and we never want a bucket per client.
|
|
27
27
|
const OUTSIDE_KEY = "";
|
|
28
28
|
|
|
29
|
-
export type NodeDataTraffic = { sent: number; received: number; };
|
|
29
|
+
export type NodeDataTraffic = { sent: number; received: number; pathValuesSent: number; pathValuesReceived: number; };
|
|
30
30
|
// All numbers are PER-SECOND rates (see file header).
|
|
31
31
|
export type TrafficStats = {
|
|
32
32
|
functionsExecuted: number;
|
|
33
33
|
pathValuesSent: number;
|
|
34
34
|
pathValuesReceived: number;
|
|
35
35
|
querysubCalls: number;
|
|
36
|
-
// Byte rates to/from each known network node, plus one aggregate for everything outside the node list.
|
|
36
|
+
// Byte and path value rates to/from each known network node, plus one aggregate for everything outside the node list.
|
|
37
37
|
perNode: { [nodeId: string]: NodeDataTraffic };
|
|
38
38
|
outside: NodeDataTraffic;
|
|
39
39
|
};
|
|
@@ -48,6 +48,8 @@ let querysubCalls: BucketMap = new Map();
|
|
|
48
48
|
// raw connection nodeId => buckets
|
|
49
49
|
let bytesSent = new Map<string, BucketMap>();
|
|
50
50
|
let bytesReceived = new Map<string, BucketMap>();
|
|
51
|
+
let pathValuesSentPerNode = new Map<string, BucketMap>();
|
|
52
|
+
let pathValuesReceivedPerNode = new Map<string, BucketMap>();
|
|
51
53
|
|
|
52
54
|
function addSimple(buckets: BucketMap, amount: number) {
|
|
53
55
|
let bucket = Math.floor(Date.now() / BUCKET_SIZE) * BUCKET_SIZE;
|
|
@@ -65,13 +67,15 @@ function addKeyed(map: Map<string, BucketMap>, key: string, amount: number) {
|
|
|
65
67
|
export function recordFunctionExecuted() {
|
|
66
68
|
addSimple(functionsExecuted, 1);
|
|
67
69
|
}
|
|
68
|
-
export function recordPathValuesSent(count: number) {
|
|
69
|
-
if (count <= 0) return;
|
|
70
|
-
addSimple(pathValuesSent, count);
|
|
70
|
+
export function recordPathValuesSent(config: { count: number; nodeId: string }) {
|
|
71
|
+
if (config.count <= 0) return;
|
|
72
|
+
addSimple(pathValuesSent, config.count);
|
|
73
|
+
addKeyed(pathValuesSentPerNode, trafficKey(config.nodeId), config.count);
|
|
71
74
|
}
|
|
72
|
-
export function recordPathValuesReceived(count: number) {
|
|
73
|
-
if (count <= 0) return;
|
|
74
|
-
addSimple(pathValuesReceived, count);
|
|
75
|
+
export function recordPathValuesReceived(config: { count: number; nodeId: string }) {
|
|
76
|
+
if (config.count <= 0) return;
|
|
77
|
+
addSimple(pathValuesReceived, config.count);
|
|
78
|
+
addKeyed(pathValuesReceivedPerNode, trafficKey(config.nodeId), config.count);
|
|
75
79
|
}
|
|
76
80
|
export function recordQuerysubCall() {
|
|
77
81
|
addSimple(querysubCalls, 1);
|
|
@@ -111,10 +115,10 @@ function bucketRate(buckets: BucketMap, windowSize: number): number {
|
|
|
111
115
|
function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeDataTraffic }; outside: NodeDataTraffic; } {
|
|
112
116
|
let known = new Set(getCachedNodeIds());
|
|
113
117
|
let perNode: { [nodeId: string]: NodeDataTraffic } = {};
|
|
114
|
-
let outside: NodeDataTraffic = { sent: 0, received: 0 };
|
|
118
|
+
let outside: NodeDataTraffic = { sent: 0, received: 0, pathValuesSent: 0, pathValuesReceived: 0 };
|
|
115
119
|
// At most (node count + 1) keys, so this whole pass is O(nodes). debugNodeId (a linear lookup) is only used for
|
|
116
120
|
// the rare server-style id that isn't already a known node.
|
|
117
|
-
let attribute = (rawNodeId: string, field:
|
|
121
|
+
let attribute = (rawNodeId: string, field: keyof NodeDataTraffic, rate: number) => {
|
|
118
122
|
if (rate <= 0) return;
|
|
119
123
|
if (rawNodeId === OUTSIDE_KEY) {
|
|
120
124
|
outside[field] += rate;
|
|
@@ -124,7 +128,7 @@ function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeD
|
|
|
124
128
|
if (known.has(nice)) {
|
|
125
129
|
let entry = perNode[nice];
|
|
126
130
|
if (!entry) {
|
|
127
|
-
entry = { sent: 0, received: 0 };
|
|
131
|
+
entry = { sent: 0, received: 0, pathValuesSent: 0, pathValuesReceived: 0 };
|
|
128
132
|
perNode[nice] = entry;
|
|
129
133
|
}
|
|
130
134
|
entry[field] += rate;
|
|
@@ -132,14 +136,16 @@ function aggregateData(windowSize: number): { perNode: { [nodeId: string]: NodeD
|
|
|
132
136
|
outside[field] += rate;
|
|
133
137
|
}
|
|
134
138
|
};
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
139
|
+
let aggregateMap = (map: Map<string, BucketMap>, field: keyof NodeDataTraffic) => {
|
|
140
|
+
for (let [rawNodeId, buckets] of Array.from(map)) {
|
|
141
|
+
attribute(rawNodeId, field, bucketRate(buckets, windowSize));
|
|
142
|
+
if (buckets.size === 0) map.delete(rawNodeId);
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
aggregateMap(bytesSent, "sent");
|
|
146
|
+
aggregateMap(bytesReceived, "received");
|
|
147
|
+
aggregateMap(pathValuesSentPerNode, "pathValuesSent");
|
|
148
|
+
aggregateMap(pathValuesReceivedPerNode, "pathValuesReceived");
|
|
143
149
|
return { perNode, outside };
|
|
144
150
|
}
|
|
145
151
|
|
|
@@ -20,7 +20,7 @@ import { formatNumber, formatPercent, formatTime } from "socket-function/src/for
|
|
|
20
20
|
setFlag(require, "cbor-x", "allowclient", true);
|
|
21
21
|
|
|
22
22
|
import * as pako from "pako";
|
|
23
|
-
import { delay } from "socket-function/src/batching";
|
|
23
|
+
import { delay, safeLoop } from "socket-function/src/batching";
|
|
24
24
|
import { LZ4 } from "../storage/LZ4";
|
|
25
25
|
import { unblockLoop } from "socket-function/src/batching";
|
|
26
26
|
setFlag(require, "pako", "allowclient", true);
|
|
@@ -345,6 +345,7 @@ class PathValueSerializer {
|
|
|
345
345
|
let encodedValues: (Buffer | number)[] = [];
|
|
346
346
|
|
|
347
347
|
let i = 0;
|
|
348
|
+
let valuesSinceYield = 0;
|
|
348
349
|
for (let values of valuesGroups) {
|
|
349
350
|
measureBlock(() => {
|
|
350
351
|
// NOTE: Writing one value at a time is about 1.6X slower. BUT, it allows us to very efficient decode
|
|
@@ -378,21 +379,23 @@ class PathValueSerializer {
|
|
|
378
379
|
i++;
|
|
379
380
|
}
|
|
380
381
|
}, "valuesWriteLoop");
|
|
381
|
-
|
|
382
|
+
valuesSinceYield += values.length;
|
|
383
|
+
if (valuesSinceYield >= 10_000) {
|
|
384
|
+
valuesSinceYield = 0;
|
|
385
|
+
await delay("paintLoop");
|
|
386
|
+
}
|
|
382
387
|
}
|
|
383
388
|
|
|
384
389
|
measureBlock(function valuesWriteTypesBuffer() {
|
|
385
390
|
writer.writeBuffer(types);
|
|
386
391
|
});
|
|
387
|
-
// Break encodeValues into groups of
|
|
388
|
-
// if we are taking too long, but not check it EVERY loop, as that would
|
|
389
|
-
// be too slow.
|
|
392
|
+
// Break encodeValues into groups, and yield based on the amount of values written — NOT by measuring time, as timing every loop is itself too slow.
|
|
390
393
|
let valueGroups: (Buffer | number)[][] = [];
|
|
391
394
|
const VALUE_GROUP_SIZE = 1000;
|
|
392
395
|
for (let i = 0; i < encodedValues.length; i += VALUE_GROUP_SIZE) {
|
|
393
396
|
valueGroups.push(encodedValues.slice(i, i + VALUE_GROUP_SIZE));
|
|
394
397
|
}
|
|
395
|
-
let
|
|
398
|
+
let encodedSinceYield = 0;
|
|
396
399
|
for (let encodedValues of valueGroups) {
|
|
397
400
|
measureBlock(function valuesWriteTypes() {
|
|
398
401
|
for (let value of encodedValues) {
|
|
@@ -403,10 +406,10 @@ class PathValueSerializer {
|
|
|
403
406
|
}
|
|
404
407
|
}
|
|
405
408
|
});
|
|
406
|
-
|
|
407
|
-
if (
|
|
409
|
+
encodedSinceYield += encodedValues.length;
|
|
410
|
+
if (encodedSinceYield >= 10_000) {
|
|
411
|
+
encodedSinceYield = 0;
|
|
408
412
|
await delay("paintLoop");
|
|
409
|
-
prevTime = now;
|
|
410
413
|
}
|
|
411
414
|
}
|
|
412
415
|
}
|
|
@@ -550,9 +553,14 @@ class PathValueSerializer {
|
|
|
550
553
|
}
|
|
551
554
|
|
|
552
555
|
startNewBuffer("pathValues");
|
|
556
|
+
let pathValuesSinceYield = 0;
|
|
553
557
|
for (let values of valueGroups) {
|
|
554
558
|
this.pathValuesWrite(writer, values, settings);
|
|
555
|
-
|
|
559
|
+
pathValuesSinceYield += values.length;
|
|
560
|
+
if (pathValuesSinceYield >= 10_000) {
|
|
561
|
+
pathValuesSinceYield = 0;
|
|
562
|
+
await delay("afterPaint");
|
|
563
|
+
}
|
|
556
564
|
}
|
|
557
565
|
|
|
558
566
|
if (!settings.noLocks) {
|
|
@@ -593,15 +601,20 @@ class PathValueSerializer {
|
|
|
593
601
|
curCount += str.length;
|
|
594
602
|
}
|
|
595
603
|
|
|
604
|
+
let stringBytesSinceYield = 0;
|
|
596
605
|
for (let strings of stringParts) {
|
|
597
606
|
let stringBuffer = StringSerialize.serializeStrings(strings);
|
|
598
607
|
settings.bufferMap!.push("strings");
|
|
599
608
|
outputBuffers.push(stringBuffer);
|
|
600
|
-
|
|
609
|
+
stringBytesSinceYield += stringBuffer.length;
|
|
610
|
+
if (stringBytesSinceYield >= 1_000_000) {
|
|
611
|
+
stringBytesSinceYield = 0;
|
|
612
|
+
await delay("paintLoop");
|
|
613
|
+
}
|
|
601
614
|
}
|
|
602
615
|
|
|
603
616
|
if (settings.compression === "lz4") {
|
|
604
|
-
let compressedBuffers = await
|
|
617
|
+
let compressedBuffers = await measureBlock(() => safeLoop({ data: outputBuffers, name: "PathValueSerializer.serialize|lz4" }, x => LZ4.compress(x)), "PathValueSerializer.serialize|lz4");
|
|
605
618
|
|
|
606
619
|
// If the compress factor is less than a threshold, use the uncompressed buffers
|
|
607
620
|
let uncompressedSize = outputBuffers.reduce((total, x) => total + x.length, 0);
|
|
@@ -723,6 +736,7 @@ class PathValueSerializer {
|
|
|
723
736
|
}
|
|
724
737
|
if (!config?.skipStrings) {
|
|
725
738
|
let stringArrays: string[][] = [];
|
|
739
|
+
let bytesSinceYield = 0;
|
|
726
740
|
for (let stringBuffer of stringBuffers) {
|
|
727
741
|
let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
|
|
728
742
|
while (true) {
|
|
@@ -731,11 +745,14 @@ class PathValueSerializer {
|
|
|
731
745
|
break;
|
|
732
746
|
}
|
|
733
747
|
stringArrays.push(nextStrings);
|
|
734
|
-
|
|
748
|
+
// Each getNextStrings call decodes ~1MB, so on buffers past that we yield per chunk decoded.
|
|
749
|
+
if (stringBuffer.length > 1_000_000) {
|
|
735
750
|
await delay("paintLoop");
|
|
736
751
|
}
|
|
737
752
|
}
|
|
738
|
-
|
|
753
|
+
bytesSinceYield += stringBuffer.length;
|
|
754
|
+
if (bytesSinceYield >= 1_000_000) {
|
|
755
|
+
bytesSinceYield = 0;
|
|
739
756
|
await delay("paintLoop");
|
|
740
757
|
}
|
|
741
758
|
}
|
|
@@ -848,6 +865,7 @@ class PathValueSerializer {
|
|
|
848
865
|
}
|
|
849
866
|
|
|
850
867
|
let stringArrays: string[][] = [];
|
|
868
|
+
let bytesSinceYield = 0;
|
|
851
869
|
for (let bufferIndex of stringBufferIndexes) {
|
|
852
870
|
let stringBuffer = buffers[bufferIndex];
|
|
853
871
|
let obj = StringSerialize.deserializeStringsLazy(stringBuffer);
|
|
@@ -857,13 +875,12 @@ class PathValueSerializer {
|
|
|
857
875
|
break;
|
|
858
876
|
}
|
|
859
877
|
stringArrays.push(nextStrings);
|
|
860
|
-
|
|
878
|
+
bytesSinceYield += stringBuffer.length;
|
|
879
|
+
if (bytesSinceYield >= 1_000_000) {
|
|
880
|
+
bytesSinceYield = 0;
|
|
861
881
|
await delay("paintLoop");
|
|
862
882
|
}
|
|
863
883
|
}
|
|
864
|
-
if (stringBufferIndexes.length > 1) {
|
|
865
|
-
await delay("paintLoop");
|
|
866
|
-
}
|
|
867
884
|
}
|
|
868
885
|
strings = stringArrays.flat();
|
|
869
886
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
1
|
+
import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
2
2
|
import { nestArchives } from "../-a-archives/archives";
|
|
3
3
|
import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
|
|
4
4
|
import { archiveJSONT } from "../-a-archives/archivesJSONT";
|
|
@@ -31,7 +31,7 @@ let CALL_TIMEOUT = timeInSecond * 5;
|
|
|
31
31
|
const SYNC_JITTER_WINDOW = timeInSecond * 10;
|
|
32
32
|
|
|
33
33
|
// Nodes this close to their scheduled shutdown are avoided as sources (only used if nothing else can satisfy the request).
|
|
34
|
-
const SHUTDOWN_AVOID_WINDOW = timeInMinute *
|
|
34
|
+
const SHUTDOWN_AVOID_WINDOW = timeInMinute * 5;
|
|
35
35
|
|
|
36
36
|
export type AuthorityEntry = {
|
|
37
37
|
nodeId: string;
|
|
@@ -82,6 +82,8 @@ class AuthorityLookup {
|
|
|
82
82
|
public async setOurSpec(spec: AuthoritySpec) {
|
|
83
83
|
if (!SocketFunction.isMounted()) throw new Error("Cannot call setOurPaths without mounting first (use Querysub.hostService).");
|
|
84
84
|
spec.nodeId = getOwnNodeId();
|
|
85
|
+
// AuthoritySpec.prefixes promises to be sorted, and this is the choke point every published spec goes through.
|
|
86
|
+
sort(spec.prefixes, prefix => prefix.originalPrefix);
|
|
85
87
|
|
|
86
88
|
|
|
87
89
|
if (this.setSpec) {
|
|
@@ -5,7 +5,7 @@ import { PathValue } from "./pathValueCore";
|
|
|
5
5
|
import { shuffle } from "../misc/random";
|
|
6
6
|
import { fastHash } from "../misc/hash";
|
|
7
7
|
import { getOwnNodeId, isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
|
|
8
|
-
import {
|
|
8
|
+
import { getNodeLatencyMedian } from "../-f-node-discovery/LatencyTracking";
|
|
9
9
|
import { unique } from "../misc";
|
|
10
10
|
import { measureFnc } from "socket-function/src/profiling/measure";
|
|
11
11
|
import { getRoutingOverride, hasPrefixHash } from "./PathRouterRouteOverride";
|
|
@@ -22,12 +22,18 @@ export { LOCAL_DOMAIN, LOCAL_DOMAIN_PATH };
|
|
|
22
22
|
// fine (two nearby servers just talk to each other). Replaces the old uniform-random pick between candidates.
|
|
23
23
|
const LATENCY_CANDIDATE_LIMIT = 5;
|
|
24
24
|
const LATENCY_WEIGHT_OFFSET = 10;
|
|
25
|
+
// How many latency samples we use per node (median of the last N). The more we actually got, the more we trust the number, and the more aggressively we cut slow candidates.
|
|
26
|
+
const LATENCY_HISTORY_COUNT = 10;
|
|
27
|
+
// A candidate whose weight is below (the current max weight / (this - its historyUsed)) is dropped to zero probability. So with full history the factor is 2 (2x the effective latency of the best is never picked), and with a single sample it is 11 (only extreme outliers are cut).
|
|
28
|
+
const LATENCY_CUTOFF_BASE_FACTOR = 12;
|
|
25
29
|
// Latency assumed for a node we haven't measured yet, so new/unreached nodes stay eligible but aren't preferred.
|
|
26
30
|
const UNKNOWN_LATENCY_MS = 50;
|
|
27
31
|
|
|
28
|
-
function getRoutingLatency(nodeId: string): number {
|
|
29
|
-
if (isOwnNodeId(nodeId)) return 0;
|
|
30
|
-
|
|
32
|
+
function getRoutingLatency(nodeId: string): { latency: number; historyUsed: number } {
|
|
33
|
+
if (isOwnNodeId(nodeId)) return { latency: 0, historyUsed: LATENCY_HISTORY_COUNT };
|
|
34
|
+
let median = getNodeLatencyMedian({ nodeId, historyCount: LATENCY_HISTORY_COUNT });
|
|
35
|
+
if (!median) return { latency: UNKNOWN_LATENCY_MS, historyUsed: 0 };
|
|
36
|
+
return median;
|
|
31
37
|
}
|
|
32
38
|
|
|
33
39
|
// In-place reorder of candidates by a latency-weighted random draw over the lowest LATENCY_CANDIDATE_LIMIT
|
|
@@ -36,20 +42,31 @@ function getRoutingLatency(nodeId: string): number {
|
|
|
36
42
|
// since an overloaded server's latency spikes, which already makes it less likely to be chosen here.
|
|
37
43
|
function latencyWeightedShuffle<T extends { nodeId: string }>(arr: T[]) {
|
|
38
44
|
if (arr.length <= 1) return;
|
|
39
|
-
let withLatency = arr.map(x =>
|
|
45
|
+
let withLatency = arr.map(x => {
|
|
46
|
+
let routingLatency = getRoutingLatency(x.nodeId);
|
|
47
|
+
return { x, latency: routingLatency.latency, historyUsed: routingLatency.historyUsed };
|
|
48
|
+
});
|
|
40
49
|
sort(withLatency, e => e.latency);
|
|
41
50
|
let pool = withLatency.slice(0, LATENCY_CANDIDATE_LIMIT);
|
|
42
51
|
let rest = withLatency.slice(LATENCY_CANDIDATE_LIMIT);
|
|
43
52
|
let ordered: T[] = [];
|
|
44
53
|
while (pool.length > 0) {
|
|
54
|
+
let weights = pool.map(e => 1 / (e.latency + LATENCY_WEIGHT_OFFSET));
|
|
55
|
+
let maxWeight = Math.max(...weights);
|
|
56
|
+
for (let i = 0; i < weights.length; i++) {
|
|
57
|
+
let cutoffFactor = LATENCY_CUTOFF_BASE_FACTOR - Math.min(pool[i].historyUsed, LATENCY_HISTORY_COUNT);
|
|
58
|
+
if (weights[i] < maxWeight / cutoffFactor) {
|
|
59
|
+
weights[i] = 0;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
45
62
|
let total = 0;
|
|
46
|
-
for (let
|
|
47
|
-
total +=
|
|
63
|
+
for (let w of weights) {
|
|
64
|
+
total += w;
|
|
48
65
|
}
|
|
49
66
|
let r = Math.random() * total;
|
|
50
67
|
let idx = 0;
|
|
51
68
|
while (idx < pool.length - 1) {
|
|
52
|
-
r -=
|
|
69
|
+
r -= weights[idx];
|
|
53
70
|
if (r <= 0) break;
|
|
54
71
|
idx++;
|
|
55
72
|
}
|
|
@@ -82,6 +99,7 @@ export type AuthoritySpec = {
|
|
|
82
99
|
routeEnd: number;
|
|
83
100
|
// If the path.startsWith(prefix), but prefix !== path, then we hash getPathIndex(path, hashIndex)
|
|
84
101
|
// - For now, let's just never add overlapping prefixes.
|
|
102
|
+
// - Sorted by originalPrefix (authorityLookup.setOurSpec sorts before publishing), so specs can be compared element-wise.
|
|
85
103
|
prefixes: PrefixMatcher[];
|
|
86
104
|
// - Make sure to set this if you just want the prefix values, otherwise you will get all that that prefixes don't match (the prefixes exclude prefix match but where route does not match).
|
|
87
105
|
excludeDefault?: boolean;
|
|
@@ -154,6 +172,18 @@ export function parsePrefixMatcher(prefixPath: string): PrefixMatcher {
|
|
|
154
172
|
};
|
|
155
173
|
}
|
|
156
174
|
|
|
175
|
+
// Full-overlap equivalence: same range, same default matching, and the same prefixes (prefixes are sorted, so element-wise comparison works).
|
|
176
|
+
export function areAuthoritySpecsEquivalent(a: AuthoritySpec, b: AuthoritySpec): boolean {
|
|
177
|
+
if (a.routeStart !== b.routeStart) return false;
|
|
178
|
+
if (a.routeEnd !== b.routeEnd) return false;
|
|
179
|
+
if (!a.excludeDefault !== !b.excludeDefault) return false;
|
|
180
|
+
if (a.prefixes.length !== b.prefixes.length) return false;
|
|
181
|
+
for (let [i, prefix] of a.prefixes.entries()) {
|
|
182
|
+
if (prefix.originalPrefix !== b.prefixes[i].originalPrefix) return false;
|
|
183
|
+
}
|
|
184
|
+
return true;
|
|
185
|
+
}
|
|
186
|
+
|
|
157
187
|
export function debugSpec(spec: AuthoritySpec) {
|
|
158
188
|
return {
|
|
159
189
|
info: `${spec.routeStart}-${spec.routeEnd} (${spec.prefixes.length} prefixes${spec.excludeDefault ? " excluding default" : ""})`,
|
|
@@ -144,7 +144,7 @@ class PathValueCommitter {
|
|
|
144
144
|
}
|
|
145
145
|
|
|
146
146
|
private broadcastValues = batchFunction(
|
|
147
|
-
{ delay:
|
|
147
|
+
{ delay: 1, throttleWindow: 2000, noMeasure: true },
|
|
148
148
|
async function internal_forwardWrites(valuesBatched: {
|
|
149
149
|
values: Set<PathValue>;
|
|
150
150
|
tryCount?: number;
|
|
@@ -327,7 +327,7 @@ class PathValueCommitter {
|
|
|
327
327
|
|
|
328
328
|
|
|
329
329
|
public ingestRemoteValuesAndValidStates = batchFunction(
|
|
330
|
-
{ delay:
|
|
330
|
+
{ delay: 1, throttleWindow: 1000, name: "ingestRemoteValuesAndValidStates", noMeasure: true },
|
|
331
331
|
async (batched: RemoteValueAndValidState[]) => {
|
|
332
332
|
const { remoteWatcher } = await import("../1-path-client/RemoteWatcher");
|
|
333
333
|
|