querysub 0.407.0 → 0.408.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/audit-disk-values.js +7 -0
- package/package.json +4 -3
- package/src/-a-archives/archiveCache.ts +12 -9
- package/src/-a-auth/certs.ts +1 -1
- package/src/-c-identity/IdentityController.ts +9 -1
- package/src/-f-node-discovery/NodeDiscovery.ts +63 -8
- package/src/0-path-value-core/AuthorityLookup.ts +8 -3
- package/src/0-path-value-core/PathRouter.ts +109 -68
- package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +4 -2
- package/src/0-path-value-core/PathValueCommitter.ts +3 -1
- package/src/0-path-value-core/PathValueController.ts +75 -4
- package/src/0-path-value-core/PathWatcher.ts +39 -0
- package/src/0-path-value-core/ShardPrefixes.ts +2 -0
- package/src/0-path-value-core/ValidStateComputer.ts +20 -8
- package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +11 -29
- package/src/0-path-value-core/pathValueArchives.ts +16 -5
- package/src/0-path-value-core/pathValueCore.ts +43 -3
- package/src/1-path-client/RemoteWatcher.ts +46 -25
- package/src/4-querysub/Querysub.ts +17 -5
- package/src/4-querysub/QuerysubController.ts +21 -10
- package/src/4-querysub/predictionQueue.tsx +3 -0
- package/src/4-querysub/querysubPrediction.ts +27 -20
- package/src/5-diagnostics/nodeMetadata.ts +17 -0
- package/src/diagnostics/NodeConnectionsPage.tsx +167 -0
- package/src/diagnostics/NodeViewer.tsx +11 -15
- package/src/diagnostics/PathDistributionInfo.tsx +102 -0
- package/src/diagnostics/auditDiskValues.ts +221 -0
- package/src/diagnostics/auditDiskValuesEntry.ts +43 -0
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +5 -1
- package/src/diagnostics/logs/TimeRangeSelector.tsx +3 -3
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +2 -0
- package/src/diagnostics/managementPages.tsx +10 -1
- package/src/diagnostics/misc-pages/ArchiveViewer.tsx +3 -2
- package/src/diagnostics/pathAuditer.ts +21 -0
- package/tempnotes.txt +5 -44
- package/test.ts +13 -301
- package/src/diagnostics/benchmark.ts +0 -139
- package/src/diagnostics/runSaturationTest.ts +0 -416
- package/src/diagnostics/satSchema.ts +0 -64
- package/src/test/mongoSatTest.tsx +0 -55
- package/src/test/satTest.ts +0 -193
- package/src/test/test.tsx +0 -552
|
@@ -6,7 +6,7 @@ import { isNode, sort } from "socket-function/src/misc";
|
|
|
6
6
|
import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
|
|
7
7
|
import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
|
|
8
8
|
import { PathValueController } from "../0-path-value-core/PathValueController";
|
|
9
|
-
import { PathRouter } from "../0-path-value-core/PathRouter";
|
|
9
|
+
import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
|
|
10
10
|
import { WatchConfig, authorityStorage } from "../0-path-value-core/pathValueCore";
|
|
11
11
|
import { pathWatcher } from "../0-path-value-core/PathWatcher";
|
|
12
12
|
import { ActionsHistory } from "../diagnostics/ActionsHistory";
|
|
@@ -19,8 +19,9 @@ import { isDevDebugbreak } from "../config";
|
|
|
19
19
|
import { auditLog, isDebugLogEnabled } from "../0-path-value-core/auditLogs";
|
|
20
20
|
import { debugNodeThread } from "../-c-identity/IdentityController";
|
|
21
21
|
import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
|
|
22
|
-
import { decodeParentFilter, encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
|
|
22
|
+
import { decodeParentFilter, encodeParentFilter, registerGetSpecForChildPath } from "../0-path-value-core/hackedPackedPathParentFiltering";
|
|
23
23
|
import { removeRange, rangesOverlap } from "../rangeMath";
|
|
24
|
+
|
|
24
25
|
setImmediate(() => import("../4-querysub/Querysub"));
|
|
25
26
|
|
|
26
27
|
// NOTE: In some cases this can half our the PathValues sent. AND, it can reduce our sync requests by N, and as those are uploads those are event more expensive. However, it also complicates other code, requiring not just checking if a value is synced, but also the parent path. We should try to make this stable, but worst case scenario, we can set this to false, and it will make our synchronization a lot easier to verify.
|
|
@@ -51,8 +52,8 @@ export class RemoteWatcher {
|
|
|
51
52
|
|
|
52
53
|
// The authorityId satisifying this request
|
|
53
54
|
authorityId: string;
|
|
55
|
+
authoritySpec: AuthoritySpec;
|
|
54
56
|
|
|
55
|
-
// All the paths that have getParentPathStr(path) === cleanPath, and PathRouter.getRouteChildKey is within start and end. We suppress these, as our parent watch implicitly watches them. HOWEVER, We have to track them So we can re-watch them if the parent watch is removed.
|
|
56
57
|
suppressedWatches: Set<string>;
|
|
57
58
|
}[];
|
|
58
59
|
}>());
|
|
@@ -132,6 +133,18 @@ export class RemoteWatcher {
|
|
|
132
133
|
}
|
|
133
134
|
|
|
134
135
|
|
|
136
|
+
public getRemoteWatchParentRange(path: string) {
|
|
137
|
+
let parentPath = getParentPathStr(path);
|
|
138
|
+
let watchObj = this.remoteWatchParents2.get(parentPath);
|
|
139
|
+
if (!watchObj) return undefined;
|
|
140
|
+
// NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
|
|
141
|
+
return watchObj.ranges.find(x => {
|
|
142
|
+
let route = PathRouter.getRouteFull({ path, spec: x.authoritySpec });
|
|
143
|
+
return x.start <= route && route < x.end;
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
|
|
135
148
|
public debugIsWatchingPath(path: string) {
|
|
136
149
|
// HACK: If there is no read node... pretend we are watching it. Because... for now, this is a known
|
|
137
150
|
// and the code calling this function isn't looking for that specific issue.
|
|
@@ -286,7 +299,6 @@ export class RemoteWatcher {
|
|
|
286
299
|
}
|
|
287
300
|
continue;
|
|
288
301
|
}
|
|
289
|
-
|
|
290
302
|
}
|
|
291
303
|
|
|
292
304
|
let { nodes } = PathRouter.getChildReadNodes(cleanPath, {
|
|
@@ -324,9 +336,11 @@ export class RemoteWatcher {
|
|
|
324
336
|
// NOTE: Nodes has no internal overlaps, and missing ranges by definition are only values that we haven't added (and itself has no overlaps). And so adding the intersection of those will also have no overlaps and only add values that we haven't added.
|
|
325
337
|
for (let missing of missingRanges) {
|
|
326
338
|
for (let node of nodes) {
|
|
339
|
+
if (!rangesOverlap(node.range, missing)) continue;
|
|
340
|
+
|
|
327
341
|
let clippedStart = Math.max(node.range.start, missing.start);
|
|
328
342
|
let clippedEnd = Math.min(node.range.end, missing.end);
|
|
329
|
-
|
|
343
|
+
|
|
330
344
|
let finalPath = encodeParentFilter({ path: cleanPath, startFraction: clippedStart, endFraction: clippedEnd });
|
|
331
345
|
let existingRange = watchObj.ranges.find(x => x.finalPath === finalPath && x.authorityId === node.nodeId);
|
|
332
346
|
if (!existingRange) {
|
|
@@ -337,6 +351,7 @@ export class RemoteWatcher {
|
|
|
337
351
|
inputPaths: new Set(),
|
|
338
352
|
authorityId: node.nodeId,
|
|
339
353
|
suppressedWatches: new Set(),
|
|
354
|
+
authoritySpec: node.authoritySpec,
|
|
340
355
|
};
|
|
341
356
|
watchObj.ranges.push(existingRange);
|
|
342
357
|
let watchesPer = watchesPerAuthority.get(node.nodeId);
|
|
@@ -394,16 +409,13 @@ export class RemoteWatcher {
|
|
|
394
409
|
if (STOP_KEYS_DOUBLE_SENDS) {
|
|
395
410
|
// NOTE: We could already be watching the value watches, in which case watching the parent watches will cause us to double watch them. This is fine. We don't need to remove all double watches. We just want to remove some.
|
|
396
411
|
paths = paths.filter(path => {
|
|
397
|
-
let
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
coveringRange.suppressedWatches.add(path);
|
|
405
|
-
auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, remoteNodeId: authorityId });
|
|
406
|
-
return false;
|
|
412
|
+
let range = this.getRemoteWatchParentRange(path);
|
|
413
|
+
if (range) {
|
|
414
|
+
range.suppressedWatches.add(path);
|
|
415
|
+
auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, remoteNodeId: authorityId });
|
|
416
|
+
return false;
|
|
417
|
+
}
|
|
418
|
+
return true;
|
|
407
419
|
});
|
|
408
420
|
}
|
|
409
421
|
|
|
@@ -583,18 +595,15 @@ export class RemoteWatcher {
|
|
|
583
595
|
this.unwatchLatest(config);
|
|
584
596
|
}
|
|
585
597
|
|
|
598
|
+
public async flushWatchers() {
|
|
599
|
+
await this.watchUnwatchSerial("flushWatchers", async () => { });
|
|
600
|
+
}
|
|
601
|
+
|
|
586
602
|
@measureFnc
|
|
587
603
|
public getExistingWatchRemoteNodeId(path: string): string | undefined {
|
|
588
|
-
let
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
let route = PathRouter.getRouteChildKey(path);
|
|
592
|
-
let range = parentWatchObj.ranges.find(x => x.start <= route && route < x.end);
|
|
593
|
-
if (range) {
|
|
594
|
-
return range.authorityId;
|
|
595
|
-
}
|
|
596
|
-
// NOTE: I think it is a bug if we hit here. There shouldn't be a parent, but have us not match it?
|
|
597
|
-
// But... I also don't see the harm in checking remoteWatchPaths
|
|
604
|
+
let parentRange = this.getRemoteWatchParentRange(path);
|
|
605
|
+
if (parentRange) {
|
|
606
|
+
return parentRange.authorityId;
|
|
598
607
|
}
|
|
599
608
|
return this.remoteWatchPaths.get(path);
|
|
600
609
|
}
|
|
@@ -665,6 +674,18 @@ export class RemoteWatcher {
|
|
|
665
674
|
}
|
|
666
675
|
}
|
|
667
676
|
|
|
677
|
+
registerGetSpecForChildPath(path => {
|
|
678
|
+
if (!PathRouter.isSelfAuthority(path)) {
|
|
679
|
+
// Depends on the remote authority for this path
|
|
680
|
+
let remoteWatchRange = remoteWatcher.getRemoteWatchParentRange(path);
|
|
681
|
+
if (remoteWatchRange) return remoteWatchRange.authoritySpec;
|
|
682
|
+
|
|
683
|
+
// NOTE: This fall through usually means that we received a value before we started watching it, which is weird. That really shouldn't happen often. It'll probably break things. We will probably ignore this value thinking it has a different hash than it does.
|
|
684
|
+
console.warn(`We do not own a path, but also aren't watching it on a remote, but... receive the value? This is weird, and will probably break things (we will likely ignore this value / not trigger parent watches for it).`, { path });
|
|
685
|
+
}
|
|
686
|
+
return authorityLookup.getOurSpec();
|
|
687
|
+
});
|
|
688
|
+
|
|
668
689
|
export const remoteWatcher = new RemoteWatcher();
|
|
669
690
|
(globalThis as any).remoteWatcher = remoteWatcher;
|
|
670
691
|
void Promise.resolve().finally(() => {
|
|
@@ -285,10 +285,10 @@ export class Querysub {
|
|
|
285
285
|
*/
|
|
286
286
|
@measureFnc
|
|
287
287
|
public static async optionalStartupWait() {
|
|
288
|
+
await authorityLookup.startSyncing();
|
|
288
289
|
await measureBlock(async () => {
|
|
289
290
|
await waitForFirstTimeSync();
|
|
290
291
|
}, "waitForFirstTimeSync");
|
|
291
|
-
await authorityLookup.startSyncing();
|
|
292
292
|
}
|
|
293
293
|
|
|
294
294
|
public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
|
|
@@ -714,6 +714,10 @@ export class Querysub {
|
|
|
714
714
|
})();
|
|
715
715
|
}
|
|
716
716
|
|
|
717
|
+
private static socketFunctionInit() {
|
|
718
|
+
SocketFunction.addGlobalHook(() => authorityLookup.startSyncing());
|
|
719
|
+
}
|
|
720
|
+
|
|
717
721
|
private static hostCalled = false;
|
|
718
722
|
private static afterBootImports: string[] = [];
|
|
719
723
|
/** NOTE: Imports occur one at a time, after the "clientsideEntryPoint", with each import waiting for the previous to finish.
|
|
@@ -744,6 +748,7 @@ export class Querysub {
|
|
|
744
748
|
sourceCheck?: MachineSourceCheck<any>;
|
|
745
749
|
}) {
|
|
746
750
|
console.log(`Hosting server with config: ${JSON.stringify(config)}`);
|
|
751
|
+
this.socketFunctionInit();
|
|
747
752
|
|
|
748
753
|
if (isClient()) {
|
|
749
754
|
throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
|
|
@@ -1021,6 +1026,8 @@ export class Querysub {
|
|
|
1021
1026
|
RequireController.addMapGetModules(async (result, args) => {
|
|
1022
1027
|
let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
|
|
1023
1028
|
if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
|
|
1029
|
+
require("debugbreak")(2);
|
|
1030
|
+
debugger;
|
|
1024
1031
|
await isAllowedToSeeSource(configObj?.signedIdentity);
|
|
1025
1032
|
//console.log(red(`Not allowed to see source`));
|
|
1026
1033
|
for (let [key, value] of Object.entries(result.modules)) {
|
|
@@ -1038,6 +1045,7 @@ export class Querysub {
|
|
|
1038
1045
|
throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
|
|
1039
1046
|
}
|
|
1040
1047
|
|
|
1048
|
+
this.socketFunctionInit();
|
|
1041
1049
|
await SocketFunction.mount({
|
|
1042
1050
|
public: isPublic(),
|
|
1043
1051
|
port,
|
|
@@ -1062,10 +1070,14 @@ export class Querysub {
|
|
|
1062
1070
|
*/
|
|
1063
1071
|
endFraction?: number;
|
|
1064
1072
|
}): (keyof T)[] {
|
|
1065
|
-
|
|
1066
|
-
console.warn(`keys() is unlikely to work clientside. If we need to emulate how the server does the filtering, we should add an additional function to help with that. But there are issues with how query sub proxies watches, which make proxying partial key watches too difficult. ALSO, this is mostly for watching very large data streams with dynamic sharding (in FunctionRunner), and so shouldn't be needed clientside (the application can handle filtering at a schema level if it really wants something similar to this).`);
|
|
1067
|
-
}
|
|
1073
|
+
|
|
1068
1074
|
let { startFraction, endFraction } = config || {};
|
|
1075
|
+
if ((startFraction === undefined) !== (endFraction === undefined)) {
|
|
1076
|
+
throw new Error(`startFraction and endFraction must both be provided, or both be undefined. If you want to get all keys, don't provide startFraction and endFraction.`);
|
|
1077
|
+
}
|
|
1078
|
+
if (!isNode() && startFraction !== undefined && endFraction !== undefined) {
|
|
1079
|
+
console.warn(`keys() with a range restriction is not supported clientside. It's too complicated for the proxy to handle it because the hashing depends on the authority server. You can synchronize all the keys client side, but you can't synchronize a restriction of the keys.`);
|
|
1080
|
+
}
|
|
1069
1081
|
if (startFraction === undefined || endFraction === undefined) {
|
|
1070
1082
|
return Object.keys(obj);
|
|
1071
1083
|
}
|
|
@@ -1297,6 +1309,7 @@ setImmediate(async () => {
|
|
|
1297
1309
|
// Import, so it registers addStatPeriodic
|
|
1298
1310
|
await import("../5-diagnostics/nodeMetadata");
|
|
1299
1311
|
await import("../diagnostics/pathAuditer");
|
|
1312
|
+
await import("../diagnostics/NodeConnectionsPage");
|
|
1300
1313
|
});
|
|
1301
1314
|
|
|
1302
1315
|
setImmediate(async () => {
|
|
@@ -1312,7 +1325,6 @@ registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
|
|
|
1312
1325
|
|
|
1313
1326
|
import "../diagnostics/watchdog";
|
|
1314
1327
|
import "../diagnostics/trackResources";
|
|
1315
|
-
import "../diagnostics/benchmark";
|
|
1316
1328
|
import { formatNumber, formatTime } from "socket-function/src/formatting/format";
|
|
1317
1329
|
import { getCountPerPaint } from "../functional/onNextPaint";
|
|
1318
1330
|
import { addEpsilons } from "../bits";
|
|
@@ -2,7 +2,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
|
|
|
2
2
|
import { cache, lazy } from "socket-function/src/caching";
|
|
3
3
|
import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
|
|
4
4
|
import { FunctionMetadata } from "../3-path-functions/syncSchema";
|
|
5
|
-
import { RemoteWatcher } from "../1-path-client/RemoteWatcher";
|
|
5
|
+
import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
|
|
6
6
|
import { getProxyPath } from "../2-proxy/pathValueProxy";
|
|
7
7
|
import { atomic, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
|
|
8
8
|
import { CallSpec, DEPTH_TO_DATA, commitCall, debugCallSpec, functionSchema, getDevFunctionSpecFromCall } from "../3-path-functions/PathFunctionRunner";
|
|
@@ -200,6 +200,7 @@ writeCall.value = async (callSpec: CallSpec, metadata: FunctionMetadata) => {
|
|
|
200
200
|
});
|
|
201
201
|
|
|
202
202
|
if (result === "run") {
|
|
203
|
+
obj.onApplied();
|
|
203
204
|
// NOTE: I think awaiting here is fine, as the the prediction blockers don't wait based on the right call finishing, and I don't think anything really waits for this to finish.
|
|
204
205
|
await baseAddCall(callSpec);
|
|
205
206
|
} else {
|
|
@@ -533,6 +534,7 @@ export class QuerysubControllerBase {
|
|
|
533
534
|
pathWatcher.unwatchPath({ callback: callerId, ...config });
|
|
534
535
|
}
|
|
535
536
|
|
|
537
|
+
// NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
|
|
536
538
|
public async addCall(call: CallSpec) {
|
|
537
539
|
if (isBootstrapOnly()) throw new Error(`Cannot add calls to bootstrap only server`);
|
|
538
540
|
if (Querysub.DEBUG_CALLS) {
|
|
@@ -661,19 +663,28 @@ export class QuerysubControllerBase {
|
|
|
661
663
|
}
|
|
662
664
|
|
|
663
665
|
|
|
666
|
+
public async debugGetPathNodeIds(paths: string[]): Promise<Map<string, string | "missing">> {
|
|
667
|
+
await remoteWatcher.flushWatchers();
|
|
668
|
+
let result = new Map<string, string | "missing">();
|
|
669
|
+
for (let path of paths) {
|
|
670
|
+
let nodeId = remoteWatcher.getExistingWatchRemoteNodeId(path) ?? PathRouter.getReadyAuthority(path)?.nodeId ?? "missing";
|
|
671
|
+
result.set(path, nodeId);
|
|
672
|
+
}
|
|
673
|
+
return result;
|
|
674
|
+
}
|
|
675
|
+
|
|
664
676
|
// NOTE: Maybe remove this? It's very useful for debugging though...
|
|
665
677
|
public async debugGetSingleReadNode(path: string) {
|
|
666
678
|
return PathRouter.getReadyAuthority(path)?.nodeId;
|
|
667
679
|
}
|
|
668
680
|
|
|
669
|
-
public async
|
|
670
|
-
let authorities = await authorityLookup.getTopology();
|
|
671
|
-
return authorities.find(x => x.nodeId === nodeId)?.authoritySpec;
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
public async debugGetTopologyEntry(nodeId: string) {
|
|
681
|
+
public async debugGetNodeSpecs(): Promise<Map<string, { routeStart: number; routeEnd: number }>> {
|
|
675
682
|
let topology = await authorityLookup.getTopology();
|
|
676
|
-
|
|
683
|
+
let result = new Map<string, { routeStart: number; routeEnd: number }>();
|
|
684
|
+
for (let entry of topology) {
|
|
685
|
+
result.set(entry.nodeId, { routeStart: entry.authoritySpec.routeStart, routeEnd: entry.authoritySpec.routeEnd });
|
|
686
|
+
}
|
|
687
|
+
return result;
|
|
677
688
|
}
|
|
678
689
|
}
|
|
679
690
|
|
|
@@ -685,9 +696,9 @@ export const QuerysubController = SocketFunction.register(
|
|
|
685
696
|
watch: { compress: true, },
|
|
686
697
|
unwatch: { compress: true, },
|
|
687
698
|
addCall: { compress: true, },
|
|
699
|
+
debugGetPathNodeIds: {},
|
|
700
|
+
debugGetNodeSpecs: {},
|
|
688
701
|
debugGetSingleReadNode: {},
|
|
689
|
-
debugGetPathAuthorities: {},
|
|
690
|
-
debugGetTopologyEntry: {},
|
|
691
702
|
getModulePath: {},
|
|
692
703
|
getDevFunctionSpecFromCall: {},
|
|
693
704
|
}),
|
|
@@ -156,6 +156,9 @@ export async function runInPredictionQueue(config: {
|
|
|
156
156
|
|
|
157
157
|
// Resolve all the values before us as well, before even our own promise, and then wait on that promise. That way, everything finishes in the same order it starts.
|
|
158
158
|
if (index >= 0) {
|
|
159
|
+
if (index > 1) {
|
|
160
|
+
console.log(`Flushing ${index} other calls due to ${callSpec.FunctionId}`);
|
|
161
|
+
}
|
|
159
162
|
for (let i = 0; i <= index; i++) {
|
|
160
163
|
commitQueue[i].resolveOrderedPromise.resolve();
|
|
161
164
|
}
|
|
@@ -8,7 +8,6 @@ import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathF
|
|
|
8
8
|
import { logErrors } from "../errors";
|
|
9
9
|
import { getPathFromStr, getPathStr1 } from "../path";
|
|
10
10
|
import { Querysub, QuerysubController, QuerysubControllerBase, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
|
|
11
|
-
import { Benchmark } from "../diagnostics/benchmark";
|
|
12
11
|
import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
|
|
13
12
|
import { magenta, red } from "socket-function/src/formatting/logColors";
|
|
14
13
|
import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
|
|
@@ -21,6 +20,7 @@ import { isPublic } from "../config";
|
|
|
21
20
|
import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
|
|
22
21
|
import { cacheAsyncLimitedJSON } from "../functional/promiseCache";
|
|
23
22
|
import { addEpsilons } from "../bits";
|
|
23
|
+
import { delay } from "socket-function/src/batching";
|
|
24
24
|
setFlag(require, "cbor-x", "allowclient", true);
|
|
25
25
|
const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
|
|
26
26
|
|
|
@@ -90,13 +90,15 @@ const predictRunCommitLoop = (run: () => Promise<PredictResult | undefined>) =>
|
|
|
90
90
|
|
|
91
91
|
// IMPORTANT! This has to be synchronous, and we need to synchronously get into the proxy watcher call. That way other calls after this know to wait for our proxy watcher to finish. Otherwise, it's very, very, very easy to write code, where you call a function, you expect it to write to a value, and then you run some other code, maybe which just uses a commit async, and tries to read from that value. But the value won't exist, because we won't even have started the prediction yet, and the proxy watcher can't possibly know to wait, because we haven't started the prediction. So... this has to be synchronous!
|
|
92
92
|
export function predictCall(call: CallSpec, metadata: FunctionMetadata): {
|
|
93
|
+
onApplied: () => void;
|
|
93
94
|
cancel: () => void;
|
|
94
95
|
predictPromise: Promise<PredictResult | undefined>;
|
|
95
96
|
} {
|
|
96
97
|
let predictObj = predictCallBase({ call, metadata });
|
|
97
98
|
let cancel = predictObj.cancel;
|
|
99
|
+
let onApplied = predictObj.onApplied;
|
|
98
100
|
registerPredictionBlocker(call.CallId, predictObj.predictPromise);
|
|
99
|
-
return { cancel, predictPromise: predictObj.predictPromise };
|
|
101
|
+
return { cancel, onApplied, predictPromise: predictObj.predictPromise };
|
|
100
102
|
}
|
|
101
103
|
|
|
102
104
|
function getPredictTime(time: Time) {
|
|
@@ -115,12 +117,33 @@ function predictCallBase(config: {
|
|
|
115
117
|
metadata: FunctionMetadata;
|
|
116
118
|
overrides?: PathValue[];
|
|
117
119
|
}): {
|
|
120
|
+
onApplied: () => void;
|
|
118
121
|
cancel: () => void;
|
|
119
122
|
predictPromise: Promise<PredictResult | undefined>;
|
|
120
123
|
} {
|
|
121
124
|
let call = config.call;
|
|
122
125
|
let pathResultWrite = getCallResultPath(call);
|
|
123
|
-
|
|
126
|
+
|
|
127
|
+
let actualValueFinished = new PromiseObj();
|
|
128
|
+
function onActualFinished() {
|
|
129
|
+
if (actualValueFinished.resolveCalled) return;
|
|
130
|
+
let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
|
|
131
|
+
let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
|
|
132
|
+
if (!result) return;
|
|
133
|
+
if (result.lastInternalLoopCount === -1) return;
|
|
134
|
+
actualValueFinished.resolve();
|
|
135
|
+
for (let callback of onPredictionFinishedCallbacks) {
|
|
136
|
+
callback({ callId: call.CallId, result, functionId: call.FunctionId });
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
function onApplied() {
|
|
140
|
+
clientWatcher.setWatches({
|
|
141
|
+
callback: onActualFinished,
|
|
142
|
+
paths: new Set([pathResultWrite]),
|
|
143
|
+
parentPaths: new Set(),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
|
|
124
147
|
let functionResult: FunctionResult = {
|
|
125
148
|
// Special prediction values (we don't even really need to put anything here, but it
|
|
126
149
|
// might as well have the right format, in case a bug causes it to be used somewhere)
|
|
@@ -183,23 +206,6 @@ function predictCallBase(config: {
|
|
|
183
206
|
readParentPaths: Set<string>;
|
|
184
207
|
};
|
|
185
208
|
|
|
186
|
-
let actualValueFinished = new PromiseObj();
|
|
187
|
-
function onActualFinished() {
|
|
188
|
-
let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
|
|
189
|
-
let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
|
|
190
|
-
if (!result) return;
|
|
191
|
-
if (result.lastInternalLoopCount === -1) return;
|
|
192
|
-
actualValueFinished.resolve();
|
|
193
|
-
for (let callback of onPredictionFinishedCallbacks) {
|
|
194
|
-
callback({ callId: call.CallId, result, functionId: call.FunctionId });
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
clientWatcher.setWatches({
|
|
198
|
-
callback: onActualFinished,
|
|
199
|
-
paths: new Set([pathResultWrite]),
|
|
200
|
-
parentPaths: new Set(),
|
|
201
|
-
});
|
|
202
|
-
|
|
203
209
|
setTimeout(() => {
|
|
204
210
|
clientWatcher.unwatch(onActualFinished);
|
|
205
211
|
}, 30 * 1000);
|
|
@@ -360,6 +366,7 @@ function predictCallBase(config: {
|
|
|
360
366
|
return {
|
|
361
367
|
cancel: rejectPrediction,
|
|
362
368
|
predictPromise,
|
|
369
|
+
onApplied,
|
|
363
370
|
};
|
|
364
371
|
}
|
|
365
372
|
|
|
@@ -13,6 +13,7 @@ import { isNode } from "typesafecss";
|
|
|
13
13
|
import { devDebugbreak } from "../config";
|
|
14
14
|
import { addStatPeriodic, addStatSumPeriodic, addTimeProfileDistribution, onTimeProfile, logNodeStats, logNodeStateStats, registerNodeMetadata } from "../-0-hooks/hooks";
|
|
15
15
|
import { magenta } from "socket-function/src/formatting/logColors";
|
|
16
|
+
import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
let POLL_INTERVAL = timeInMinute * 5;
|
|
@@ -295,6 +296,16 @@ class NodeMetadataBase {
|
|
|
295
296
|
};
|
|
296
297
|
}));
|
|
297
298
|
}
|
|
299
|
+
|
|
300
|
+
public async debugGetPathAuthorities(nodeId: string) {
|
|
301
|
+
let authorities = await authorityLookup.getTopology();
|
|
302
|
+
return authorities.find(x => x.nodeId === nodeId)?.authoritySpec;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
public async debugGetTopologyEntry(nodeId: string) {
|
|
306
|
+
let topology = await authorityLookup.getTopology();
|
|
307
|
+
return topology.find(x => x.nodeId === nodeId);
|
|
308
|
+
}
|
|
298
309
|
}
|
|
299
310
|
export const NodeMetadataController = SocketFunction.register(
|
|
300
311
|
"NodeMetadataController-8be51a16-27c2-449e-b044-dcef6c3efcaf",
|
|
@@ -303,5 +314,11 @@ export const NodeMetadataController = SocketFunction.register(
|
|
|
303
314
|
getExtraMetadatas: {
|
|
304
315
|
hooks: [requiresNetworkTrustHook],
|
|
305
316
|
},
|
|
317
|
+
debugGetPathAuthorities: {
|
|
318
|
+
hooks: [requiresNetworkTrustHook],
|
|
319
|
+
},
|
|
320
|
+
debugGetTopologyEntry: {
|
|
321
|
+
hooks: [requiresNetworkTrustHook],
|
|
322
|
+
},
|
|
306
323
|
}),
|
|
307
324
|
);
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
2
|
+
import { qreact } from "../4-dom/qreact";
|
|
3
|
+
import { assertIsManagementUser } from "./managementPages";
|
|
4
|
+
import { css } from "typesafecss";
|
|
5
|
+
import { getSyncedController } from "../library-components/SyncedController";
|
|
6
|
+
import { getAllNodeIds, getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
|
|
7
|
+
import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
|
|
8
|
+
import { debugNodeId } from "../-c-identity/IdentityController";
|
|
9
|
+
import { fastHash } from "../misc/hash";
|
|
10
|
+
import { NodeCapabilitiesController } from "../-g-core-values/NodeCapabilities";
|
|
11
|
+
import { sort } from "socket-function/src/misc";
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
function getColorForNodeId(friendlyNodeId: string): string {
|
|
16
|
+
let hash = fastHash(friendlyNodeId);
|
|
17
|
+
let hue = hash % 360;
|
|
18
|
+
return `hsl(${hue}, 70%, 60%)`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export class NodeConnectionsPage extends qreact.Component {
|
|
22
|
+
render() {
|
|
23
|
+
let browserNode = getBrowserUrlNode();
|
|
24
|
+
let controller = nodeConnectionsController(browserNode);
|
|
25
|
+
|
|
26
|
+
let nodeIds = controller.getAllNodeIds();
|
|
27
|
+
if (!nodeIds) {
|
|
28
|
+
return <div class={css.pad2(10)}>Loading...</div>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
let connectableNodes: Array<{ nodeId: string; friendlyNodeId: string; entryPoint: string | undefined; connections: Array<{ nodeId: string; friendlyNodeId: string }> }> = [];
|
|
32
|
+
let nonConnectableNodes: string[] = [];
|
|
33
|
+
|
|
34
|
+
for (let nodeId of nodeIds) {
|
|
35
|
+
try {
|
|
36
|
+
let result = controller.getNodeConnections_forBrowser(nodeId);
|
|
37
|
+
if (result) {
|
|
38
|
+
let friendlyNodeId = debugNodeId(nodeId);
|
|
39
|
+
let entryPoint: string | undefined;
|
|
40
|
+
try {
|
|
41
|
+
entryPoint = controller.getEntryPoint_forBrowser(nodeId);
|
|
42
|
+
} catch (e) {
|
|
43
|
+
entryPoint = undefined;
|
|
44
|
+
}
|
|
45
|
+
connectableNodes.push({
|
|
46
|
+
nodeId,
|
|
47
|
+
friendlyNodeId,
|
|
48
|
+
entryPoint,
|
|
49
|
+
connections: result.connections,
|
|
50
|
+
});
|
|
51
|
+
}
|
|
52
|
+
} catch (e) {
|
|
53
|
+
nonConnectableNodes.push(nodeId);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return (
|
|
58
|
+
<div class={css.pad2(10).vbox(20).fillWidth}>
|
|
59
|
+
<div class={css.vbox(10).fillWidth}>
|
|
60
|
+
<h1>Connectable Nodes</h1>
|
|
61
|
+
{connectableNodes.map(node => {
|
|
62
|
+
let color = getColorForNodeId(node.friendlyNodeId);
|
|
63
|
+
return (
|
|
64
|
+
<div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
|
|
65
|
+
<div class={css.vbox(3)}>
|
|
66
|
+
{node.entryPoint && (
|
|
67
|
+
<div class={css.fontSize(14).fontWeight("bold")}>
|
|
68
|
+
{node.entryPoint}
|
|
69
|
+
</div>
|
|
70
|
+
)}
|
|
71
|
+
<div class={css.hbox(5)}>
|
|
72
|
+
<span class={css.background(color).pad2(4, 2).fontWeight("bold")}>
|
|
73
|
+
{node.friendlyNodeId}
|
|
74
|
+
</span>
|
|
75
|
+
<span class={css.color("gray").fontSize(12)}>
|
|
76
|
+
({node.nodeId})
|
|
77
|
+
</span>
|
|
78
|
+
</div>
|
|
79
|
+
</div>
|
|
80
|
+
<div class={css.vbox(3).paddingLeft(20)}>
|
|
81
|
+
{node.connections.length === 0 && (
|
|
82
|
+
<div class={css.color("gray")}>No connections</div>
|
|
83
|
+
)}
|
|
84
|
+
{node.connections.map(conn => {
|
|
85
|
+
let connColor = getColorForNodeId(conn.friendlyNodeId);
|
|
86
|
+
return (
|
|
87
|
+
<div class={css.hbox(5)}>
|
|
88
|
+
<span>→ Connected to:</span>
|
|
89
|
+
<span class={css.background(connColor).pad2(4, 2).fontWeight("bold")}>
|
|
90
|
+
{conn.friendlyNodeId}
|
|
91
|
+
</span>
|
|
92
|
+
<span class={css.color("gray").fontSize(12)}>
|
|
93
|
+
({conn.nodeId})
|
|
94
|
+
</span>
|
|
95
|
+
</div>
|
|
96
|
+
);
|
|
97
|
+
})}
|
|
98
|
+
</div>
|
|
99
|
+
</div>
|
|
100
|
+
);
|
|
101
|
+
})}
|
|
102
|
+
</div>
|
|
103
|
+
|
|
104
|
+
<div class={css.vbox(10).fillWidth}>
|
|
105
|
+
<h1>Non-Connectable Nodes</h1>
|
|
106
|
+
<div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
|
|
107
|
+
{nonConnectableNodes.map(nodeId => {
|
|
108
|
+
let friendlyNodeId = debugNodeId(nodeId);
|
|
109
|
+
return (
|
|
110
|
+
<div class={css.hbox(5)}>
|
|
111
|
+
<span class={css.background("lightgray").pad2(4, 2).fontWeight("bold")}>
|
|
112
|
+
{friendlyNodeId}
|
|
113
|
+
</span>
|
|
114
|
+
<span class={css.color("gray").fontSize(12)}>
|
|
115
|
+
({nodeId})
|
|
116
|
+
</span>
|
|
117
|
+
<span class={css.color("gray")}>- Can't connect</span>
|
|
118
|
+
</div>
|
|
119
|
+
);
|
|
120
|
+
})}
|
|
121
|
+
</div>
|
|
122
|
+
</div>
|
|
123
|
+
</div>
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
class NodeConnectionsControllerBase {
|
|
129
|
+
public async getNodeConnections() {
|
|
130
|
+
let factories = debugGetAllCallFactories();
|
|
131
|
+
let connections = factories.filter(x => x.isConnected).map(factory => ({
|
|
132
|
+
nodeId: factory.nodeId,
|
|
133
|
+
friendlyNodeId: debugNodeId(factory.nodeId),
|
|
134
|
+
}));
|
|
135
|
+
return { connections };
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
public async getNodeConnections_forBrowser(nodeId: string) {
|
|
139
|
+
let result = await NodeConnectionsController.nodes[nodeId].getNodeConnections();
|
|
140
|
+
sort(result.connections, x => x.friendlyNodeId);
|
|
141
|
+
return result;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
public async getEntryPoint_forBrowser(nodeId: string) {
|
|
145
|
+
return await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
public async getAllNodeIds() {
|
|
149
|
+
return await getAllNodeIds();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
export const NodeConnectionsController = SocketFunction.register(
|
|
154
|
+
"NodeConnectionsController-8f3e2a1b-4c5d-4f2e-9a3b-7c8d1e2f3a4b",
|
|
155
|
+
new NodeConnectionsControllerBase(),
|
|
156
|
+
() => ({
|
|
157
|
+
getNodeConnections: {},
|
|
158
|
+
getNodeConnections_forBrowser: {},
|
|
159
|
+
getEntryPoint_forBrowser: {},
|
|
160
|
+
getAllNodeIds: {},
|
|
161
|
+
}),
|
|
162
|
+
() => ({
|
|
163
|
+
hooks: [assertIsManagementUser],
|
|
164
|
+
})
|
|
165
|
+
);
|
|
166
|
+
|
|
167
|
+
export const nodeConnectionsController = getSyncedController(NodeConnectionsController);
|