querysub 0.407.0 → 0.408.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/bin/audit-disk-values.js +7 -0
  2. package/package.json +4 -3
  3. package/src/-a-archives/archiveCache.ts +12 -9
  4. package/src/-a-auth/certs.ts +1 -1
  5. package/src/-c-identity/IdentityController.ts +9 -1
  6. package/src/-f-node-discovery/NodeDiscovery.ts +63 -8
  7. package/src/0-path-value-core/AuthorityLookup.ts +8 -3
  8. package/src/0-path-value-core/PathRouter.ts +109 -68
  9. package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +4 -2
  10. package/src/0-path-value-core/PathValueCommitter.ts +3 -1
  11. package/src/0-path-value-core/PathValueController.ts +75 -4
  12. package/src/0-path-value-core/PathWatcher.ts +39 -0
  13. package/src/0-path-value-core/ShardPrefixes.ts +2 -0
  14. package/src/0-path-value-core/ValidStateComputer.ts +20 -8
  15. package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +11 -29
  16. package/src/0-path-value-core/pathValueArchives.ts +16 -5
  17. package/src/0-path-value-core/pathValueCore.ts +43 -3
  18. package/src/1-path-client/RemoteWatcher.ts +46 -25
  19. package/src/4-querysub/Querysub.ts +17 -5
  20. package/src/4-querysub/QuerysubController.ts +21 -10
  21. package/src/4-querysub/predictionQueue.tsx +3 -0
  22. package/src/4-querysub/querysubPrediction.ts +27 -20
  23. package/src/5-diagnostics/nodeMetadata.ts +17 -0
  24. package/src/diagnostics/NodeConnectionsPage.tsx +167 -0
  25. package/src/diagnostics/NodeViewer.tsx +11 -15
  26. package/src/diagnostics/PathDistributionInfo.tsx +102 -0
  27. package/src/diagnostics/auditDiskValues.ts +221 -0
  28. package/src/diagnostics/auditDiskValuesEntry.ts +43 -0
  29. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +5 -1
  30. package/src/diagnostics/logs/TimeRangeSelector.tsx +3 -3
  31. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +2 -0
  32. package/src/diagnostics/managementPages.tsx +10 -1
  33. package/src/diagnostics/misc-pages/ArchiveViewer.tsx +3 -2
  34. package/src/diagnostics/pathAuditer.ts +21 -0
  35. package/tempnotes.txt +5 -44
  36. package/test.ts +13 -301
  37. package/src/diagnostics/benchmark.ts +0 -139
  38. package/src/diagnostics/runSaturationTest.ts +0 -416
  39. package/src/diagnostics/satSchema.ts +0 -64
  40. package/src/test/mongoSatTest.tsx +0 -55
  41. package/src/test/satTest.ts +0 -193
  42. package/src/test/test.tsx +0 -552
@@ -6,7 +6,7 @@ import { isNode, sort } from "socket-function/src/misc";
6
6
  import { measureFnc, measureBlock } from "socket-function/src/profiling/measure";
7
7
  import { isOwnNodeId } from "../-f-node-discovery/NodeDiscovery";
8
8
  import { PathValueController } from "../0-path-value-core/PathValueController";
9
- import { PathRouter } from "../0-path-value-core/PathRouter";
9
+ import { AuthoritySpec, PathRouter } from "../0-path-value-core/PathRouter";
10
10
  import { WatchConfig, authorityStorage } from "../0-path-value-core/pathValueCore";
11
11
  import { pathWatcher } from "../0-path-value-core/PathWatcher";
12
12
  import { ActionsHistory } from "../diagnostics/ActionsHistory";
@@ -19,8 +19,9 @@ import { isDevDebugbreak } from "../config";
19
19
  import { auditLog, isDebugLogEnabled } from "../0-path-value-core/auditLogs";
20
20
  import { debugNodeThread } from "../-c-identity/IdentityController";
21
21
  import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
22
- import { decodeParentFilter, encodeParentFilter } from "../0-path-value-core/hackedPackedPathParentFiltering";
22
+ import { decodeParentFilter, encodeParentFilter, registerGetSpecForChildPath } from "../0-path-value-core/hackedPackedPathParentFiltering";
23
23
  import { removeRange, rangesOverlap } from "../rangeMath";
24
+
24
25
  setImmediate(() => import("../4-querysub/Querysub"));
25
26
 
26
27
  // NOTE: In some cases this can half our the PathValues sent. AND, it can reduce our sync requests by N, and as those are uploads those are event more expensive. However, it also complicates other code, requiring not just checking if a value is synced, but also the parent path. We should try to make this stable, but worst case scenario, we can set this to false, and it will make our synchronization a lot easier to verify.
@@ -51,8 +52,8 @@ export class RemoteWatcher {
51
52
 
52
53
  // The authorityId satisifying this request
53
54
  authorityId: string;
55
+ authoritySpec: AuthoritySpec;
54
56
 
55
- // All the paths that have getParentPathStr(path) === cleanPath, and PathRouter.getRouteChildKey is within start and end. We suppress these, as our parent watch implicitly watches them. HOWEVER, We have to track them So we can re-watch them if the parent watch is removed.
56
57
  suppressedWatches: Set<string>;
57
58
  }[];
58
59
  }>());
@@ -132,6 +133,18 @@ export class RemoteWatcher {
132
133
  }
133
134
 
134
135
 
136
+ public getRemoteWatchParentRange(path: string) {
137
+ let parentPath = getParentPathStr(path);
138
+ let watchObj = this.remoteWatchParents2.get(parentPath);
139
+ if (!watchObj) return undefined;
140
+ // NOTE: PathRouter.getChildReadNodes PROMISES That all of the nodes will hash in the same way, and if they all hash in the same way, and they have different ranges, then every path will uniquely map to a single value.
141
+ return watchObj.ranges.find(x => {
142
+ let route = PathRouter.getRouteFull({ path, spec: x.authoritySpec });
143
+ return x.start <= route && route < x.end;
144
+ });
145
+ }
146
+
147
+
135
148
  public debugIsWatchingPath(path: string) {
136
149
  // HACK: If there is no read node... pretend we are watching it. Because... for now, this is a known
137
150
  // and the code calling this function isn't looking for that specific issue.
@@ -286,7 +299,6 @@ export class RemoteWatcher {
286
299
  }
287
300
  continue;
288
301
  }
289
-
290
302
  }
291
303
 
292
304
  let { nodes } = PathRouter.getChildReadNodes(cleanPath, {
@@ -324,9 +336,11 @@ export class RemoteWatcher {
324
336
  // NOTE: Nodes has no internal overlaps, and missing ranges by definition are only values that we haven't added (and itself has no overlaps). And so adding the intersection of those will also have no overlaps and only add values that we haven't added.
325
337
  for (let missing of missingRanges) {
326
338
  for (let node of nodes) {
339
+ if (!rangesOverlap(node.range, missing)) continue;
340
+
327
341
  let clippedStart = Math.max(node.range.start, missing.start);
328
342
  let clippedEnd = Math.min(node.range.end, missing.end);
329
- if (clippedEnd - clippedStart <= 0) continue;
343
+
330
344
  let finalPath = encodeParentFilter({ path: cleanPath, startFraction: clippedStart, endFraction: clippedEnd });
331
345
  let existingRange = watchObj.ranges.find(x => x.finalPath === finalPath && x.authorityId === node.nodeId);
332
346
  if (!existingRange) {
@@ -337,6 +351,7 @@ export class RemoteWatcher {
337
351
  inputPaths: new Set(),
338
352
  authorityId: node.nodeId,
339
353
  suppressedWatches: new Set(),
354
+ authoritySpec: node.authoritySpec,
340
355
  };
341
356
  watchObj.ranges.push(existingRange);
342
357
  let watchesPer = watchesPerAuthority.get(node.nodeId);
@@ -394,16 +409,13 @@ export class RemoteWatcher {
394
409
  if (STOP_KEYS_DOUBLE_SENDS) {
395
410
  // NOTE: We could already be watching the value watches, in which case watching the parent watches will cause us to double watch them. This is fine. We don't need to remove all double watches. We just want to remove some.
396
411
  paths = paths.filter(path => {
397
- let cleanParent = getParentPathStr(path);
398
- let watchObj = this.remoteWatchParents2.get(cleanParent);
399
- if (!watchObj) return true;
400
- let routeKey = PathRouter.getRouteChildKey(path);
401
- // If we're watching it from any authority, don't watch it now.
402
- let coveringRange = watchObj.ranges.find(x => x.start <= routeKey && routeKey < x.end);
403
- if (!coveringRange) return true;
404
- coveringRange.suppressedWatches.add(path);
405
- auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, remoteNodeId: authorityId });
406
- return false;
412
+ let range = this.getRemoteWatchParentRange(path);
413
+ if (range) {
414
+ range.suppressedWatches.add(path);
415
+ auditLog("remoteWatcher outer WATCH SUPPRESSED", { path, remoteNodeId: authorityId });
416
+ return false;
417
+ }
418
+ return true;
407
419
  });
408
420
  }
409
421
 
@@ -583,18 +595,15 @@ export class RemoteWatcher {
583
595
  this.unwatchLatest(config);
584
596
  }
585
597
 
598
+ public async flushWatchers() {
599
+ await this.watchUnwatchSerial("flushWatchers", async () => { });
600
+ }
601
+
586
602
  @measureFnc
587
603
  public getExistingWatchRemoteNodeId(path: string): string | undefined {
588
- let parentPath = getParentPathStr(path);
589
- let parentWatchObj = this.remoteWatchParents2.get(parentPath);
590
- if (parentWatchObj) {
591
- let route = PathRouter.getRouteChildKey(path);
592
- let range = parentWatchObj.ranges.find(x => x.start <= route && route < x.end);
593
- if (range) {
594
- return range.authorityId;
595
- }
596
- // NOTE: I think it is a bug if we hit here. There shouldn't be a parent, but have us not match it?
597
- // But... I also don't see the harm in checking remoteWatchPaths
604
+ let parentRange = this.getRemoteWatchParentRange(path);
605
+ if (parentRange) {
606
+ return parentRange.authorityId;
598
607
  }
599
608
  return this.remoteWatchPaths.get(path);
600
609
  }
@@ -665,6 +674,18 @@ export class RemoteWatcher {
665
674
  }
666
675
  }
667
676
 
677
+ registerGetSpecForChildPath(path => {
678
+ if (!PathRouter.isSelfAuthority(path)) {
679
+ // Depends on the remote authority for this path
680
+ let remoteWatchRange = remoteWatcher.getRemoteWatchParentRange(path);
681
+ if (remoteWatchRange) return remoteWatchRange.authoritySpec;
682
+
683
+ // NOTE: This fall through usually means that we received a value before we started watching it, which is weird. That really shouldn't happen often. It'll probably break things. We will probably ignore this value thinking it has a different hash than it does.
684
+ console.warn(`We do not own a path, but also aren't watching it on a remote, but... receive the value? This is weird, and will probably break things (we will likely ignore this value / not trigger parent watches for it).`, { path });
685
+ }
686
+ return authorityLookup.getOurSpec();
687
+ });
688
+
668
689
  export const remoteWatcher = new RemoteWatcher();
669
690
  (globalThis as any).remoteWatcher = remoteWatcher;
670
691
  void Promise.resolve().finally(() => {
@@ -285,10 +285,10 @@ export class Querysub {
285
285
  */
286
286
  @measureFnc
287
287
  public static async optionalStartupWait() {
288
+ await authorityLookup.startSyncing();
288
289
  await measureBlock(async () => {
289
290
  await waitForFirstTimeSync();
290
291
  }, "waitForFirstTimeSync");
291
- await authorityLookup.startSyncing();
292
292
  }
293
293
 
294
294
  public static createWatcher(watcher: (obj: SyncWatcher) => void, options?: Partial<WatcherOptions<unknown>>): {
@@ -714,6 +714,10 @@ export class Querysub {
714
714
  })();
715
715
  }
716
716
 
717
+ private static socketFunctionInit() {
718
+ SocketFunction.addGlobalHook(() => authorityLookup.startSyncing());
719
+ }
720
+
717
721
  private static hostCalled = false;
718
722
  private static afterBootImports: string[] = [];
719
723
  /** NOTE: Imports occur one at a time, after the "clientsideEntryPoint", with each import waiting for the previous to finish.
@@ -744,6 +748,7 @@ export class Querysub {
744
748
  sourceCheck?: MachineSourceCheck<any>;
745
749
  }) {
746
750
  console.log(`Hosting server with config: ${JSON.stringify(config)}`);
751
+ this.socketFunctionInit();
747
752
 
748
753
  if (isClient()) {
749
754
  throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
@@ -1021,6 +1026,8 @@ export class Querysub {
1021
1026
  RequireController.addMapGetModules(async (result, args) => {
1022
1027
  let configObj = args[2] as { signedIdentity: SignedIdentity | undefined } | undefined;
1023
1028
  if (!await isAllowedToSeeSource(configObj?.signedIdentity)) {
1029
+ require("debugbreak")(2);
1030
+ debugger;
1024
1031
  await isAllowedToSeeSource(configObj?.signedIdentity);
1025
1032
  //console.log(red(`Not allowed to see source`));
1026
1033
  for (let [key, value] of Object.entries(result.modules)) {
@@ -1038,6 +1045,7 @@ export class Querysub {
1038
1045
  throw new Error(`--client processes cannot host a service. Either stop passing --client and keep the process on the network and trusted, or stop calling hostServer and call Querysub.configRootDiscoveryLocation instead.`);
1039
1046
  }
1040
1047
 
1048
+ this.socketFunctionInit();
1041
1049
  await SocketFunction.mount({
1042
1050
  public: isPublic(),
1043
1051
  port,
@@ -1062,10 +1070,14 @@ export class Querysub {
1062
1070
  */
1063
1071
  endFraction?: number;
1064
1072
  }): (keyof T)[] {
1065
- if (!isNode()) {
1066
- console.warn(`keys() is unlikely to work clientside. If we need to emulate how the server does the filtering, we should add an additional function to help with that. But there are issues with how query sub proxies watches, which make proxying partial key watches too difficult. ALSO, this is mostly for watching very large data streams with dynamic sharding (in FunctionRunner), and so shouldn't be needed clientside (the application can handle filtering at a schema level if it really wants something similar to this).`);
1067
- }
1073
+
1068
1074
  let { startFraction, endFraction } = config || {};
1075
+ if ((startFraction === undefined) !== (endFraction === undefined)) {
1076
+ throw new Error(`startFraction and endFraction must both be provided, or both be undefined. If you want to get all keys, don't provide startFraction and endFraction.`);
1077
+ }
1078
+ if (!isNode() && startFraction !== undefined && endFraction !== undefined) {
1079
+ console.warn(`keys() with a range restriction is not supported clientside. It's too complicated for the proxy to handle it because the hashing depends on the authority server. You can synchronize all the keys client side, but you can't synchronize a restriction of the keys.`);
1080
+ }
1069
1081
  if (startFraction === undefined || endFraction === undefined) {
1070
1082
  return Object.keys(obj);
1071
1083
  }
@@ -1297,6 +1309,7 @@ setImmediate(async () => {
1297
1309
  // Import, so it registers addStatPeriodic
1298
1310
  await import("../5-diagnostics/nodeMetadata");
1299
1311
  await import("../diagnostics/pathAuditer");
1312
+ await import("../diagnostics/NodeConnectionsPage");
1300
1313
  });
1301
1314
 
1302
1315
  setImmediate(async () => {
@@ -1312,7 +1325,6 @@ registerGetCompressNetwork(() => Querysub.COMPRESS_NETWORK);
1312
1325
 
1313
1326
  import "../diagnostics/watchdog";
1314
1327
  import "../diagnostics/trackResources";
1315
- import "../diagnostics/benchmark";
1316
1328
  import { formatNumber, formatTime } from "socket-function/src/formatting/format";
1317
1329
  import { getCountPerPaint } from "../functional/onNextPaint";
1318
1330
  import { addEpsilons } from "../bits";
@@ -2,7 +2,7 @@ import { SocketFunction } from "socket-function/SocketFunction";
2
2
  import { cache, lazy } from "socket-function/src/caching";
3
3
  import { appendToPathStr, getPathDepth, getPathStr1, getPathStr3 } from "../path";
4
4
  import { FunctionMetadata } from "../3-path-functions/syncSchema";
5
- import { RemoteWatcher } from "../1-path-client/RemoteWatcher";
5
+ import { RemoteWatcher, remoteWatcher } from "../1-path-client/RemoteWatcher";
6
6
  import { getProxyPath } from "../2-proxy/pathValueProxy";
7
7
  import { atomic, proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
8
8
  import { CallSpec, DEPTH_TO_DATA, commitCall, debugCallSpec, functionSchema, getDevFunctionSpecFromCall } from "../3-path-functions/PathFunctionRunner";
@@ -200,6 +200,7 @@ writeCall.value = async (callSpec: CallSpec, metadata: FunctionMetadata) => {
200
200
  });
201
201
 
202
202
  if (result === "run") {
203
+ obj.onApplied();
203
204
  // NOTE: I think awaiting here is fine, as the the prediction blockers don't wait based on the right call finishing, and I don't think anything really waits for this to finish.
204
205
  await baseAddCall(callSpec);
205
206
  } else {
@@ -533,6 +534,7 @@ export class QuerysubControllerBase {
533
534
  pathWatcher.unwatchPath({ callback: callerId, ...config });
534
535
  }
535
536
 
537
+ // NOTE: Calls are going to be temporary and random. Any user can use any call ID, so technically you could clobber other users' call IDs, or your our. There wouldn't really be any benefit. Nothing would really happen if you do that, so I don't believe these need to be kept secret. I think if you know someone else's call ID you might be able to read that data, but also it's securely random, so you're not going to be able to guess the call ID.
536
538
  public async addCall(call: CallSpec) {
537
539
  if (isBootstrapOnly()) throw new Error(`Cannot add calls to bootstrap only server`);
538
540
  if (Querysub.DEBUG_CALLS) {
@@ -661,19 +663,28 @@ export class QuerysubControllerBase {
661
663
  }
662
664
 
663
665
 
666
+ public async debugGetPathNodeIds(paths: string[]): Promise<Map<string, string | "missing">> {
667
+ await remoteWatcher.flushWatchers();
668
+ let result = new Map<string, string | "missing">();
669
+ for (let path of paths) {
670
+ let nodeId = remoteWatcher.getExistingWatchRemoteNodeId(path) ?? PathRouter.getReadyAuthority(path)?.nodeId ?? "missing";
671
+ result.set(path, nodeId);
672
+ }
673
+ return result;
674
+ }
675
+
664
676
  // NOTE: Maybe remove this? It's very useful for debugging though...
665
677
  public async debugGetSingleReadNode(path: string) {
666
678
  return PathRouter.getReadyAuthority(path)?.nodeId;
667
679
  }
668
680
 
669
- public async debugGetPathAuthorities(nodeId: string) {
670
- let authorities = await authorityLookup.getTopology();
671
- return authorities.find(x => x.nodeId === nodeId)?.authoritySpec;
672
- }
673
-
674
- public async debugGetTopologyEntry(nodeId: string) {
681
+ public async debugGetNodeSpecs(): Promise<Map<string, { routeStart: number; routeEnd: number }>> {
675
682
  let topology = await authorityLookup.getTopology();
676
- return topology.find(x => x.nodeId === nodeId);
683
+ let result = new Map<string, { routeStart: number; routeEnd: number }>();
684
+ for (let entry of topology) {
685
+ result.set(entry.nodeId, { routeStart: entry.authoritySpec.routeStart, routeEnd: entry.authoritySpec.routeEnd });
686
+ }
687
+ return result;
677
688
  }
678
689
  }
679
690
 
@@ -685,9 +696,9 @@ export const QuerysubController = SocketFunction.register(
685
696
  watch: { compress: true, },
686
697
  unwatch: { compress: true, },
687
698
  addCall: { compress: true, },
699
+ debugGetPathNodeIds: {},
700
+ debugGetNodeSpecs: {},
688
701
  debugGetSingleReadNode: {},
689
- debugGetPathAuthorities: {},
690
- debugGetTopologyEntry: {},
691
702
  getModulePath: {},
692
703
  getDevFunctionSpecFromCall: {},
693
704
  }),
@@ -156,6 +156,9 @@ export async function runInPredictionQueue(config: {
156
156
 
157
157
  // Resolve all the values before us as well, before even our own promise, and then wait on that promise. That way, everything finishes in the same order it starts.
158
158
  if (index >= 0) {
159
+ if (index > 1) {
160
+ console.log(`Flushing ${index} other calls due to ${callSpec.FunctionId}`);
161
+ }
159
162
  for (let i = 0; i <= index; i++) {
160
163
  commitQueue[i].resolveOrderedPromise.resolve();
161
164
  }
@@ -8,7 +8,6 @@ import { getModuleFromConfig, setGitURLMapping } from "../3-path-functions/pathF
8
8
  import { logErrors } from "../errors";
9
9
  import { getPathFromStr, getPathStr1 } from "../path";
10
10
  import { Querysub, QuerysubController, QuerysubControllerBase, registerPredictionBlocker, querysubNodeId } from "./QuerysubController";
11
- import { Benchmark } from "../diagnostics/benchmark";
12
11
  import { parseArgs } from "../3-path-functions/PathFunctionHelpers";
13
12
  import { magenta, red } from "socket-function/src/formatting/logColors";
14
13
  import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
@@ -21,6 +20,7 @@ import { isPublic } from "../config";
21
20
  import { clientWatcher } from "../1-path-client/pathValueClientWatcher";
22
21
  import { cacheAsyncLimitedJSON } from "../functional/promiseCache";
23
22
  import { addEpsilons } from "../bits";
23
+ import { delay } from "socket-function/src/batching";
24
24
  setFlag(require, "cbor-x", "allowclient", true);
25
25
  const cborEncoder = lazy(() => new cbor.Encoder({ structuredClone: true }));
26
26
 
@@ -90,13 +90,15 @@ const predictRunCommitLoop = (run: () => Promise<PredictResult | undefined>) =>
90
90
 
91
91
  // IMPORTANT! This has to be synchronous, and we need to synchronously get into the proxy watcher call. That way other calls after this know to wait for our proxy watcher to finish. Otherwise, it's very, very, very easy to write code, where you call a function, you expect it to write to a value, and then you run some other code, maybe which just uses a commit async, and tries to read from that value. But the value won't exist, because we won't even have started the prediction yet, and the proxy watcher can't possibly know to wait, because we haven't started the prediction. So... this has to be synchronous!
92
92
  export function predictCall(call: CallSpec, metadata: FunctionMetadata): {
93
+ onApplied: () => void;
93
94
  cancel: () => void;
94
95
  predictPromise: Promise<PredictResult | undefined>;
95
96
  } {
96
97
  let predictObj = predictCallBase({ call, metadata });
97
98
  let cancel = predictObj.cancel;
99
+ let onApplied = predictObj.onApplied;
98
100
  registerPredictionBlocker(call.CallId, predictObj.predictPromise);
99
- return { cancel, predictPromise: predictObj.predictPromise };
101
+ return { cancel, onApplied, predictPromise: predictObj.predictPromise };
100
102
  }
101
103
 
102
104
  function getPredictTime(time: Time) {
@@ -115,12 +117,33 @@ function predictCallBase(config: {
115
117
  metadata: FunctionMetadata;
116
118
  overrides?: PathValue[];
117
119
  }): {
120
+ onApplied: () => void;
118
121
  cancel: () => void;
119
122
  predictPromise: Promise<PredictResult | undefined>;
120
123
  } {
121
124
  let call = config.call;
122
125
  let pathResultWrite = getCallResultPath(call);
123
- Benchmark.onStartPredictCall(pathResultWrite);
126
+
127
+ let actualValueFinished = new PromiseObj();
128
+ function onActualFinished() {
129
+ if (actualValueFinished.resolveCalled) return;
130
+ let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
131
+ let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
132
+ if (!result) return;
133
+ if (result.lastInternalLoopCount === -1) return;
134
+ actualValueFinished.resolve();
135
+ for (let callback of onPredictionFinishedCallbacks) {
136
+ callback({ callId: call.CallId, result, functionId: call.FunctionId });
137
+ }
138
+ }
139
+ function onApplied() {
140
+ clientWatcher.setWatches({
141
+ callback: onActualFinished,
142
+ paths: new Set([pathResultWrite]),
143
+ parentPaths: new Set(),
144
+ });
145
+ }
146
+
124
147
  let functionResult: FunctionResult = {
125
148
  // Special prediction values (we don't even really need to put anything here, but it
126
149
  // might as well have the right format, in case a bug causes it to be used somewhere)
@@ -183,23 +206,6 @@ function predictCallBase(config: {
183
206
  readParentPaths: Set<string>;
184
207
  };
185
208
 
186
- let actualValueFinished = new PromiseObj();
187
- function onActualFinished() {
188
- let resultObj = authorityStorage.getValueAtTime(pathResultWrite, undefined);
189
- let result = pathValueSerializer.getPathValue(resultObj) as FunctionResult | undefined;
190
- if (!result) return;
191
- if (result.lastInternalLoopCount === -1) return;
192
- actualValueFinished.resolve();
193
- for (let callback of onPredictionFinishedCallbacks) {
194
- callback({ callId: call.CallId, result, functionId: call.FunctionId });
195
- }
196
- }
197
- clientWatcher.setWatches({
198
- callback: onActualFinished,
199
- paths: new Set([pathResultWrite]),
200
- parentPaths: new Set(),
201
- });
202
-
203
209
  setTimeout(() => {
204
210
  clientWatcher.unwatch(onActualFinished);
205
211
  }, 30 * 1000);
@@ -360,6 +366,7 @@ function predictCallBase(config: {
360
366
  return {
361
367
  cancel: rejectPrediction,
362
368
  predictPromise,
369
+ onApplied,
363
370
  };
364
371
  }
365
372
 
@@ -13,6 +13,7 @@ import { isNode } from "typesafecss";
13
13
  import { devDebugbreak } from "../config";
14
14
  import { addStatPeriodic, addStatSumPeriodic, addTimeProfileDistribution, onTimeProfile, logNodeStats, logNodeStateStats, registerNodeMetadata } from "../-0-hooks/hooks";
15
15
  import { magenta } from "socket-function/src/formatting/logColors";
16
+ import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
16
17
 
17
18
 
18
19
  let POLL_INTERVAL = timeInMinute * 5;
@@ -295,6 +296,16 @@ class NodeMetadataBase {
295
296
  };
296
297
  }));
297
298
  }
299
+
300
+ public async debugGetPathAuthorities(nodeId: string) {
301
+ let authorities = await authorityLookup.getTopology();
302
+ return authorities.find(x => x.nodeId === nodeId)?.authoritySpec;
303
+ }
304
+
305
+ public async debugGetTopologyEntry(nodeId: string) {
306
+ let topology = await authorityLookup.getTopology();
307
+ return topology.find(x => x.nodeId === nodeId);
308
+ }
298
309
  }
299
310
  export const NodeMetadataController = SocketFunction.register(
300
311
  "NodeMetadataController-8be51a16-27c2-449e-b044-dcef6c3efcaf",
@@ -303,5 +314,11 @@ export const NodeMetadataController = SocketFunction.register(
303
314
  getExtraMetadatas: {
304
315
  hooks: [requiresNetworkTrustHook],
305
316
  },
317
+ debugGetPathAuthorities: {
318
+ hooks: [requiresNetworkTrustHook],
319
+ },
320
+ debugGetTopologyEntry: {
321
+ hooks: [requiresNetworkTrustHook],
322
+ },
306
323
  }),
307
324
  );
@@ -0,0 +1,167 @@
1
+ import { SocketFunction } from "socket-function/SocketFunction";
2
+ import { qreact } from "../4-dom/qreact";
3
+ import { assertIsManagementUser } from "./managementPages";
4
+ import { css } from "typesafecss";
5
+ import { getSyncedController } from "../library-components/SyncedController";
6
+ import { getAllNodeIds, getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
7
+ import { debugGetAllCallFactories } from "socket-function/src/nodeCache";
8
+ import { debugNodeId } from "../-c-identity/IdentityController";
9
+ import { fastHash } from "../misc/hash";
10
+ import { NodeCapabilitiesController } from "../-g-core-values/NodeCapabilities";
11
+ import { sort } from "socket-function/src/misc";
12
+
13
+
14
+
15
+ function getColorForNodeId(friendlyNodeId: string): string {
16
+ let hash = fastHash(friendlyNodeId);
17
+ let hue = hash % 360;
18
+ return `hsl(${hue}, 70%, 60%)`;
19
+ }
20
+
21
+ export class NodeConnectionsPage extends qreact.Component {
22
+ render() {
23
+ let browserNode = getBrowserUrlNode();
24
+ let controller = nodeConnectionsController(browserNode);
25
+
26
+ let nodeIds = controller.getAllNodeIds();
27
+ if (!nodeIds) {
28
+ return <div class={css.pad2(10)}>Loading...</div>;
29
+ }
30
+
31
+ let connectableNodes: Array<{ nodeId: string; friendlyNodeId: string; entryPoint: string | undefined; connections: Array<{ nodeId: string; friendlyNodeId: string }> }> = [];
32
+ let nonConnectableNodes: string[] = [];
33
+
34
+ for (let nodeId of nodeIds) {
35
+ try {
36
+ let result = controller.getNodeConnections_forBrowser(nodeId);
37
+ if (result) {
38
+ let friendlyNodeId = debugNodeId(nodeId);
39
+ let entryPoint: string | undefined;
40
+ try {
41
+ entryPoint = controller.getEntryPoint_forBrowser(nodeId);
42
+ } catch (e) {
43
+ entryPoint = undefined;
44
+ }
45
+ connectableNodes.push({
46
+ nodeId,
47
+ friendlyNodeId,
48
+ entryPoint,
49
+ connections: result.connections,
50
+ });
51
+ }
52
+ } catch (e) {
53
+ nonConnectableNodes.push(nodeId);
54
+ }
55
+ }
56
+
57
+ return (
58
+ <div class={css.pad2(10).vbox(20).fillWidth}>
59
+ <div class={css.vbox(10).fillWidth}>
60
+ <h1>Connectable Nodes</h1>
61
+ {connectableNodes.map(node => {
62
+ let color = getColorForNodeId(node.friendlyNodeId);
63
+ return (
64
+ <div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
65
+ <div class={css.vbox(3)}>
66
+ {node.entryPoint && (
67
+ <div class={css.fontSize(14).fontWeight("bold")}>
68
+ {node.entryPoint}
69
+ </div>
70
+ )}
71
+ <div class={css.hbox(5)}>
72
+ <span class={css.background(color).pad2(4, 2).fontWeight("bold")}>
73
+ {node.friendlyNodeId}
74
+ </span>
75
+ <span class={css.color("gray").fontSize(12)}>
76
+ ({node.nodeId})
77
+ </span>
78
+ </div>
79
+ </div>
80
+ <div class={css.vbox(3).paddingLeft(20)}>
81
+ {node.connections.length === 0 && (
82
+ <div class={css.color("gray")}>No connections</div>
83
+ )}
84
+ {node.connections.map(conn => {
85
+ let connColor = getColorForNodeId(conn.friendlyNodeId);
86
+ return (
87
+ <div class={css.hbox(5)}>
88
+ <span>→ Connected to:</span>
89
+ <span class={css.background(connColor).pad2(4, 2).fontWeight("bold")}>
90
+ {conn.friendlyNodeId}
91
+ </span>
92
+ <span class={css.color("gray").fontSize(12)}>
93
+ ({conn.nodeId})
94
+ </span>
95
+ </div>
96
+ );
97
+ })}
98
+ </div>
99
+ </div>
100
+ );
101
+ })}
102
+ </div>
103
+
104
+ <div class={css.vbox(10).fillWidth}>
105
+ <h1>Non-Connectable Nodes</h1>
106
+ <div class={css.vbox(5).pad2(10).maxHeight(600).overflowAuto.bord(1, { h: 0, s: 0, l: 80 })}>
107
+ {nonConnectableNodes.map(nodeId => {
108
+ let friendlyNodeId = debugNodeId(nodeId);
109
+ return (
110
+ <div class={css.hbox(5)}>
111
+ <span class={css.background("lightgray").pad2(4, 2).fontWeight("bold")}>
112
+ {friendlyNodeId}
113
+ </span>
114
+ <span class={css.color("gray").fontSize(12)}>
115
+ ({nodeId})
116
+ </span>
117
+ <span class={css.color("gray")}>- Can't connect</span>
118
+ </div>
119
+ );
120
+ })}
121
+ </div>
122
+ </div>
123
+ </div>
124
+ );
125
+ }
126
+ }
127
+
128
+ class NodeConnectionsControllerBase {
129
+ public async getNodeConnections() {
130
+ let factories = debugGetAllCallFactories();
131
+ let connections = factories.filter(x => x.isConnected).map(factory => ({
132
+ nodeId: factory.nodeId,
133
+ friendlyNodeId: debugNodeId(factory.nodeId),
134
+ }));
135
+ return { connections };
136
+ }
137
+
138
+ public async getNodeConnections_forBrowser(nodeId: string) {
139
+ let result = await NodeConnectionsController.nodes[nodeId].getNodeConnections();
140
+ sort(result.connections, x => x.friendlyNodeId);
141
+ return result;
142
+ }
143
+
144
+ public async getEntryPoint_forBrowser(nodeId: string) {
145
+ return await NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
146
+ }
147
+
148
+ public async getAllNodeIds() {
149
+ return await getAllNodeIds();
150
+ }
151
+ }
152
+
153
+ export const NodeConnectionsController = SocketFunction.register(
154
+ "NodeConnectionsController-8f3e2a1b-4c5d-4f2e-9a3b-7c8d1e2f3a4b",
155
+ new NodeConnectionsControllerBase(),
156
+ () => ({
157
+ getNodeConnections: {},
158
+ getNodeConnections_forBrowser: {},
159
+ getEntryPoint_forBrowser: {},
160
+ getAllNodeIds: {},
161
+ }),
162
+ () => ({
163
+ hooks: [assertIsManagementUser],
164
+ })
165
+ );
166
+
167
+ export const nodeConnectionsController = getSyncedController(NodeConnectionsController);