querysub 0.403.0 → 0.404.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/.cursorrules +2 -0
  2. package/bin/audit-imports.js +4 -0
  3. package/package.json +7 -4
  4. package/spec.txt +77 -0
  5. package/src/-a-archives/archiveCache.ts +9 -4
  6. package/src/-a-archives/archivesBackBlaze.ts +1039 -1039
  7. package/src/-a-auth/certs.ts +0 -12
  8. package/src/-c-identity/IdentityController.ts +12 -3
  9. package/src/-f-node-discovery/NodeDiscovery.ts +32 -26
  10. package/src/-g-core-values/NodeCapabilities.ts +12 -2
  11. package/src/0-path-value-core/AuthorityLookup.ts +239 -0
  12. package/src/0-path-value-core/LockWatcher2.ts +150 -0
  13. package/src/0-path-value-core/PathRouter.ts +535 -0
  14. package/src/0-path-value-core/PathRouterRouteOverride.ts +72 -0
  15. package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +65 -0
  16. package/src/0-path-value-core/PathValueCommitter.ts +222 -488
  17. package/src/0-path-value-core/PathValueController.ts +277 -239
  18. package/src/0-path-value-core/PathWatcher.ts +534 -0
  19. package/src/0-path-value-core/ShardPrefixes.ts +31 -0
  20. package/src/0-path-value-core/ValidStateComputer.ts +303 -0
  21. package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +1 -1
  22. package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +80 -44
  23. package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +13 -16
  24. package/src/0-path-value-core/auditLogs.ts +2 -0
  25. package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +97 -0
  26. package/src/0-path-value-core/pathValueArchives.ts +490 -492
  27. package/src/0-path-value-core/pathValueCore.ts +195 -1496
  28. package/src/0-path-value-core/startupAuthority.ts +74 -0
  29. package/src/1-path-client/RemoteWatcher.ts +90 -82
  30. package/src/1-path-client/pathValueClientWatcher.ts +808 -815
  31. package/src/2-proxy/PathValueProxyWatcher.ts +10 -8
  32. package/src/2-proxy/archiveMoveHarness.ts +182 -214
  33. package/src/2-proxy/garbageCollection.ts +9 -8
  34. package/src/2-proxy/schema2.ts +21 -1
  35. package/src/3-path-functions/PathFunctionHelpers.ts +206 -180
  36. package/src/3-path-functions/PathFunctionRunner.ts +943 -766
  37. package/src/3-path-functions/PathFunctionRunnerMain.ts +5 -3
  38. package/src/3-path-functions/pathFunctionLoader.ts +2 -2
  39. package/src/3-path-functions/syncSchema.ts +592 -521
  40. package/src/4-deploy/deployFunctions.ts +19 -4
  41. package/src/4-deploy/deployGetFunctionsInner.ts +8 -2
  42. package/src/4-deploy/deployMain.ts +51 -68
  43. package/src/4-deploy/edgeClientWatcher.tsx +1 -1
  44. package/src/4-deploy/edgeNodes.ts +2 -2
  45. package/src/4-dom/qreact.tsx +2 -4
  46. package/src/4-dom/qreactTest.tsx +7 -13
  47. package/src/4-querysub/Querysub.ts +21 -8
  48. package/src/4-querysub/QuerysubController.ts +45 -29
  49. package/src/4-querysub/permissions.ts +2 -2
  50. package/src/4-querysub/querysubPrediction.ts +80 -70
  51. package/src/4-querysub/schemaHelpers.ts +5 -1
  52. package/src/5-diagnostics/GenericFormat.tsx +14 -9
  53. package/src/archiveapps/archiveGCEntry.tsx +9 -2
  54. package/src/archiveapps/archiveJoinEntry.ts +87 -84
  55. package/src/archiveapps/archiveMergeEntry.tsx +2 -0
  56. package/src/bits.ts +19 -0
  57. package/src/config.ts +21 -3
  58. package/src/config2.ts +23 -48
  59. package/src/deployManager/components/DeployPage.tsx +7 -3
  60. package/src/deployManager/machineSchema.ts +4 -1
  61. package/src/diagnostics/ActionsHistory.ts +3 -8
  62. package/src/diagnostics/AuditLogPage.tsx +2 -3
  63. package/src/diagnostics/FunctionCallInfo.tsx +141 -0
  64. package/src/diagnostics/FunctionCallInfoState.ts +162 -0
  65. package/src/diagnostics/MachineThreadInfo.tsx +1 -1
  66. package/src/diagnostics/NodeViewer.tsx +37 -48
  67. package/src/diagnostics/SyncTestPage.tsx +241 -0
  68. package/src/diagnostics/auditImportViolations.ts +185 -0
  69. package/src/diagnostics/listenOnDebugger.ts +3 -3
  70. package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +10 -4
  71. package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +2 -2
  72. package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +24 -22
  73. package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
  74. package/src/diagnostics/logs/diskLogGlobalContext.ts +1 -0
  75. package/src/diagnostics/logs/errorNotifications2/logWatcher.ts +1 -3
  76. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +34 -16
  77. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +4 -6
  78. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleInstanceTableView.tsx +36 -5
  79. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +19 -5
  80. package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +15 -7
  81. package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +28 -106
  82. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +2 -0
  83. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMisc.ts +0 -0
  84. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +18 -7
  85. package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +3 -0
  86. package/src/diagnostics/managementPages.tsx +10 -3
  87. package/src/diagnostics/misc-pages/ArchiveViewer.tsx +20 -26
  88. package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -4
  89. package/src/diagnostics/misc-pages/ComponentSyncStats.tsx +2 -2
  90. package/src/diagnostics/misc-pages/LocalWatchViewer.tsx +7 -9
  91. package/src/diagnostics/misc-pages/SnapshotViewer.tsx +23 -12
  92. package/src/diagnostics/misc-pages/archiveViewerShared.tsx +1 -1
  93. package/src/diagnostics/pathAuditer.ts +486 -0
  94. package/src/diagnostics/pathAuditerCallback.ts +20 -0
  95. package/src/diagnostics/watchdog.ts +8 -1
  96. package/src/library-components/URLParam.ts +1 -1
  97. package/src/misc/hash.ts +1 -0
  98. package/src/path.ts +21 -7
  99. package/src/server.ts +54 -47
  100. package/src/user-implementation/loginEmail.tsx +1 -1
  101. package/tempnotes.txt +67 -0
  102. package/test.ts +288 -95
  103. package/src/0-path-value-core/NodePathAuthorities.ts +0 -1057
  104. package/src/0-path-value-core/PathController.ts +0 -1
  105. package/src/5-diagnostics/diskValueAudit.ts +0 -218
  106. package/src/5-diagnostics/memoryValueAudit.ts +0 -438
  107. package/src/archiveapps/lockTest.ts +0 -127
@@ -1,43 +1,34 @@
1
- import preact from "preact"; import { qreact } from "../../src/4-dom/qreact";
1
+ import { qreact } from "../../src/4-dom/qreact";
2
2
  import { SocketFunction } from "socket-function/SocketFunction";
3
3
  import { css } from "typesafecss";
4
- import { getAllNodeIds, getBrowserUrlNode, getOwnNodeId, syncNodesNow } from "../../src/-f-node-discovery/NodeDiscovery";
4
+ import { getAllNodeIds, getBrowserUrlNode, syncNodesNow } from "../../src/-f-node-discovery/NodeDiscovery";
5
5
  import { errorToUndefined, logErrors, timeoutToError } from "../../src/errors";
6
- import { Querysub } from "../../src/4-querysub/QuerysubController";
6
+ import { Querysub, QuerysubController } from "../../src/4-querysub/QuerysubController";
7
7
  import { NodeCapabilitiesController, getControllerNodeIdList } from "../../src/-g-core-values/NodeCapabilities";
8
- import { AuthorityPath, debug_getAuthorityPaths, debug_isReadReady, nodePathAuthority } from "../../src/0-path-value-core/NodePathAuthorities";
9
8
  import { lazy } from "socket-function/src/caching";
10
9
  import { atomicObjectWrite } from "../../src/2-proxy/PathValueProxyWatcher";
11
- import { formatNumber } from "socket-function/src/formatting/format";
12
10
  import { Button } from "../../src/library-components/Button";
13
11
  import { URLParam } from "../../src/library-components/URLParam";
14
12
  import { InputLabelURL } from "../../src/library-components/InputLabel";
15
13
  import { ColumnType, ColumnsType, RowType, Table, TableType } from "../../src/5-diagnostics/Table";
16
- import { MaybePromise } from "socket-function/src/types";
17
- import { requiresNetworkTrustHook } from "../../src/-d-trust/NetworkTrust2";
18
- import { getDomain, isNoNetwork } from "../../src/config";
14
+ import { getDomain } from "../../src/config";
19
15
  import { NodeMetadataController } from "../../src/5-diagnostics/nodeMetadata";
20
- import { isDefined } from "../../src/misc";
21
- import { encodeFormattedSelector, errorMessage, formatValue, toSpaceCase } from "../../src/5-diagnostics/GenericFormat";
22
- import { ValueAuditController } from "../../src/5-diagnostics/memoryValueAudit";
16
+ import { errorMessage, formatValue, toSpaceCase } from "../../src/5-diagnostics/GenericFormat";
23
17
  import { getExternalIP } from "../../src/misc/networking";
24
18
  import dns from "dns";
25
- import { getNodeIdDomain, getNodeIdDomainMaybeUndefined, getNodeIdIP } from "socket-function/src/nodeCache";
26
- import ws from "ws";
27
- import https from "https";
28
- import debugbreak from "debugbreak";
19
+ import { getNodeIdDomainMaybeUndefined, getNodeIdIP } from "socket-function/src/nodeCache";
29
20
  import { forwardPort } from "socket-function/src/forwardPort";
30
21
  import { setRecord } from "../-b-authorities/dnsAuthority";
31
- import tls from "tls";
32
- import net from "net";
33
22
  import { ButtonSelector } from "../library-components/ButtonSelector";
34
- import { assertIsManagementUser, managementPageURL } from "./managementPages";
23
+ import { assertIsManagementUser } from "./managementPages";
35
24
  import { SocketRegistered } from "socket-function/SocketFunctionTypes";
36
25
  import { ATag } from "../library-components/ATag";
37
26
  import { getSyncedController } from "../library-components/SyncedController";
38
27
  import child_process from "child_process";
39
28
  import { getHTTPSKeyCert } from "../-e-certs/certAuthority";
40
29
  import { getLogViewerParams } from "./logs/IndexedLogs/LogViewerParams";
30
+ import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
31
+ import { AuthoritySpec } from "../0-path-value-core/PathRouter";
41
32
 
42
33
 
43
34
  type NodeData = {
@@ -46,10 +37,11 @@ type NodeData = {
46
37
  ip?: string;
47
38
  devToolsURL?: string;
48
39
 
49
- live_isReadReady?: number;
50
- live_authorityPaths?: AuthorityPath[];
40
+ live_isReadReady?: boolean;
41
+ live_authorityPaths?: AuthoritySpec;
51
42
  live_exposedControllers?: string[];
52
43
  live_entryPoint?: string;
44
+ live_trueTimeOffset?: number;
53
45
  apiError?: string;
54
46
  table?: {
55
47
  columns: ColumnsType;
@@ -86,16 +78,11 @@ export class NodeViewer extends qreact.Component {
86
78
  let time = Date.now();
87
79
  let data: NodeData = { nodeId };
88
80
  try {
89
- if (nodeId.includes("b3a757")) {
90
- debugger;
91
- }
92
81
  data.table = await controller.getMiscInfo(nodeId);
93
- if (nodeId.includes("b3a757")) {
94
- debugger;
95
- }
96
82
  data.live_isReadReady = await controller.live_isReadReady(nodeId);
97
83
  data.live_exposedControllers = await controller.getExposedControllers(nodeId);
98
84
  data.live_entryPoint = await controller.live_getEntryPoint(nodeId);
85
+ data.live_trueTimeOffset = await controller.live_getTrueTimeOffset(nodeId);
99
86
  data.live_authorityPaths = await controller.live_getAuthorityPaths(nodeId);
100
87
  data.ip = await controller.getNodeIP(nodeId);
101
88
  if (data.ip === ourExternalIP || data.ip === ourIP) {
@@ -110,6 +97,7 @@ export class NodeViewer extends qreact.Component {
110
97
  debugger;
111
98
  }
112
99
  } catch (e: any) {
100
+ if (e.stack.includes("Tried to call a different thread")) return;
113
101
  data.apiError = "Error: " + e.stack;
114
102
  if (nodeId.includes("b3a757")) {
115
103
  debugger;
@@ -187,7 +175,7 @@ export class NodeViewer extends qreact.Component {
187
175
  tables["PathValueServer"] = [];
188
176
  tables["Querysub"] = [];
189
177
  for (let datum of nodeDatas) {
190
- if (datum.live_authorityPaths?.length) {
178
+ if (datum.live_authorityPaths) {
191
179
  tables["PathValueServer"].push(datum);
192
180
  } else if (datum.live_exposedControllers?.includes("QuerysubController-6db5ef05-7563-4473-a440-2f64f03fe6ef")) {
193
181
  tables["Querysub"].push(datum);
@@ -199,7 +187,7 @@ export class NodeViewer extends qreact.Component {
199
187
  }
200
188
 
201
189
  let builtinGroups = {
202
- "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "% Profiled", "All Memory", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
190
+ "Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "% Profiled", "All Memory", "port", "threadId", "machineId", "apiError", "live_entryPoint", "live_trueTimeOffset"],
203
191
  "Performance": [
204
192
  "All Memory",
205
193
  "Heap", "Buffers",
@@ -266,8 +254,7 @@ export class NodeViewer extends qreact.Component {
266
254
  buttons: {
267
255
  formatter: () => {
268
256
  return <span class={css.hbox(6)}>
269
- <Button onClick={() => NodeViewerController.nodes[getBrowserUrlNode()].diskAuditNow(x.nodeId)}>Disk Audit</Button>
270
- <Button onClick={() => NodeViewerController.nodes[getBrowserUrlNode()].memoryAuditNow(x.nodeId)}>Memory Audit</Button>
257
+
271
258
  </span>;
272
259
  }
273
260
  },
@@ -306,6 +293,9 @@ export class NodeViewer extends qreact.Component {
306
293
  },
307
294
  ip: {},
308
295
  loadTime: { title: "Meta Time", formatter: "timeSpan" },
296
+ live_trueTimeOffset: {
297
+ title: "Time Offset", formatter: x => String(x)
298
+ },
309
299
  ...x.table?.columns,
310
300
  //capabilities: null,
311
301
  apiError: { formatter: "error" },
@@ -503,14 +493,19 @@ class NodeViewerControllerBase {
503
493
  }
504
494
 
505
495
  public async live_isReadReady(nodeId: string) {
506
- return debug_isReadReady(nodeId);
496
+ let topo = await QuerysubController.nodes[nodeId].debugGetTopologyEntry(nodeId);
497
+ return !!topo?.isReady;
507
498
  }
508
499
  public async live_getAuthorityPaths(nodeId: string) {
509
- return debug_getAuthorityPaths(nodeId);
500
+ let topo = await QuerysubController.nodes[nodeId].debugGetTopologyEntry(nodeId);
501
+ return topo?.authoritySpec;
510
502
  }
511
503
  public async live_getEntryPoint(nodeId: string) {
512
504
  return NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
513
505
  }
506
+ public async live_getTrueTimeOffset(nodeId: string) {
507
+ return NodeCapabilitiesController.nodes[nodeId].getTrueTimeOffset();
508
+ }
514
509
 
515
510
  public async getMiscInfo(nodeId: string): Promise<{
516
511
  columns: ColumnsType;
@@ -529,8 +524,8 @@ class NodeViewerControllerBase {
529
524
  }
530
525
  let promises = [
531
526
  wrapAddTableValue("paths|paths", {}, async () => {
532
- let live_authorityPaths = await debug_getAuthorityPaths(nodeId);
533
- return live_authorityPaths.map(x => JSON.stringify(x)).join(" | ");
527
+ let live_authorityPaths = (await QuerysubController.nodes[nodeId].debugGetPathAuthorities(nodeId));
528
+ return JSON.stringify(live_authorityPaths);
534
529
  }),
535
530
  wrapAddTableValue("paths|functions", {}, async () => {
536
531
  let live_functionRunnerShards = await NodeCapabilitiesController.nodes[nodeId].getFunctionRunnerShards();
@@ -554,13 +549,15 @@ class NodeViewerControllerBase {
554
549
  return nodeId.split(".").at(-3);
555
550
  }),
556
551
  wrapAddTableValue("miscAudits", { formatter: "varray:error" }, async () => {
557
- let live_authorityPaths = await debug_getAuthorityPaths(nodeId);
558
- let cached_authority = await nodePathAuthority.debug_getAuthority(nodeId);
559
- let live_isReadReady = await debug_isReadReady(nodeId);
552
+ let liveTopo = await QuerysubController.nodes[nodeId].debugGetTopologyEntry(nodeId);
553
+ let live_authorityPaths = liveTopo?.authoritySpec;
554
+ let obj = await authorityLookup.getTopology();
555
+ let cached_authority = await obj.find(x => x.nodeId === nodeId);
556
+ let live_isReadReady = liveTopo?.isReady;
560
557
  return [
561
558
  live_isReadReady && !cached_authority && `Authority is missing in cache`,
562
- JSON.stringify(live_authorityPaths) !== JSON.stringify(cached_authority?.authorityPaths) && `Paths are out of sync. Cache has ${JSON.stringify(cached_authority?.authorityPaths)} live has ${JSON.stringify(live_authorityPaths)}`,
563
- live_isReadReady !== cached_authority?.isReadReady && `isReadReady is out of sync. Cache has ${cached_authority?.isReadReady} live has ${live_isReadReady}`,
559
+ JSON.stringify(live_authorityPaths) !== JSON.stringify(cached_authority) && `Paths are out of sync. Cache has ${JSON.stringify(cached_authority)} live has ${JSON.stringify(live_authorityPaths)}`,
560
+ live_isReadReady !== cached_authority?.isReady && `isReadReady is out of sync. Cache has ${cached_authority?.isReady} live has ${live_isReadReady}`,
564
561
  ].filter(x => x);
565
562
  }),
566
563
  wrapAddTableValue("capabilities|capabilities", {}, async () => {
@@ -585,13 +582,6 @@ class NodeViewerControllerBase {
585
582
  row,
586
583
  };
587
584
  }
588
-
589
- public async diskAuditNow(nodeId: string) {
590
- await ValueAuditController.nodes[nodeId].diskAuditNow();
591
- }
592
- public async memoryAuditNow(nodeId: string) {
593
- await ValueAuditController.nodes[nodeId].memoryAuditNow();
594
- }
595
585
  }
596
586
 
597
587
  export const NodeViewerController = SocketFunction.register(
@@ -609,10 +599,9 @@ export const NodeViewerController = SocketFunction.register(
609
599
  live_isReadReady: {},
610
600
  live_getAuthorityPaths: {},
611
601
  live_getEntryPoint: {},
602
+ live_getTrueTimeOffset: {},
612
603
  getMiscInfo: {},
613
604
 
614
- diskAuditNow: {},
615
- memoryAuditNow: {},
616
605
  forceRefreshNodes: {},
617
606
  }),
618
607
  () => ({
@@ -0,0 +1,241 @@
1
+ import { SocketFunction } from "socket-function/SocketFunction";
2
+ import { qreact } from "../4-dom/qreact";
3
+ import { assertIsManagementUser } from "./managementPages";
4
+ import { getSyncedController } from "../library-components/SyncedController";
5
+ import { Querysub } from "../4-querysub/QuerysubController";
6
+ import { isRegisteredUserPERMISSIONS, isSuperUserPERMISSIONS } from "../user-implementation/userData";
7
+ import { css } from "typesafecss";
8
+ import { Button } from "../library-components/Button";
9
+ import { InputLabel } from "../library-components/InputLabel";
10
+ import { Input } from "../library-components/Input";
11
+ import { authorityStorage } from "../0-path-value-core/pathValueCore";
12
+ import { getAllNodeIds } from "../-f-node-discovery/NodeDiscovery";
13
+ import { MachineThreadInfoController } from "./MachineThreadInfo";
14
+ import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
15
+ import { sort } from "socket-function/src/misc";
16
+ import { PathAuditerController } from "./pathAuditer";
17
+ import { t } from "../2-proxy/schema2";
18
+
19
+
20
+ let { data, functions } = Querysub.createSchema<{
21
+ values: {
22
+ [key: string]: number;
23
+ };
24
+ }>()({
25
+ functions: {
26
+ setValue(key: string, value: number) {
27
+ data().values[key] = value;
28
+ },
29
+ incrementValue(key: string) {
30
+ data().values[key]++;
31
+ },
32
+ },
33
+ module,
34
+ moduleId: "syncText",
35
+ functionMetadata: {
36
+ setValue: {
37
+ keyOverride: {
38
+ getPrefix: data => data().values,
39
+ getKey: (key: string, _value: number) => key,
40
+ },
41
+ },
42
+ incrementValue: {
43
+ keyOverride: {
44
+ getPrefix: data => data().values,
45
+ getKey: (key: string) => key,
46
+ },
47
+ },
48
+ },
49
+ permissions: {
50
+ PERMISSIONS: isSuperUserPERMISSIONS
51
+ }
52
+ });
53
+
54
+
55
+ function getUniqueThreadName(thread: { entrypoint: string; nodeId: string }, allThreads: { entrypoint: string; nodeId: string }[]): string {
56
+ function getDisplayName(entrypoint: string): string {
57
+ return entrypoint.replaceAll("\\", "/").split("/").slice(-3).join("/");
58
+ }
59
+ let baseName = getDisplayName(thread.entrypoint);
60
+ let duplicates = allThreads.filter(t => getDisplayName(t.entrypoint) === baseName);
61
+ if (duplicates.length > 1) {
62
+ let index = duplicates.findIndex(t => t.nodeId === thread.nodeId);
63
+ return `${baseName} [${index + 1}]`;
64
+ }
65
+ return baseName;
66
+ }
67
+
68
+ export class SyncTestPage extends qreact.Component {
69
+ state = t.state({
70
+ auditRunning: t.lookup({
71
+ isRunning: t.boolean
72
+ })
73
+ });
74
+
75
+ async runAudit(nodeId: string) {
76
+ this.state.auditRunning[nodeId] = { isRunning: true };
77
+ try {
78
+ await PathAuditerController.nodes[SocketFunction.browserNodeId()].runAuditNow_forBrowser(nodeId);
79
+ } finally {
80
+ Querysub.commit(() => {
81
+ this.state.auditRunning[nodeId] = { isRunning: false };
82
+ });
83
+ }
84
+ }
85
+
86
+ render() {
87
+ let values = data().values;
88
+ let allNodes = MachineThreadInfoController(SocketFunction.browserNodeId()).getAllInfo();
89
+ let allThreads = Array.from(allNodes?.byThread.values() || []);
90
+ allThreads = allThreads.slice();
91
+ sort(allThreads, x => x.entrypoint);
92
+
93
+ return (
94
+ <div className={css.pad2(20).vbox(40)}>
95
+ <h1>Sync Test Page</h1>
96
+
97
+ <div className={css.vbox(10)}>
98
+ <div className={css.boldStyle}>Audit Controls</div>
99
+ <div className={css.hbox(10, 5).wrap}>
100
+ {allThreads.map(thread => {
101
+ let auditStatus = this.state.auditRunning[thread.nodeId];
102
+ let isRunning = auditStatus && "isRunning" in auditStatus && auditStatus.isRunning;
103
+ return (
104
+ <div className={css.hbox(5).hsl(0, 0, 95).pad2(5)} title={JSON.stringify(thread)}>
105
+ <div>{getUniqueThreadName(thread, allThreads)}</div>
106
+ <Button
107
+ onClick={() => this.runAudit(thread.nodeId)}
108
+ disabled={isRunning}
109
+ >
110
+ {isRunning && "Running..." || "Run Audit"}
111
+ </Button>
112
+ </div>
113
+ );
114
+ })}
115
+ </div>
116
+ </div>
117
+
118
+ <div className={css.vbox(20)}>
119
+ {Object.entries(values).map(([key, value]) => {
120
+ const valuePath = Querysub.getPath(() => data().values[key]);
121
+ if (!valuePath) return undefined;
122
+ const path = valuePath;
123
+ return (
124
+ <div className={css.vbox(0)}>
125
+ <div className={css.hbox(10)}>
126
+ <span class={css.boldStyle}>{key}</span>
127
+ <span>{value}</span>
128
+ <Button onClick={() => functions.incrementValue(key)}>Increment</Button>
129
+ <InputLabel value={value} number onChangeValue={value => functions.setValue(key, +value)} />
130
+ </div>
131
+ <div className={css.hbox(10, 2).wrap.marginLeft(20)}>
132
+ {allThreads.map(thread => {
133
+ let result;
134
+ let hasError = false;
135
+ try {
136
+ result = syncTestController(SocketFunction.browserNodeId()).getValueAndTime_forBrowser({
137
+ path: path,
138
+ nodeId: thread.nodeId
139
+ });
140
+ } catch {
141
+ hasError = true;
142
+ }
143
+ if (hasError) return undefined;
144
+ return (
145
+ <div className={css.hbox(5).hsl(0, 0, 100).pad2(2)} title={JSON.stringify(thread)}>
146
+ <div>{getUniqueThreadName(thread, allThreads)}</div>
147
+ <Button onClick={async () => {
148
+ await SyncTestController.nodes[SocketFunction.browserNodeId()].DEBUG_secretlyDeleteAllValuesWithPath_forBrowser({
149
+ path: path,
150
+ nodeId: thread.nodeId
151
+ });
152
+ }}>
153
+ Delete Value
154
+ </Button>
155
+ {hasError && <div>Error</div> || <>
156
+ <div>Value: {result?.value}</div>
157
+ <div>Time: {result?.time}</div>
158
+ </>}
159
+ </div>
160
+ );
161
+ })}
162
+ </div>
163
+ </div>
164
+ );
165
+ })}
166
+ </div>
167
+ <InputLabel
168
+ label="Insert new key"
169
+ onChangeValue={value => {
170
+ functions.setValue(value, 0);
171
+ }}
172
+ />
173
+ <Button onClick={() => {
174
+ syncTestController.refreshAll();
175
+ }}>
176
+ Refresh
177
+ </Button>
178
+ </div>
179
+ );
180
+ }
181
+ }
182
+
183
+
184
+ class SyncTestControllerBase {
185
+ async test() {
186
+ return Date.now();
187
+ }
188
+
189
+ public async DEBUG_secretlyDeleteAllValuesWithPath_forBrowser(config: {
190
+ path: string;
191
+ nodeId: string;
192
+ }) {
193
+ await SyncTestController.nodes[config.nodeId].DEBUG_secretlyDeleteAllValuesWithPath(config.path);
194
+ }
195
+
196
+ public async DEBUG_secretlyDeleteAllValuesWithPath(path: string) {
197
+ await authorityStorage.DEBUG_secretlyDeleteAllValuesWithPath(path);
198
+ }
199
+
200
+ public async getValueAndTime_forBrowser(config: {
201
+ path: string;
202
+ nodeId: string;
203
+ }) {
204
+ return await SyncTestController.nodes[config.nodeId].getValueAndTime(config.path);
205
+ }
206
+
207
+ public async getValueAndTime(path: string) {
208
+ let pathValue = authorityStorage.getValueAtTime(path);
209
+ let value = pathValueSerializer.getPathValue(pathValue);
210
+ let time = pathValue?.time.time;
211
+ return { value, time };
212
+ }
213
+ }
214
+
215
+ export const SyncTestController = SocketFunction.register(
216
+ "SyncTestController-019cc520-f449-710b-94df-80b7b3e0f787",
217
+ new SyncTestControllerBase(),
218
+ () => ({
219
+ test: {},
220
+ DEBUG_secretlyDeleteAllValuesWithPath_forBrowser: {},
221
+ DEBUG_secretlyDeleteAllValuesWithPath: {},
222
+ getValueAndTime_forBrowser: {},
223
+ getValueAndTime: {},
224
+ }),
225
+ () => ({
226
+ hooks: [assertIsManagementUser],
227
+ }),
228
+ {
229
+ // Auto expose, as we want our test functions to be exposed
230
+ //noAutoExpose: true,
231
+ }
232
+ );
233
+
234
+ let syncTestController = getSyncedController(SyncTestController, {
235
+ reads: {
236
+ getValueAndTime_forBrowser: ["value"],
237
+ },
238
+ writes: {
239
+ DEBUG_secretlyDeleteAllValuesWithPath_forBrowser: ["value"],
240
+ },
241
+ });
@@ -0,0 +1,185 @@
1
+ import * as typescript from "typescript";
2
+ import * as fs from "fs";
3
+ import * as path from "path";
4
+
5
+ interface Violation {
6
+ importPath: string;
7
+ importStage: number;
8
+ fileStage: number;
9
+ }
10
+
11
+ interface FileViolations {
12
+ filePath: string;
13
+ fileStage: number;
14
+ violations: Violation[];
15
+ }
16
+
17
+ function getStageNumber(filePath: string): number | undefined {
18
+ let parts = filePath.split(/[\\/]/);
19
+ for (let part of parts) {
20
+ let match = part.match(/^(\d+)-/);
21
+ if (match) {
22
+ return parseInt(match[1]);
23
+ }
24
+ }
25
+ return undefined;
26
+ }
27
+
28
+ function getAllTypeScriptFiles(dir: string): string[] {
29
+ let results: string[] = [];
30
+ let entries = fs.readdirSync(dir, { withFileTypes: true });
31
+
32
+ for (let entry of entries) {
33
+ let fullPath = path.join(dir, entry.name);
34
+ if (entry.isDirectory()) {
35
+ if (entry.name === "node_modules" || entry.name === ".git") {
36
+ continue;
37
+ }
38
+ results.push(...getAllTypeScriptFiles(fullPath));
39
+ } else if (entry.isFile() && (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx"))) {
40
+ results.push(fullPath);
41
+ }
42
+ }
43
+
44
+ return results;
45
+ }
46
+
47
+ function resolveImportPath(importPath: string, fromFile: string, rootDir: string): string | undefined {
48
+ if (!importPath.startsWith(".")) {
49
+ return undefined;
50
+ }
51
+
52
+ let fromDir = path.dirname(fromFile);
53
+ let resolved = path.resolve(fromDir, importPath);
54
+
55
+ for (let ext of [".ts", ".tsx", ".js", ".jsx"]) {
56
+ if (fs.existsSync(resolved + ext)) {
57
+ return path.relative(rootDir, resolved + ext);
58
+ }
59
+ }
60
+
61
+ if (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()) {
62
+ for (let indexFile of ["index.ts", "index.tsx", "index.js", "index.jsx"]) {
63
+ let indexPath = path.join(resolved, indexFile);
64
+ if (fs.existsSync(indexPath)) {
65
+ return path.relative(rootDir, indexPath);
66
+ }
67
+ }
68
+ }
69
+
70
+ return undefined;
71
+ }
72
+
73
+ function extractImports(filePath: string): string[] {
74
+ let content = fs.readFileSync(filePath, "utf-8");
75
+ let sourceFile = typescript.createSourceFile(
76
+ filePath,
77
+ content,
78
+ typescript.ScriptTarget.Latest,
79
+ true
80
+ );
81
+
82
+ let imports: string[] = [];
83
+
84
+ function visit(node: typescript.Node) {
85
+ if (typescript.isImportDeclaration(node)) {
86
+ let moduleSpecifier = node.moduleSpecifier;
87
+ if (typescript.isStringLiteral(moduleSpecifier)) {
88
+ imports.push(moduleSpecifier.text);
89
+ }
90
+ } else if (typescript.isExportDeclaration(node) && node.moduleSpecifier) {
91
+ if (typescript.isStringLiteral(node.moduleSpecifier)) {
92
+ imports.push(node.moduleSpecifier.text);
93
+ }
94
+ }
95
+ // NOTE: Dynamic imports (import("...")) are intentionally NOT included
96
+ // because they are lazy-loaded at runtime and should be allowed to
97
+ // import from higher stages
98
+
99
+ typescript.forEachChild(node, visit);
100
+ }
101
+
102
+ visit(sourceFile);
103
+ return imports;
104
+ }
105
+
106
+ function checkFile(filePath: string, rootDir: string): FileViolations | undefined {
107
+ let fileStage = getStageNumber(filePath);
108
+ if (fileStage === undefined) {
109
+ return undefined;
110
+ }
111
+
112
+ let imports = extractImports(filePath);
113
+ let violations: Violation[] = [];
114
+
115
+ for (let importPath of imports) {
116
+ let resolvedPath = resolveImportPath(importPath, filePath, rootDir);
117
+ if (!resolvedPath) {
118
+ continue;
119
+ }
120
+
121
+ let importStage = getStageNumber(resolvedPath);
122
+ if (importStage !== undefined && importStage > fileStage) {
123
+ violations.push({
124
+ importPath: resolvedPath,
125
+ importStage,
126
+ fileStage
127
+ });
128
+ }
129
+ }
130
+
131
+ if (violations.length > 0) {
132
+ return {
133
+ filePath,
134
+ fileStage,
135
+ violations
136
+ };
137
+ }
138
+
139
+ return undefined;
140
+ }
141
+
142
+ async function main() {
143
+ let auditRoot = ".";
144
+ let rootDir = path.resolve(auditRoot);
145
+
146
+ console.log("Auditing import violations...");
147
+ console.log(`Root directory: ${rootDir}\n`);
148
+
149
+ let allFiles = getAllTypeScriptFiles(rootDir);
150
+ let allViolations: FileViolations[] = [];
151
+
152
+ for (let file of allFiles) {
153
+ let violations = checkFile(file, rootDir);
154
+ if (violations) {
155
+ allViolations.push(violations);
156
+ }
157
+ }
158
+
159
+ if (allViolations.length === 0) {
160
+ console.log("✓ No violations found!");
161
+ return;
162
+ }
163
+
164
+ console.log("VIOLATIONS FOUND:\n");
165
+ console.log("=".repeat(80));
166
+
167
+ for (let fileViolation of allViolations) {
168
+ console.log(`\n${fileViolation.filePath} (stage ${fileViolation.fileStage}):`);
169
+ for (let violation of fileViolation.violations) {
170
+ console.log(` → imports ${violation.importPath} (stage ${violation.importStage})`);
171
+ }
172
+ }
173
+
174
+ console.log("\n" + "=".repeat(80));
175
+ console.log("\nSUMMARY:");
176
+ console.log(`Total files with violations: ${allViolations.length}`);
177
+ console.log(`Total violations: ${allViolations.reduce((sum, fv) => sum + fv.violations.length, 0)}`);
178
+
179
+ console.log("\nViolations per file:");
180
+ for (let fileViolation of allViolations) {
181
+ console.log(` ${fileViolation.filePath}: ${fileViolation.violations.length} violation${fileViolation.violations.length === 1 ? "" : "s"}`);
182
+ }
183
+ }
184
+
185
+ main().catch(console.error).finally(() => process.exit(0));
@@ -9,7 +9,7 @@ import { lazy } from "socket-function/src/caching";
9
9
 
10
10
  let devToolsUrl = "";
11
11
 
12
- const listenOnDebugger = lazy(async function listenOnDebugger() {
12
+ const listenOnDebugger = lazy(function listenOnDebugger() {
13
13
  if (devToolsUrl) return;
14
14
  if (!isNode()) return;
15
15
  // IMPORTANT! NEVER use the built-in port. This adds a tiny bit of security. Not much,
@@ -37,14 +37,14 @@ const listenOnDebugger = lazy(async function listenOnDebugger() {
37
37
  }
38
38
  }
39
39
  });
40
- export async function getDebuggerUrl() {
40
+ export function getDebuggerUrl() {
41
41
  // NOTE: Very unfortunate, as this means that if a developer is debugging a server for a bit,
42
42
  // the inspector is left open. It's kind of hard to tell when they are done though.
43
43
  // - This is still somewhat secure, as only localhost can connect, but... having local network
44
44
  // accessing turning into remote execution access isn't great (especially for developers,
45
45
  // who could then get pwned just by clicking on a link).
46
46
  // TODO: The security of this can be improved, see NodeViewer.tsx:getExternalInspectURL
47
- await listenOnDebugger();
47
+ listenOnDebugger();
48
48
  return devToolsUrl;
49
49
  }
50
50
  function doesProcessExist(pid: number) {