querysub 0.403.0 → 0.404.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +2 -0
- package/bin/audit-imports.js +4 -0
- package/package.json +7 -4
- package/spec.txt +77 -0
- package/src/-a-archives/archiveCache.ts +9 -4
- package/src/-a-archives/archivesBackBlaze.ts +1039 -1039
- package/src/-a-auth/certs.ts +0 -12
- package/src/-c-identity/IdentityController.ts +12 -3
- package/src/-f-node-discovery/NodeDiscovery.ts +32 -26
- package/src/-g-core-values/NodeCapabilities.ts +12 -2
- package/src/0-path-value-core/AuthorityLookup.ts +239 -0
- package/src/0-path-value-core/LockWatcher2.ts +150 -0
- package/src/0-path-value-core/PathRouter.ts +535 -0
- package/src/0-path-value-core/PathRouterRouteOverride.ts +72 -0
- package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +65 -0
- package/src/0-path-value-core/PathValueCommitter.ts +222 -488
- package/src/0-path-value-core/PathValueController.ts +277 -239
- package/src/0-path-value-core/PathWatcher.ts +534 -0
- package/src/0-path-value-core/ShardPrefixes.ts +31 -0
- package/src/0-path-value-core/ValidStateComputer.ts +303 -0
- package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +1 -1
- package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +80 -44
- package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +13 -16
- package/src/0-path-value-core/auditLogs.ts +2 -0
- package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +97 -0
- package/src/0-path-value-core/pathValueArchives.ts +490 -492
- package/src/0-path-value-core/pathValueCore.ts +195 -1496
- package/src/0-path-value-core/startupAuthority.ts +74 -0
- package/src/1-path-client/RemoteWatcher.ts +90 -82
- package/src/1-path-client/pathValueClientWatcher.ts +808 -815
- package/src/2-proxy/PathValueProxyWatcher.ts +10 -8
- package/src/2-proxy/archiveMoveHarness.ts +182 -214
- package/src/2-proxy/garbageCollection.ts +9 -8
- package/src/2-proxy/schema2.ts +21 -1
- package/src/3-path-functions/PathFunctionHelpers.ts +206 -180
- package/src/3-path-functions/PathFunctionRunner.ts +943 -766
- package/src/3-path-functions/PathFunctionRunnerMain.ts +5 -3
- package/src/3-path-functions/pathFunctionLoader.ts +2 -2
- package/src/3-path-functions/syncSchema.ts +592 -521
- package/src/4-deploy/deployFunctions.ts +19 -4
- package/src/4-deploy/deployGetFunctionsInner.ts +8 -2
- package/src/4-deploy/deployMain.ts +51 -68
- package/src/4-deploy/edgeClientWatcher.tsx +1 -1
- package/src/4-deploy/edgeNodes.ts +2 -2
- package/src/4-dom/qreact.tsx +2 -4
- package/src/4-dom/qreactTest.tsx +7 -13
- package/src/4-querysub/Querysub.ts +21 -8
- package/src/4-querysub/QuerysubController.ts +45 -29
- package/src/4-querysub/permissions.ts +2 -2
- package/src/4-querysub/querysubPrediction.ts +80 -70
- package/src/4-querysub/schemaHelpers.ts +5 -1
- package/src/5-diagnostics/GenericFormat.tsx +14 -9
- package/src/archiveapps/archiveGCEntry.tsx +9 -2
- package/src/archiveapps/archiveJoinEntry.ts +87 -84
- package/src/archiveapps/archiveMergeEntry.tsx +2 -0
- package/src/bits.ts +19 -0
- package/src/config.ts +21 -3
- package/src/config2.ts +23 -48
- package/src/deployManager/components/DeployPage.tsx +7 -3
- package/src/deployManager/machineSchema.ts +4 -1
- package/src/diagnostics/ActionsHistory.ts +3 -8
- package/src/diagnostics/AuditLogPage.tsx +2 -3
- package/src/diagnostics/FunctionCallInfo.tsx +141 -0
- package/src/diagnostics/FunctionCallInfoState.ts +162 -0
- package/src/diagnostics/MachineThreadInfo.tsx +1 -1
- package/src/diagnostics/NodeViewer.tsx +37 -48
- package/src/diagnostics/SyncTestPage.tsx +241 -0
- package/src/diagnostics/auditImportViolations.ts +185 -0
- package/src/diagnostics/listenOnDebugger.ts +3 -3
- package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +10 -4
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +2 -2
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +24 -22
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
- package/src/diagnostics/logs/diskLogGlobalContext.ts +1 -0
- package/src/diagnostics/logs/errorNotifications2/logWatcher.ts +1 -3
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +34 -16
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +4 -6
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleInstanceTableView.tsx +36 -5
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +19 -5
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +15 -7
- package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +28 -106
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +2 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMisc.ts +0 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +18 -7
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +3 -0
- package/src/diagnostics/managementPages.tsx +10 -3
- package/src/diagnostics/misc-pages/ArchiveViewer.tsx +20 -26
- package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -4
- package/src/diagnostics/misc-pages/ComponentSyncStats.tsx +2 -2
- package/src/diagnostics/misc-pages/LocalWatchViewer.tsx +7 -9
- package/src/diagnostics/misc-pages/SnapshotViewer.tsx +23 -12
- package/src/diagnostics/misc-pages/archiveViewerShared.tsx +1 -1
- package/src/diagnostics/pathAuditer.ts +486 -0
- package/src/diagnostics/pathAuditerCallback.ts +20 -0
- package/src/diagnostics/watchdog.ts +8 -1
- package/src/library-components/URLParam.ts +1 -1
- package/src/misc/hash.ts +1 -0
- package/src/path.ts +21 -7
- package/src/server.ts +54 -47
- package/src/user-implementation/loginEmail.tsx +1 -1
- package/tempnotes.txt +67 -0
- package/test.ts +288 -95
- package/src/0-path-value-core/NodePathAuthorities.ts +0 -1057
- package/src/0-path-value-core/PathController.ts +0 -1
- package/src/5-diagnostics/diskValueAudit.ts +0 -218
- package/src/5-diagnostics/memoryValueAudit.ts +0 -438
- package/src/archiveapps/lockTest.ts +0 -127
|
@@ -1,43 +1,34 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { qreact } from "../../src/4-dom/qreact";
|
|
2
2
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
3
3
|
import { css } from "typesafecss";
|
|
4
|
-
import { getAllNodeIds, getBrowserUrlNode,
|
|
4
|
+
import { getAllNodeIds, getBrowserUrlNode, syncNodesNow } from "../../src/-f-node-discovery/NodeDiscovery";
|
|
5
5
|
import { errorToUndefined, logErrors, timeoutToError } from "../../src/errors";
|
|
6
|
-
import { Querysub } from "../../src/4-querysub/QuerysubController";
|
|
6
|
+
import { Querysub, QuerysubController } from "../../src/4-querysub/QuerysubController";
|
|
7
7
|
import { NodeCapabilitiesController, getControllerNodeIdList } from "../../src/-g-core-values/NodeCapabilities";
|
|
8
|
-
import { AuthorityPath, debug_getAuthorityPaths, debug_isReadReady, nodePathAuthority } from "../../src/0-path-value-core/NodePathAuthorities";
|
|
9
8
|
import { lazy } from "socket-function/src/caching";
|
|
10
9
|
import { atomicObjectWrite } from "../../src/2-proxy/PathValueProxyWatcher";
|
|
11
|
-
import { formatNumber } from "socket-function/src/formatting/format";
|
|
12
10
|
import { Button } from "../../src/library-components/Button";
|
|
13
11
|
import { URLParam } from "../../src/library-components/URLParam";
|
|
14
12
|
import { InputLabelURL } from "../../src/library-components/InputLabel";
|
|
15
13
|
import { ColumnType, ColumnsType, RowType, Table, TableType } from "../../src/5-diagnostics/Table";
|
|
16
|
-
import {
|
|
17
|
-
import { requiresNetworkTrustHook } from "../../src/-d-trust/NetworkTrust2";
|
|
18
|
-
import { getDomain, isNoNetwork } from "../../src/config";
|
|
14
|
+
import { getDomain } from "../../src/config";
|
|
19
15
|
import { NodeMetadataController } from "../../src/5-diagnostics/nodeMetadata";
|
|
20
|
-
import {
|
|
21
|
-
import { encodeFormattedSelector, errorMessage, formatValue, toSpaceCase } from "../../src/5-diagnostics/GenericFormat";
|
|
22
|
-
import { ValueAuditController } from "../../src/5-diagnostics/memoryValueAudit";
|
|
16
|
+
import { errorMessage, formatValue, toSpaceCase } from "../../src/5-diagnostics/GenericFormat";
|
|
23
17
|
import { getExternalIP } from "../../src/misc/networking";
|
|
24
18
|
import dns from "dns";
|
|
25
|
-
import {
|
|
26
|
-
import ws from "ws";
|
|
27
|
-
import https from "https";
|
|
28
|
-
import debugbreak from "debugbreak";
|
|
19
|
+
import { getNodeIdDomainMaybeUndefined, getNodeIdIP } from "socket-function/src/nodeCache";
|
|
29
20
|
import { forwardPort } from "socket-function/src/forwardPort";
|
|
30
21
|
import { setRecord } from "../-b-authorities/dnsAuthority";
|
|
31
|
-
import tls from "tls";
|
|
32
|
-
import net from "net";
|
|
33
22
|
import { ButtonSelector } from "../library-components/ButtonSelector";
|
|
34
|
-
import { assertIsManagementUser
|
|
23
|
+
import { assertIsManagementUser } from "./managementPages";
|
|
35
24
|
import { SocketRegistered } from "socket-function/SocketFunctionTypes";
|
|
36
25
|
import { ATag } from "../library-components/ATag";
|
|
37
26
|
import { getSyncedController } from "../library-components/SyncedController";
|
|
38
27
|
import child_process from "child_process";
|
|
39
28
|
import { getHTTPSKeyCert } from "../-e-certs/certAuthority";
|
|
40
29
|
import { getLogViewerParams } from "./logs/IndexedLogs/LogViewerParams";
|
|
30
|
+
import { authorityLookup } from "../0-path-value-core/AuthorityLookup";
|
|
31
|
+
import { AuthoritySpec } from "../0-path-value-core/PathRouter";
|
|
41
32
|
|
|
42
33
|
|
|
43
34
|
type NodeData = {
|
|
@@ -46,10 +37,11 @@ type NodeData = {
|
|
|
46
37
|
ip?: string;
|
|
47
38
|
devToolsURL?: string;
|
|
48
39
|
|
|
49
|
-
live_isReadReady?:
|
|
50
|
-
live_authorityPaths?:
|
|
40
|
+
live_isReadReady?: boolean;
|
|
41
|
+
live_authorityPaths?: AuthoritySpec;
|
|
51
42
|
live_exposedControllers?: string[];
|
|
52
43
|
live_entryPoint?: string;
|
|
44
|
+
live_trueTimeOffset?: number;
|
|
53
45
|
apiError?: string;
|
|
54
46
|
table?: {
|
|
55
47
|
columns: ColumnsType;
|
|
@@ -86,16 +78,11 @@ export class NodeViewer extends qreact.Component {
|
|
|
86
78
|
let time = Date.now();
|
|
87
79
|
let data: NodeData = { nodeId };
|
|
88
80
|
try {
|
|
89
|
-
if (nodeId.includes("b3a757")) {
|
|
90
|
-
debugger;
|
|
91
|
-
}
|
|
92
81
|
data.table = await controller.getMiscInfo(nodeId);
|
|
93
|
-
if (nodeId.includes("b3a757")) {
|
|
94
|
-
debugger;
|
|
95
|
-
}
|
|
96
82
|
data.live_isReadReady = await controller.live_isReadReady(nodeId);
|
|
97
83
|
data.live_exposedControllers = await controller.getExposedControllers(nodeId);
|
|
98
84
|
data.live_entryPoint = await controller.live_getEntryPoint(nodeId);
|
|
85
|
+
data.live_trueTimeOffset = await controller.live_getTrueTimeOffset(nodeId);
|
|
99
86
|
data.live_authorityPaths = await controller.live_getAuthorityPaths(nodeId);
|
|
100
87
|
data.ip = await controller.getNodeIP(nodeId);
|
|
101
88
|
if (data.ip === ourExternalIP || data.ip === ourIP) {
|
|
@@ -110,6 +97,7 @@ export class NodeViewer extends qreact.Component {
|
|
|
110
97
|
debugger;
|
|
111
98
|
}
|
|
112
99
|
} catch (e: any) {
|
|
100
|
+
if (e.stack.includes("Tried to call a different thread")) return;
|
|
113
101
|
data.apiError = "Error: " + e.stack;
|
|
114
102
|
if (nodeId.includes("b3a757")) {
|
|
115
103
|
debugger;
|
|
@@ -187,7 +175,7 @@ export class NodeViewer extends qreact.Component {
|
|
|
187
175
|
tables["PathValueServer"] = [];
|
|
188
176
|
tables["Querysub"] = [];
|
|
189
177
|
for (let datum of nodeDatas) {
|
|
190
|
-
if (datum.live_authorityPaths
|
|
178
|
+
if (datum.live_authorityPaths) {
|
|
191
179
|
tables["PathValueServer"].push(datum);
|
|
192
180
|
} else if (datum.live_exposedControllers?.includes("QuerysubController-6db5ef05-7563-4473-a440-2f64f03fe6ef")) {
|
|
193
181
|
tables["Querysub"].push(datum);
|
|
@@ -199,7 +187,7 @@ export class NodeViewer extends qreact.Component {
|
|
|
199
187
|
}
|
|
200
188
|
|
|
201
189
|
let builtinGroups = {
|
|
202
|
-
"Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "% Profiled", "All Memory", "port", "threadId", "machineId", "apiError", "live_entryPoint"],
|
|
190
|
+
"Default": ["buttons", "devToolsURL", "nodeId", "ip", "uptime", "loadTime", "% Profiled", "All Memory", "port", "threadId", "machineId", "apiError", "live_entryPoint", "live_trueTimeOffset"],
|
|
203
191
|
"Performance": [
|
|
204
192
|
"All Memory",
|
|
205
193
|
"Heap", "Buffers",
|
|
@@ -266,8 +254,7 @@ export class NodeViewer extends qreact.Component {
|
|
|
266
254
|
buttons: {
|
|
267
255
|
formatter: () => {
|
|
268
256
|
return <span class={css.hbox(6)}>
|
|
269
|
-
|
|
270
|
-
<Button onClick={() => NodeViewerController.nodes[getBrowserUrlNode()].memoryAuditNow(x.nodeId)}>Memory Audit</Button>
|
|
257
|
+
|
|
271
258
|
</span>;
|
|
272
259
|
}
|
|
273
260
|
},
|
|
@@ -306,6 +293,9 @@ export class NodeViewer extends qreact.Component {
|
|
|
306
293
|
},
|
|
307
294
|
ip: {},
|
|
308
295
|
loadTime: { title: "Meta Time", formatter: "timeSpan" },
|
|
296
|
+
live_trueTimeOffset: {
|
|
297
|
+
title: "Time Offset", formatter: x => String(x)
|
|
298
|
+
},
|
|
309
299
|
...x.table?.columns,
|
|
310
300
|
//capabilities: null,
|
|
311
301
|
apiError: { formatter: "error" },
|
|
@@ -503,14 +493,19 @@ class NodeViewerControllerBase {
|
|
|
503
493
|
}
|
|
504
494
|
|
|
505
495
|
public async live_isReadReady(nodeId: string) {
|
|
506
|
-
|
|
496
|
+
let topo = await QuerysubController.nodes[nodeId].debugGetTopologyEntry(nodeId);
|
|
497
|
+
return !!topo?.isReady;
|
|
507
498
|
}
|
|
508
499
|
public async live_getAuthorityPaths(nodeId: string) {
|
|
509
|
-
|
|
500
|
+
let topo = await QuerysubController.nodes[nodeId].debugGetTopologyEntry(nodeId);
|
|
501
|
+
return topo?.authoritySpec;
|
|
510
502
|
}
|
|
511
503
|
public async live_getEntryPoint(nodeId: string) {
|
|
512
504
|
return NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
|
|
513
505
|
}
|
|
506
|
+
public async live_getTrueTimeOffset(nodeId: string) {
|
|
507
|
+
return NodeCapabilitiesController.nodes[nodeId].getTrueTimeOffset();
|
|
508
|
+
}
|
|
514
509
|
|
|
515
510
|
public async getMiscInfo(nodeId: string): Promise<{
|
|
516
511
|
columns: ColumnsType;
|
|
@@ -529,8 +524,8 @@ class NodeViewerControllerBase {
|
|
|
529
524
|
}
|
|
530
525
|
let promises = [
|
|
531
526
|
wrapAddTableValue("paths|paths", {}, async () => {
|
|
532
|
-
let live_authorityPaths = await
|
|
533
|
-
return
|
|
527
|
+
let live_authorityPaths = (await QuerysubController.nodes[nodeId].debugGetPathAuthorities(nodeId));
|
|
528
|
+
return JSON.stringify(live_authorityPaths);
|
|
534
529
|
}),
|
|
535
530
|
wrapAddTableValue("paths|functions", {}, async () => {
|
|
536
531
|
let live_functionRunnerShards = await NodeCapabilitiesController.nodes[nodeId].getFunctionRunnerShards();
|
|
@@ -554,13 +549,15 @@ class NodeViewerControllerBase {
|
|
|
554
549
|
return nodeId.split(".").at(-3);
|
|
555
550
|
}),
|
|
556
551
|
wrapAddTableValue("miscAudits", { formatter: "varray:error" }, async () => {
|
|
557
|
-
let
|
|
558
|
-
let
|
|
559
|
-
let
|
|
552
|
+
let liveTopo = await QuerysubController.nodes[nodeId].debugGetTopologyEntry(nodeId);
|
|
553
|
+
let live_authorityPaths = liveTopo?.authoritySpec;
|
|
554
|
+
let obj = await authorityLookup.getTopology();
|
|
555
|
+
let cached_authority = await obj.find(x => x.nodeId === nodeId);
|
|
556
|
+
let live_isReadReady = liveTopo?.isReady;
|
|
560
557
|
return [
|
|
561
558
|
live_isReadReady && !cached_authority && `Authority is missing in cache`,
|
|
562
|
-
JSON.stringify(live_authorityPaths) !== JSON.stringify(cached_authority
|
|
563
|
-
live_isReadReady !== cached_authority?.
|
|
559
|
+
JSON.stringify(live_authorityPaths) !== JSON.stringify(cached_authority) && `Paths are out of sync. Cache has ${JSON.stringify(cached_authority)} live has ${JSON.stringify(live_authorityPaths)}`,
|
|
560
|
+
live_isReadReady !== cached_authority?.isReady && `isReadReady is out of sync. Cache has ${cached_authority?.isReady} live has ${live_isReadReady}`,
|
|
564
561
|
].filter(x => x);
|
|
565
562
|
}),
|
|
566
563
|
wrapAddTableValue("capabilities|capabilities", {}, async () => {
|
|
@@ -585,13 +582,6 @@ class NodeViewerControllerBase {
|
|
|
585
582
|
row,
|
|
586
583
|
};
|
|
587
584
|
}
|
|
588
|
-
|
|
589
|
-
public async diskAuditNow(nodeId: string) {
|
|
590
|
-
await ValueAuditController.nodes[nodeId].diskAuditNow();
|
|
591
|
-
}
|
|
592
|
-
public async memoryAuditNow(nodeId: string) {
|
|
593
|
-
await ValueAuditController.nodes[nodeId].memoryAuditNow();
|
|
594
|
-
}
|
|
595
585
|
}
|
|
596
586
|
|
|
597
587
|
export const NodeViewerController = SocketFunction.register(
|
|
@@ -609,10 +599,9 @@ export const NodeViewerController = SocketFunction.register(
|
|
|
609
599
|
live_isReadReady: {},
|
|
610
600
|
live_getAuthorityPaths: {},
|
|
611
601
|
live_getEntryPoint: {},
|
|
602
|
+
live_getTrueTimeOffset: {},
|
|
612
603
|
getMiscInfo: {},
|
|
613
604
|
|
|
614
|
-
diskAuditNow: {},
|
|
615
|
-
memoryAuditNow: {},
|
|
616
605
|
forceRefreshNodes: {},
|
|
617
606
|
}),
|
|
618
607
|
() => ({
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
import { SocketFunction } from "socket-function/SocketFunction";
|
|
2
|
+
import { qreact } from "../4-dom/qreact";
|
|
3
|
+
import { assertIsManagementUser } from "./managementPages";
|
|
4
|
+
import { getSyncedController } from "../library-components/SyncedController";
|
|
5
|
+
import { Querysub } from "../4-querysub/QuerysubController";
|
|
6
|
+
import { isRegisteredUserPERMISSIONS, isSuperUserPERMISSIONS } from "../user-implementation/userData";
|
|
7
|
+
import { css } from "typesafecss";
|
|
8
|
+
import { Button } from "../library-components/Button";
|
|
9
|
+
import { InputLabel } from "../library-components/InputLabel";
|
|
10
|
+
import { Input } from "../library-components/Input";
|
|
11
|
+
import { authorityStorage } from "../0-path-value-core/pathValueCore";
|
|
12
|
+
import { getAllNodeIds } from "../-f-node-discovery/NodeDiscovery";
|
|
13
|
+
import { MachineThreadInfoController } from "./MachineThreadInfo";
|
|
14
|
+
import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
|
|
15
|
+
import { sort } from "socket-function/src/misc";
|
|
16
|
+
import { PathAuditerController } from "./pathAuditer";
|
|
17
|
+
import { t } from "../2-proxy/schema2";
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
let { data, functions } = Querysub.createSchema<{
|
|
21
|
+
values: {
|
|
22
|
+
[key: string]: number;
|
|
23
|
+
};
|
|
24
|
+
}>()({
|
|
25
|
+
functions: {
|
|
26
|
+
setValue(key: string, value: number) {
|
|
27
|
+
data().values[key] = value;
|
|
28
|
+
},
|
|
29
|
+
incrementValue(key: string) {
|
|
30
|
+
data().values[key]++;
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
module,
|
|
34
|
+
moduleId: "syncText",
|
|
35
|
+
functionMetadata: {
|
|
36
|
+
setValue: {
|
|
37
|
+
keyOverride: {
|
|
38
|
+
getPrefix: data => data().values,
|
|
39
|
+
getKey: (key: string, _value: number) => key,
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
incrementValue: {
|
|
43
|
+
keyOverride: {
|
|
44
|
+
getPrefix: data => data().values,
|
|
45
|
+
getKey: (key: string) => key,
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
permissions: {
|
|
50
|
+
PERMISSIONS: isSuperUserPERMISSIONS
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
function getUniqueThreadName(thread: { entrypoint: string; nodeId: string }, allThreads: { entrypoint: string; nodeId: string }[]): string {
|
|
56
|
+
function getDisplayName(entrypoint: string): string {
|
|
57
|
+
return entrypoint.replaceAll("\\", "/").split("/").slice(-3).join("/");
|
|
58
|
+
}
|
|
59
|
+
let baseName = getDisplayName(thread.entrypoint);
|
|
60
|
+
let duplicates = allThreads.filter(t => getDisplayName(t.entrypoint) === baseName);
|
|
61
|
+
if (duplicates.length > 1) {
|
|
62
|
+
let index = duplicates.findIndex(t => t.nodeId === thread.nodeId);
|
|
63
|
+
return `${baseName} [${index + 1}]`;
|
|
64
|
+
}
|
|
65
|
+
return baseName;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export class SyncTestPage extends qreact.Component {
|
|
69
|
+
state = t.state({
|
|
70
|
+
auditRunning: t.lookup({
|
|
71
|
+
isRunning: t.boolean
|
|
72
|
+
})
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
async runAudit(nodeId: string) {
|
|
76
|
+
this.state.auditRunning[nodeId] = { isRunning: true };
|
|
77
|
+
try {
|
|
78
|
+
await PathAuditerController.nodes[SocketFunction.browserNodeId()].runAuditNow_forBrowser(nodeId);
|
|
79
|
+
} finally {
|
|
80
|
+
Querysub.commit(() => {
|
|
81
|
+
this.state.auditRunning[nodeId] = { isRunning: false };
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
render() {
|
|
87
|
+
let values = data().values;
|
|
88
|
+
let allNodes = MachineThreadInfoController(SocketFunction.browserNodeId()).getAllInfo();
|
|
89
|
+
let allThreads = Array.from(allNodes?.byThread.values() || []);
|
|
90
|
+
allThreads = allThreads.slice();
|
|
91
|
+
sort(allThreads, x => x.entrypoint);
|
|
92
|
+
|
|
93
|
+
return (
|
|
94
|
+
<div className={css.pad2(20).vbox(40)}>
|
|
95
|
+
<h1>Sync Test Page</h1>
|
|
96
|
+
|
|
97
|
+
<div className={css.vbox(10)}>
|
|
98
|
+
<div className={css.boldStyle}>Audit Controls</div>
|
|
99
|
+
<div className={css.hbox(10, 5).wrap}>
|
|
100
|
+
{allThreads.map(thread => {
|
|
101
|
+
let auditStatus = this.state.auditRunning[thread.nodeId];
|
|
102
|
+
let isRunning = auditStatus && "isRunning" in auditStatus && auditStatus.isRunning;
|
|
103
|
+
return (
|
|
104
|
+
<div className={css.hbox(5).hsl(0, 0, 95).pad2(5)} title={JSON.stringify(thread)}>
|
|
105
|
+
<div>{getUniqueThreadName(thread, allThreads)}</div>
|
|
106
|
+
<Button
|
|
107
|
+
onClick={() => this.runAudit(thread.nodeId)}
|
|
108
|
+
disabled={isRunning}
|
|
109
|
+
>
|
|
110
|
+
{isRunning && "Running..." || "Run Audit"}
|
|
111
|
+
</Button>
|
|
112
|
+
</div>
|
|
113
|
+
);
|
|
114
|
+
})}
|
|
115
|
+
</div>
|
|
116
|
+
</div>
|
|
117
|
+
|
|
118
|
+
<div className={css.vbox(20)}>
|
|
119
|
+
{Object.entries(values).map(([key, value]) => {
|
|
120
|
+
const valuePath = Querysub.getPath(() => data().values[key]);
|
|
121
|
+
if (!valuePath) return undefined;
|
|
122
|
+
const path = valuePath;
|
|
123
|
+
return (
|
|
124
|
+
<div className={css.vbox(0)}>
|
|
125
|
+
<div className={css.hbox(10)}>
|
|
126
|
+
<span class={css.boldStyle}>{key}</span>
|
|
127
|
+
<span>{value}</span>
|
|
128
|
+
<Button onClick={() => functions.incrementValue(key)}>Increment</Button>
|
|
129
|
+
<InputLabel value={value} number onChangeValue={value => functions.setValue(key, +value)} />
|
|
130
|
+
</div>
|
|
131
|
+
<div className={css.hbox(10, 2).wrap.marginLeft(20)}>
|
|
132
|
+
{allThreads.map(thread => {
|
|
133
|
+
let result;
|
|
134
|
+
let hasError = false;
|
|
135
|
+
try {
|
|
136
|
+
result = syncTestController(SocketFunction.browserNodeId()).getValueAndTime_forBrowser({
|
|
137
|
+
path: path,
|
|
138
|
+
nodeId: thread.nodeId
|
|
139
|
+
});
|
|
140
|
+
} catch {
|
|
141
|
+
hasError = true;
|
|
142
|
+
}
|
|
143
|
+
if (hasError) return undefined;
|
|
144
|
+
return (
|
|
145
|
+
<div className={css.hbox(5).hsl(0, 0, 100).pad2(2)} title={JSON.stringify(thread)}>
|
|
146
|
+
<div>{getUniqueThreadName(thread, allThreads)}</div>
|
|
147
|
+
<Button onClick={async () => {
|
|
148
|
+
await SyncTestController.nodes[SocketFunction.browserNodeId()].DEBUG_secretlyDeleteAllValuesWithPath_forBrowser({
|
|
149
|
+
path: path,
|
|
150
|
+
nodeId: thread.nodeId
|
|
151
|
+
});
|
|
152
|
+
}}>
|
|
153
|
+
Delete Value
|
|
154
|
+
</Button>
|
|
155
|
+
{hasError && <div>Error</div> || <>
|
|
156
|
+
<div>Value: {result?.value}</div>
|
|
157
|
+
<div>Time: {result?.time}</div>
|
|
158
|
+
</>}
|
|
159
|
+
</div>
|
|
160
|
+
);
|
|
161
|
+
})}
|
|
162
|
+
</div>
|
|
163
|
+
</div>
|
|
164
|
+
);
|
|
165
|
+
})}
|
|
166
|
+
</div>
|
|
167
|
+
<InputLabel
|
|
168
|
+
label="Insert new key"
|
|
169
|
+
onChangeValue={value => {
|
|
170
|
+
functions.setValue(value, 0);
|
|
171
|
+
}}
|
|
172
|
+
/>
|
|
173
|
+
<Button onClick={() => {
|
|
174
|
+
syncTestController.refreshAll();
|
|
175
|
+
}}>
|
|
176
|
+
Refresh
|
|
177
|
+
</Button>
|
|
178
|
+
</div>
|
|
179
|
+
);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class SyncTestControllerBase {
|
|
185
|
+
async test() {
|
|
186
|
+
return Date.now();
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
public async DEBUG_secretlyDeleteAllValuesWithPath_forBrowser(config: {
|
|
190
|
+
path: string;
|
|
191
|
+
nodeId: string;
|
|
192
|
+
}) {
|
|
193
|
+
await SyncTestController.nodes[config.nodeId].DEBUG_secretlyDeleteAllValuesWithPath(config.path);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
public async DEBUG_secretlyDeleteAllValuesWithPath(path: string) {
|
|
197
|
+
await authorityStorage.DEBUG_secretlyDeleteAllValuesWithPath(path);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
public async getValueAndTime_forBrowser(config: {
|
|
201
|
+
path: string;
|
|
202
|
+
nodeId: string;
|
|
203
|
+
}) {
|
|
204
|
+
return await SyncTestController.nodes[config.nodeId].getValueAndTime(config.path);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
public async getValueAndTime(path: string) {
|
|
208
|
+
let pathValue = authorityStorage.getValueAtTime(path);
|
|
209
|
+
let value = pathValueSerializer.getPathValue(pathValue);
|
|
210
|
+
let time = pathValue?.time.time;
|
|
211
|
+
return { value, time };
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
export const SyncTestController = SocketFunction.register(
|
|
216
|
+
"SyncTestController-019cc520-f449-710b-94df-80b7b3e0f787",
|
|
217
|
+
new SyncTestControllerBase(),
|
|
218
|
+
() => ({
|
|
219
|
+
test: {},
|
|
220
|
+
DEBUG_secretlyDeleteAllValuesWithPath_forBrowser: {},
|
|
221
|
+
DEBUG_secretlyDeleteAllValuesWithPath: {},
|
|
222
|
+
getValueAndTime_forBrowser: {},
|
|
223
|
+
getValueAndTime: {},
|
|
224
|
+
}),
|
|
225
|
+
() => ({
|
|
226
|
+
hooks: [assertIsManagementUser],
|
|
227
|
+
}),
|
|
228
|
+
{
|
|
229
|
+
// Auto expose, as we want our test functions to be exposed
|
|
230
|
+
//noAutoExpose: true,
|
|
231
|
+
}
|
|
232
|
+
);
|
|
233
|
+
|
|
234
|
+
let syncTestController = getSyncedController(SyncTestController, {
|
|
235
|
+
reads: {
|
|
236
|
+
getValueAndTime_forBrowser: ["value"],
|
|
237
|
+
},
|
|
238
|
+
writes: {
|
|
239
|
+
DEBUG_secretlyDeleteAllValuesWithPath_forBrowser: ["value"],
|
|
240
|
+
},
|
|
241
|
+
});
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
import * as typescript from "typescript";
|
|
2
|
+
import * as fs from "fs";
|
|
3
|
+
import * as path from "path";
|
|
4
|
+
|
|
5
|
+
interface Violation {
|
|
6
|
+
importPath: string;
|
|
7
|
+
importStage: number;
|
|
8
|
+
fileStage: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
interface FileViolations {
|
|
12
|
+
filePath: string;
|
|
13
|
+
fileStage: number;
|
|
14
|
+
violations: Violation[];
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
function getStageNumber(filePath: string): number | undefined {
|
|
18
|
+
let parts = filePath.split(/[\\/]/);
|
|
19
|
+
for (let part of parts) {
|
|
20
|
+
let match = part.match(/^(\d+)-/);
|
|
21
|
+
if (match) {
|
|
22
|
+
return parseInt(match[1]);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function getAllTypeScriptFiles(dir: string): string[] {
|
|
29
|
+
let results: string[] = [];
|
|
30
|
+
let entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
31
|
+
|
|
32
|
+
for (let entry of entries) {
|
|
33
|
+
let fullPath = path.join(dir, entry.name);
|
|
34
|
+
if (entry.isDirectory()) {
|
|
35
|
+
if (entry.name === "node_modules" || entry.name === ".git") {
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
results.push(...getAllTypeScriptFiles(fullPath));
|
|
39
|
+
} else if (entry.isFile() && (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx"))) {
|
|
40
|
+
results.push(fullPath);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
return results;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function resolveImportPath(importPath: string, fromFile: string, rootDir: string): string | undefined {
|
|
48
|
+
if (!importPath.startsWith(".")) {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
let fromDir = path.dirname(fromFile);
|
|
53
|
+
let resolved = path.resolve(fromDir, importPath);
|
|
54
|
+
|
|
55
|
+
for (let ext of [".ts", ".tsx", ".js", ".jsx"]) {
|
|
56
|
+
if (fs.existsSync(resolved + ext)) {
|
|
57
|
+
return path.relative(rootDir, resolved + ext);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
if (fs.existsSync(resolved) && fs.statSync(resolved).isDirectory()) {
|
|
62
|
+
for (let indexFile of ["index.ts", "index.tsx", "index.js", "index.jsx"]) {
|
|
63
|
+
let indexPath = path.join(resolved, indexFile);
|
|
64
|
+
if (fs.existsSync(indexPath)) {
|
|
65
|
+
return path.relative(rootDir, indexPath);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return undefined;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function extractImports(filePath: string): string[] {
|
|
74
|
+
let content = fs.readFileSync(filePath, "utf-8");
|
|
75
|
+
let sourceFile = typescript.createSourceFile(
|
|
76
|
+
filePath,
|
|
77
|
+
content,
|
|
78
|
+
typescript.ScriptTarget.Latest,
|
|
79
|
+
true
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
let imports: string[] = [];
|
|
83
|
+
|
|
84
|
+
function visit(node: typescript.Node) {
|
|
85
|
+
if (typescript.isImportDeclaration(node)) {
|
|
86
|
+
let moduleSpecifier = node.moduleSpecifier;
|
|
87
|
+
if (typescript.isStringLiteral(moduleSpecifier)) {
|
|
88
|
+
imports.push(moduleSpecifier.text);
|
|
89
|
+
}
|
|
90
|
+
} else if (typescript.isExportDeclaration(node) && node.moduleSpecifier) {
|
|
91
|
+
if (typescript.isStringLiteral(node.moduleSpecifier)) {
|
|
92
|
+
imports.push(node.moduleSpecifier.text);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// NOTE: Dynamic imports (import("...")) are intentionally NOT included
|
|
96
|
+
// because they are lazy-loaded at runtime and should be allowed to
|
|
97
|
+
// import from higher stages
|
|
98
|
+
|
|
99
|
+
typescript.forEachChild(node, visit);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
visit(sourceFile);
|
|
103
|
+
return imports;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function checkFile(filePath: string, rootDir: string): FileViolations | undefined {
|
|
107
|
+
let fileStage = getStageNumber(filePath);
|
|
108
|
+
if (fileStage === undefined) {
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
let imports = extractImports(filePath);
|
|
113
|
+
let violations: Violation[] = [];
|
|
114
|
+
|
|
115
|
+
for (let importPath of imports) {
|
|
116
|
+
let resolvedPath = resolveImportPath(importPath, filePath, rootDir);
|
|
117
|
+
if (!resolvedPath) {
|
|
118
|
+
continue;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
let importStage = getStageNumber(resolvedPath);
|
|
122
|
+
if (importStage !== undefined && importStage > fileStage) {
|
|
123
|
+
violations.push({
|
|
124
|
+
importPath: resolvedPath,
|
|
125
|
+
importStage,
|
|
126
|
+
fileStage
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
if (violations.length > 0) {
|
|
132
|
+
return {
|
|
133
|
+
filePath,
|
|
134
|
+
fileStage,
|
|
135
|
+
violations
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
async function main() {
|
|
143
|
+
let auditRoot = ".";
|
|
144
|
+
let rootDir = path.resolve(auditRoot);
|
|
145
|
+
|
|
146
|
+
console.log("Auditing import violations...");
|
|
147
|
+
console.log(`Root directory: ${rootDir}\n`);
|
|
148
|
+
|
|
149
|
+
let allFiles = getAllTypeScriptFiles(rootDir);
|
|
150
|
+
let allViolations: FileViolations[] = [];
|
|
151
|
+
|
|
152
|
+
for (let file of allFiles) {
|
|
153
|
+
let violations = checkFile(file, rootDir);
|
|
154
|
+
if (violations) {
|
|
155
|
+
allViolations.push(violations);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (allViolations.length === 0) {
|
|
160
|
+
console.log("✓ No violations found!");
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
console.log("VIOLATIONS FOUND:\n");
|
|
165
|
+
console.log("=".repeat(80));
|
|
166
|
+
|
|
167
|
+
for (let fileViolation of allViolations) {
|
|
168
|
+
console.log(`\n${fileViolation.filePath} (stage ${fileViolation.fileStage}):`);
|
|
169
|
+
for (let violation of fileViolation.violations) {
|
|
170
|
+
console.log(` → imports ${violation.importPath} (stage ${violation.importStage})`);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
console.log("\n" + "=".repeat(80));
|
|
175
|
+
console.log("\nSUMMARY:");
|
|
176
|
+
console.log(`Total files with violations: ${allViolations.length}`);
|
|
177
|
+
console.log(`Total violations: ${allViolations.reduce((sum, fv) => sum + fv.violations.length, 0)}`);
|
|
178
|
+
|
|
179
|
+
console.log("\nViolations per file:");
|
|
180
|
+
for (let fileViolation of allViolations) {
|
|
181
|
+
console.log(` ${fileViolation.filePath}: ${fileViolation.violations.length} violation${fileViolation.violations.length === 1 ? "" : "s"}`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
main().catch(console.error).finally(() => process.exit(0));
|
|
@@ -9,7 +9,7 @@ import { lazy } from "socket-function/src/caching";
|
|
|
9
9
|
|
|
10
10
|
let devToolsUrl = "";
|
|
11
11
|
|
|
12
|
-
const listenOnDebugger = lazy(
|
|
12
|
+
const listenOnDebugger = lazy(function listenOnDebugger() {
|
|
13
13
|
if (devToolsUrl) return;
|
|
14
14
|
if (!isNode()) return;
|
|
15
15
|
// IMPORTANT! NEVER use the built-in port. This adds a tiny bit of security. Not much,
|
|
@@ -37,14 +37,14 @@ const listenOnDebugger = lazy(async function listenOnDebugger() {
|
|
|
37
37
|
}
|
|
38
38
|
}
|
|
39
39
|
});
|
|
40
|
-
export
|
|
40
|
+
export function getDebuggerUrl() {
|
|
41
41
|
// NOTE: Very unfortunate, as this means that if a developer is debugging a server for a bit,
|
|
42
42
|
// the inspector is left open. It's kind of hard to tell when they are done though.
|
|
43
43
|
// - This is still somewhat secure, as only localhost can connect, but... having local network
|
|
44
44
|
// accessing turning into remote execution access isn't great (especially for developers,
|
|
45
45
|
// who could then get pwned just by clicking on a link).
|
|
46
46
|
// TODO: The security of this can be improved, see NodeViewer.tsx:getExternalInspectURL
|
|
47
|
-
|
|
47
|
+
listenOnDebugger();
|
|
48
48
|
return devToolsUrl;
|
|
49
49
|
}
|
|
50
50
|
function doesProcessExist(pid: number) {
|