querysub 0.402.0 → 0.404.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.cursorrules +2 -0
- package/bin/audit-imports.js +4 -0
- package/package.json +7 -4
- package/spec.txt +77 -0
- package/src/-a-archives/archiveCache.ts +9 -4
- package/src/-a-archives/archivesBackBlaze.ts +1039 -1039
- package/src/-a-auth/certs.ts +0 -12
- package/src/-c-identity/IdentityController.ts +12 -3
- package/src/-f-node-discovery/NodeDiscovery.ts +32 -26
- package/src/-g-core-values/NodeCapabilities.ts +12 -2
- package/src/0-path-value-core/AuthorityLookup.ts +239 -0
- package/src/0-path-value-core/LockWatcher2.ts +150 -0
- package/src/0-path-value-core/PathRouter.ts +535 -0
- package/src/0-path-value-core/PathRouterRouteOverride.ts +72 -0
- package/src/0-path-value-core/PathRouterServerAuthoritySpec.tsx +65 -0
- package/src/0-path-value-core/PathValueCommitter.ts +222 -488
- package/src/0-path-value-core/PathValueController.ts +277 -239
- package/src/0-path-value-core/PathWatcher.ts +534 -0
- package/src/0-path-value-core/ShardPrefixes.ts +31 -0
- package/src/0-path-value-core/ValidStateComputer.ts +303 -0
- package/src/0-path-value-core/archiveLocks/ArchiveLocks.ts +1 -1
- package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +80 -44
- package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts +13 -16
- package/src/0-path-value-core/auditLogs.ts +2 -0
- package/src/0-path-value-core/hackedPackedPathParentFiltering.ts +97 -0
- package/src/0-path-value-core/pathValueArchives.ts +490 -492
- package/src/0-path-value-core/pathValueCore.ts +195 -1492
- package/src/0-path-value-core/startupAuthority.ts +74 -0
- package/src/1-path-client/RemoteWatcher.ts +100 -83
- package/src/1-path-client/pathValueClientWatcher.ts +808 -815
- package/src/2-proxy/PathValueProxyWatcher.ts +10 -8
- package/src/2-proxy/archiveMoveHarness.ts +182 -214
- package/src/2-proxy/garbageCollection.ts +9 -8
- package/src/2-proxy/schema2.ts +21 -1
- package/src/3-path-functions/PathFunctionHelpers.ts +206 -180
- package/src/3-path-functions/PathFunctionRunner.ts +943 -766
- package/src/3-path-functions/PathFunctionRunnerMain.ts +5 -3
- package/src/3-path-functions/pathFunctionLoader.ts +2 -2
- package/src/3-path-functions/syncSchema.ts +592 -521
- package/src/4-deploy/deployFunctions.ts +19 -4
- package/src/4-deploy/deployGetFunctionsInner.ts +8 -2
- package/src/4-deploy/deployMain.ts +51 -68
- package/src/4-deploy/edgeClientWatcher.tsx +1 -1
- package/src/4-deploy/edgeNodes.ts +2 -2
- package/src/4-dom/qreact.tsx +2 -4
- package/src/4-dom/qreactTest.tsx +7 -13
- package/src/4-querysub/Querysub.ts +21 -8
- package/src/4-querysub/QuerysubController.ts +45 -29
- package/src/4-querysub/permissions.ts +2 -2
- package/src/4-querysub/querysubPrediction.ts +80 -70
- package/src/4-querysub/schemaHelpers.ts +5 -1
- package/src/5-diagnostics/GenericFormat.tsx +14 -9
- package/src/archiveapps/archiveGCEntry.tsx +9 -2
- package/src/archiveapps/archiveJoinEntry.ts +87 -84
- package/src/archiveapps/archiveMergeEntry.tsx +2 -0
- package/src/bits.ts +19 -0
- package/src/config.ts +21 -3
- package/src/config2.ts +23 -48
- package/src/deployManager/components/DeployPage.tsx +7 -3
- package/src/deployManager/machineSchema.ts +4 -1
- package/src/diagnostics/ActionsHistory.ts +3 -8
- package/src/diagnostics/AuditLogPage.tsx +2 -3
- package/src/diagnostics/FunctionCallInfo.tsx +141 -0
- package/src/diagnostics/FunctionCallInfoState.ts +162 -0
- package/src/diagnostics/MachineThreadInfo.tsx +1 -1
- package/src/diagnostics/NodeViewer.tsx +37 -48
- package/src/diagnostics/SyncTestPage.tsx +241 -0
- package/src/diagnostics/auditImportViolations.ts +185 -0
- package/src/diagnostics/listenOnDebugger.ts +3 -3
- package/src/diagnostics/logs/IndexedLogs/BufferUnitSet.ts +10 -4
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +2 -2
- package/src/diagnostics/logs/IndexedLogs/LogViewer3.tsx +24 -22
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +1 -1
- package/src/diagnostics/logs/diskLogGlobalContext.ts +1 -0
- package/src/diagnostics/logs/errorNotifications2/logWatcher.ts +1 -3
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryEditor.tsx +39 -17
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleEntryReadMode.tsx +4 -6
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleInstanceTableView.tsx +36 -5
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCyclePage.tsx +19 -5
- package/src/diagnostics/logs/lifeCycleAnalysis/LifeCycleRenderer.tsx +15 -7
- package/src/diagnostics/logs/lifeCycleAnalysis/NestedLifeCycleInfo.tsx +28 -106
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMatching.ts +2 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleMisc.ts +0 -0
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycleSearch.tsx +18 -7
- package/src/diagnostics/logs/lifeCycleAnalysis/lifeCycles.tsx +3 -0
- package/src/diagnostics/managementPages.tsx +10 -3
- package/src/diagnostics/misc-pages/ArchiveViewer.tsx +20 -26
- package/src/diagnostics/misc-pages/ArchiveViewerTree.tsx +6 -4
- package/src/diagnostics/misc-pages/ComponentSyncStats.tsx +2 -2
- package/src/diagnostics/misc-pages/LocalWatchViewer.tsx +7 -9
- package/src/diagnostics/misc-pages/SnapshotViewer.tsx +23 -12
- package/src/diagnostics/misc-pages/archiveViewerShared.tsx +1 -1
- package/src/diagnostics/pathAuditer.ts +486 -0
- package/src/diagnostics/pathAuditerCallback.ts +20 -0
- package/src/diagnostics/watchdog.ts +8 -1
- package/src/library-components/URLParam.ts +1 -1
- package/src/misc/hash.ts +1 -0
- package/src/path.ts +21 -7
- package/src/server.ts +54 -47
- package/src/user-implementation/loginEmail.tsx +1 -1
- package/tempnotes.txt +67 -0
- package/test.ts +288 -95
- package/src/0-path-value-core/NodePathAuthorities.ts +0 -1057
- package/src/0-path-value-core/PathController.ts +0 -1
- package/src/5-diagnostics/diskValueAudit.ts +0 -218
- package/src/5-diagnostics/memoryValueAudit.ts +0 -438
- package/src/archiveapps/lockTest.ts +0 -127
|
@@ -1,55 +1,42 @@
|
|
|
1
1
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
2
2
|
import { css } from "typesafecss";
|
|
3
|
-
import { t } from "../../../2-proxy/schema2";
|
|
4
3
|
import { qreact } from "../../../4-dom/qreact";
|
|
5
|
-
import {
|
|
6
|
-
import { Button } from "../../../library-components/Button";
|
|
7
|
-
import { niceStringify } from "../../../niceStringify";
|
|
8
|
-
import { LogDatum } from "../diskLogger";
|
|
9
|
-
import { LifeCycle, LifeCyclesController, getVariables } from "./lifeCycles";
|
|
10
|
-
import { createLifeCycleSearch, LifecycleInstance } from "./lifeCycleSearch";
|
|
11
|
-
import { IndexedLogResults } from "../IndexedLogs/BufferIndexHelpers";
|
|
4
|
+
import { LifeCycle, LifeCycleEntry, LifeCyclesController } from "./lifeCycles";
|
|
12
5
|
import { formatSearchString } from "../IndexedLogs/LogViewerParams";
|
|
13
|
-
import {
|
|
6
|
+
import { ATag } from "../../../library-components/ATag";
|
|
7
|
+
import { lifecycleIdURL, limitURL, additionalSearchURL, phase2AdditionalSearchURL } from "./LifeCyclePage";
|
|
14
8
|
|
|
15
9
|
export class NestedLifeCycleInfo extends qreact.Component<{
|
|
16
10
|
keyName: string;
|
|
17
11
|
value: unknown;
|
|
18
|
-
|
|
12
|
+
entry: LifeCycleEntry;
|
|
19
13
|
currentLifeCycleId: string;
|
|
20
14
|
}> {
|
|
21
|
-
state = t.state({
|
|
22
|
-
searchingLifeCycleId: t.atomic<string | undefined>(undefined),
|
|
23
|
-
phase1Results: t.atomic<LogDatum[]>([]),
|
|
24
|
-
phase1InvalidResults: t.atomic<Array<{ datum: LogDatum; missingKeys: string[] }>>([]),
|
|
25
|
-
phase1Stats: t.atomic<IndexedLogResults | undefined>(undefined),
|
|
26
|
-
phase1Searching: t.boolean(false),
|
|
27
|
-
phase2Results: t.atomic<LogDatum[]>([]),
|
|
28
|
-
phase2Stats: t.atomic<IndexedLogResults | undefined>(undefined),
|
|
29
|
-
phase2Searching: t.boolean(false),
|
|
30
|
-
phase2HitLimit: t.boolean(false),
|
|
31
|
-
lifecycleInstances: t.atomic<LifecycleInstance[]>([]),
|
|
32
|
-
});
|
|
33
|
-
|
|
34
15
|
controller = LifeCyclesController(SocketFunction.browserNodeId());
|
|
35
|
-
search = createLifeCycleSearch(this);
|
|
36
16
|
|
|
37
17
|
render() {
|
|
38
18
|
let lifeCycles = this.controller.getLifeCycles();
|
|
39
19
|
if (!lifeCycles) return null;
|
|
40
20
|
|
|
41
|
-
let { keyName, value,
|
|
21
|
+
let { keyName, value, entry, currentLifeCycleId } = this.props;
|
|
22
|
+
|
|
23
|
+
let isGroupByKey = Array.isArray(entry.groupByKeys) && entry.groupByKeys.some(k => k.ourKey === keyName);
|
|
24
|
+
let aliases = isGroupByKey && entry.groupByKeys.find(k => k.ourKey === keyName)?.aliases || entry.variables[keyName]?.aliases || [];
|
|
42
25
|
|
|
43
26
|
let allKeysToMatch = [keyName, ...aliases].filter(x => !["__threadId", "__machineId"].includes(x));
|
|
44
27
|
let matchingLifeCycles: Array<{ lifecycle: LifeCycle; matchedKey: string }> = [];
|
|
45
28
|
|
|
46
29
|
for (let lifecycle of lifeCycles) {
|
|
47
|
-
|
|
30
|
+
let keySources = allKeysToMatch;
|
|
31
|
+
// Don't match our keyName if we are the same lifecycle
|
|
32
|
+
if (lifecycle.id === currentLifeCycleId) {
|
|
33
|
+
keySources = aliases;
|
|
34
|
+
}
|
|
48
35
|
|
|
49
36
|
let matchedKey: string | undefined = undefined;
|
|
50
37
|
for (let entry of lifecycle.entries) {
|
|
51
38
|
for (let groupByKey of entry.groupByKeys) {
|
|
52
|
-
if (
|
|
39
|
+
if (keySources.includes(groupByKey.ourKey)) {
|
|
53
40
|
matchedKey = groupByKey.ourKey;
|
|
54
41
|
break;
|
|
55
42
|
}
|
|
@@ -66,85 +53,20 @@ export class NestedLifeCycleInfo extends qreact.Component<{
|
|
|
66
53
|
|
|
67
54
|
return <>
|
|
68
55
|
{matchingLifeCycles.map(({ lifecycle, matchedKey }) => {
|
|
69
|
-
let
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
let stats = this.state.phase1Stats;
|
|
84
|
-
let blocksScanned = stats && (stats.localFilesSearched + stats.backblazeFilesSearched) || 0;
|
|
85
|
-
let resultsCount = this.state.phase1Results.length;
|
|
86
|
-
parts.push(`${blocksScanned} > ${resultsCount}`);
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
if (this.state.phase2Searching) {
|
|
90
|
-
let stats = this.state.phase2Stats;
|
|
91
|
-
let blocksScanned = stats && (stats.localFilesSearched + stats.backblazeFilesSearched) || 0;
|
|
92
|
-
let resultsCount = this.state.phase2Results.length;
|
|
93
|
-
parts.push(`${blocksScanned} > ${resultsCount}`);
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
progressText = parts.length > 0 && `(${parts.join(" | ")})` || "(...)";
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
return <>
|
|
100
|
-
<Button
|
|
101
|
-
hue={180}
|
|
102
|
-
onClick={() => {
|
|
103
|
-
void this.search.searchLifeCycle({
|
|
104
|
-
lifeCycleId: lifecycle.id,
|
|
105
|
-
limit: 10,
|
|
106
|
-
additionalSearch: formatSearchString({ [matchedKey]: value })
|
|
107
|
-
});
|
|
108
|
-
}}
|
|
109
|
-
>
|
|
110
|
-
{lifecycle.title} {isSearching && "⏳" || ""} {progressText} {hasSearched && `(${instances.length})` || ""}
|
|
111
|
-
</Button>
|
|
112
|
-
{hasSearched && !hasResults && (
|
|
113
|
-
<div className={css.pad2(4).colorhsl(0, 0, 50)}>No results found</div>
|
|
114
|
-
)}
|
|
115
|
-
{hasResults && (() => {
|
|
116
|
-
let firstInstance = instances[0];
|
|
117
|
-
let totalCount = instances.length;
|
|
118
|
-
let hitLimit = totalCount >= 10;
|
|
119
|
-
|
|
120
|
-
let keys = lifecycle.entries.map(x => x.groupByKeys.map(y => y.ourKey).concat(Object.keys(x.variables))).flat();
|
|
121
|
-
|
|
122
|
-
function valueHash(key: string, value: unknown) {
|
|
123
|
-
return getPathStr2(key, String(value));
|
|
124
|
-
}
|
|
125
|
-
let values = new Map<string, { key: string; value: unknown }>();
|
|
126
|
-
for (let entry of firstInstance.entries) {
|
|
127
|
-
for (let [key, value] of Object.entries(entry.datum)) {
|
|
128
|
-
if (!keys.includes(key)) continue;
|
|
129
|
-
if (key === matchedKey) continue;
|
|
130
|
-
if (["__threadId", "__machineId"].includes(key)) continue;
|
|
131
|
-
values.set(valueHash(key, value), { key, value });
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
return <>
|
|
136
|
-
<div className={css.hbox(4)}>
|
|
137
|
-
{hitLimit && <span className={css.colorhsl(30, 80, 40).boldStyle}>(hit limit!)</span>}
|
|
138
|
-
</div>
|
|
139
|
-
{Array.from(values.values()).map(({ key, value }, idx) => (
|
|
140
|
-
<div key={idx} className={css.hbox(4)}>
|
|
141
|
-
<span className={css.colorhsl(200, 50, 60).boldStyle}>{key}:</span>
|
|
142
|
-
<span className={css.colorhsl(0, 0, 50)}>{niceStringify(value)}</span>
|
|
143
|
-
</div>
|
|
144
|
-
))}
|
|
145
|
-
</>;
|
|
146
|
-
})()}
|
|
147
|
-
</>;
|
|
56
|
+
let additionalSearch = formatSearchString({ [matchedKey]: value });
|
|
57
|
+
|
|
58
|
+
return <ATag
|
|
59
|
+
key={lifecycle.id}
|
|
60
|
+
values={[
|
|
61
|
+
lifecycleIdURL.getOverride(lifecycle.id),
|
|
62
|
+
limitURL.getOverride(10),
|
|
63
|
+
additionalSearchURL.getOverride(additionalSearch),
|
|
64
|
+
phase2AdditionalSearchURL.getOverride("")
|
|
65
|
+
]}
|
|
66
|
+
className={css.pad2(8, 4).hsl(180, 60, 90).hslhover(180, 60, 85).colorhsl(180, 80, 30).textDecoration("none").borderRadius(4).display("inline-block")}
|
|
67
|
+
>
|
|
68
|
+
{lifecycle.title}
|
|
69
|
+
</ATag>;
|
|
148
70
|
})}
|
|
149
71
|
</>;
|
|
150
72
|
}
|
|
@@ -42,6 +42,8 @@ export function getLifecycleMatchesForDatum(
|
|
|
42
42
|
entry,
|
|
43
43
|
datum: extractedDatum,
|
|
44
44
|
});
|
|
45
|
+
// For now, this is fine. Sometimes we actually have overlapping entries, which is bad, but doesn't really matter, And if we don't break here, we'll get multiple matches. And all of our entries have the same group by keys and variables anyway, so if we match one, it's the same as matching all of them.
|
|
46
|
+
break;
|
|
45
47
|
}
|
|
46
48
|
}
|
|
47
49
|
|
|
File without changes
|
|
@@ -89,9 +89,10 @@ export function createLifeCycleSearch(
|
|
|
89
89
|
let searchLifeCycle = async (params: {
|
|
90
90
|
lifeCycleId: string;
|
|
91
91
|
additionalSearch: string;
|
|
92
|
+
phase2AdditionalSearch: string;
|
|
92
93
|
limit: number;
|
|
93
94
|
}) => {
|
|
94
|
-
let { lifeCycleId, additionalSearch, limit } = params;
|
|
95
|
+
let { lifeCycleId, additionalSearch, phase2AdditionalSearch, limit } = params;
|
|
95
96
|
let lifeCycles = await context.controller.getLifeCycles.promise();
|
|
96
97
|
if (!lifeCycles) return;
|
|
97
98
|
|
|
@@ -128,6 +129,7 @@ export function createLifeCycleSearch(
|
|
|
128
129
|
lifeCycle,
|
|
129
130
|
phase1Results,
|
|
130
131
|
currentSequenceNumber,
|
|
132
|
+
phase2AdditionalSearch,
|
|
131
133
|
limit,
|
|
132
134
|
});
|
|
133
135
|
|
|
@@ -164,10 +166,13 @@ export function createLifeCycleSearch(
|
|
|
164
166
|
resetPhase1State();
|
|
165
167
|
});
|
|
166
168
|
|
|
167
|
-
let searchQuery = startEntries.map(entry =>
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
169
|
+
let searchQuery = startEntries.map(entry => {
|
|
170
|
+
let pattern = entry.matchPattern;
|
|
171
|
+
if (additionalSearch) {
|
|
172
|
+
pattern = pattern + "&" + additionalSearch;
|
|
173
|
+
}
|
|
174
|
+
return pattern;
|
|
175
|
+
}).join("|");
|
|
171
176
|
let searchBuffer = Buffer.from(searchQuery, "utf8");
|
|
172
177
|
|
|
173
178
|
let sourceTypes = new Set<string>();
|
|
@@ -298,9 +303,10 @@ export function createLifeCycleSearch(
|
|
|
298
303
|
lifeCycle: LifeCycle;
|
|
299
304
|
phase1Results: LogDatum[];
|
|
300
305
|
currentSequenceNumber: number;
|
|
306
|
+
phase2AdditionalSearch: string;
|
|
301
307
|
limit: number;
|
|
302
308
|
}): Promise<{ results: LogDatum[]; hitLimit: boolean }> => {
|
|
303
|
-
let { lifeCycle, phase1Results, currentSequenceNumber, limit } = params;
|
|
309
|
+
let { lifeCycle, phase1Results, currentSequenceNumber, phase2AdditionalSearch, limit } = params;
|
|
304
310
|
|
|
305
311
|
if (phase1Results.length === 0) {
|
|
306
312
|
return { results: [], hitLimit: false };
|
|
@@ -411,7 +417,12 @@ export function createLifeCycleSearch(
|
|
|
411
417
|
sourceType === "warning" && loggers.warnLogs ||
|
|
412
418
|
loggers.errorLogs;
|
|
413
419
|
|
|
414
|
-
let searchQuery = Array.from(queries).
|
|
420
|
+
let searchQuery = Array.from(queries).map(query => {
|
|
421
|
+
if (phase2AdditionalSearch) {
|
|
422
|
+
return query + "&" + phase2AdditionalSearch;
|
|
423
|
+
}
|
|
424
|
+
return query;
|
|
425
|
+
}).join("|");
|
|
415
426
|
let searchBuffer = Buffer.from(searchQuery, "utf8");
|
|
416
427
|
|
|
417
428
|
let results: LogDatum[] = [];
|
|
@@ -15,6 +15,9 @@ export type LifeCycle = {
|
|
|
15
15
|
entries: LifeCycleEntry[];
|
|
16
16
|
};
|
|
17
17
|
|
|
18
|
+
// TODO: Actually, maybe we should make it so the group by keys and the variables are just in the life cycle, as we keep wanting them to be identical for all the entries. It would be a pretty simple refactor. Although we would have to retrofit the old data, But then again, we could just have the AI create a script for that...
|
|
19
|
+
// IF we do any changes, we SHOULD refactor it so we have the same groupByKeys and variables
|
|
20
|
+
// - Aliases for variables is very useful, and for group by keys. However, start key is not needed. We're never going to use that. We'll just change the logging code to log the key that we want.
|
|
18
21
|
export type LifeCycleEntry = {
|
|
19
22
|
description?: string;
|
|
20
23
|
|
|
@@ -13,7 +13,6 @@ import { t } from "../../src/2-proxy/schema2";
|
|
|
13
13
|
import { SocketRegistered } from "socket-function/SocketFunctionTypes";
|
|
14
14
|
import { nextId } from "socket-function/src/misc";
|
|
15
15
|
import { IdentityController_getSecureIP, IdentityController_getMachineId } from "../-c-identity/IdentityController";
|
|
16
|
-
import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
|
|
17
16
|
import { getNextTime } from "../0-path-value-core/pathValueCore";
|
|
18
17
|
import { encodeArgs } from "../3-path-functions/PathFunctionHelpers";
|
|
19
18
|
import { CallSpec, getCurrentCall, getCurrentCallAllowUndefined, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
|
|
@@ -28,6 +27,7 @@ import { addComponentButton } from "../5-diagnostics/qreactDebug";
|
|
|
28
27
|
import { closeAllModals } from "../5-diagnostics/Modal";
|
|
29
28
|
import { delay } from "socket-function/src/batching";
|
|
30
29
|
import { currentViewParam, selectedServiceIdParam } from "../deployManager/urlParams";
|
|
30
|
+
import { FunctionCallInfo } from "./FunctionCallInfo";
|
|
31
31
|
|
|
32
32
|
export const managementPageURL = new URLParam("managementpage", "");
|
|
33
33
|
export const showingManagementURL = new URLParam("showingmanagement", false);
|
|
@@ -87,7 +87,7 @@ export async function registerManagementPages2(config: {
|
|
|
87
87
|
getModule: () => import("./NodeViewer"),
|
|
88
88
|
});
|
|
89
89
|
inputPages.push({
|
|
90
|
-
title: "
|
|
90
|
+
title: "LOG VIEWER",
|
|
91
91
|
componentName: "LogViewer3",
|
|
92
92
|
getModule: () => import("./logs/IndexedLogs/LogViewer3"),
|
|
93
93
|
});
|
|
@@ -142,6 +142,12 @@ export async function registerManagementPages2(config: {
|
|
|
142
142
|
componentName: "RequireAuditPage",
|
|
143
143
|
getModule: () => import("./misc-pages/RequireAuditPage"),
|
|
144
144
|
});
|
|
145
|
+
inputPages.push({
|
|
146
|
+
title: "Sync Test",
|
|
147
|
+
componentName: "SyncTestPage",
|
|
148
|
+
controllerName: "SyncTestController",
|
|
149
|
+
getModule: () => import("./SyncTestPage"),
|
|
150
|
+
});
|
|
145
151
|
inputPages.push(...config.pages);
|
|
146
152
|
|
|
147
153
|
// NOTE: We don't store the UI in the database (here, or anywhere else, at least
|
|
@@ -343,7 +349,7 @@ class ManagementRoot extends qreact.Component {
|
|
|
343
349
|
<ATag values={[
|
|
344
350
|
managementPageURL.getOverride("MachinesPage"),
|
|
345
351
|
currentViewParam.getOverride("services"),
|
|
346
|
-
]}>Deploy
|
|
352
|
+
]}>Deploy Framework</ATag>
|
|
347
353
|
<ATag values={[
|
|
348
354
|
managementPageURL.getOverride("MachinesPage"),
|
|
349
355
|
currentViewParam.getOverride("deploy"),
|
|
@@ -351,6 +357,7 @@ class ManagementRoot extends qreact.Component {
|
|
|
351
357
|
{pages.map(page =>
|
|
352
358
|
<ATag values={[{ param: managementPageURL, value: page.componentName }]}>{page.title}</ATag>
|
|
353
359
|
)}
|
|
360
|
+
<FunctionCallInfo />
|
|
354
361
|
</div>
|
|
355
362
|
{currentPage &&
|
|
356
363
|
<div
|
|
@@ -1,28 +1,24 @@
|
|
|
1
1
|
import { DecodedValuePath, PathValueArchives, pathValueArchives } from "../../0-path-value-core/pathValueArchives";
|
|
2
2
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
3
|
-
import {
|
|
4
|
-
import preact from "preact"; import { qreact } from "../../4-dom/qreact";
|
|
3
|
+
import { qreact } from "../../4-dom/qreact";
|
|
5
4
|
import { logErrors } from "../../errors";
|
|
6
5
|
import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
|
|
7
|
-
import {
|
|
8
|
-
import { getDomain } from "../../config";
|
|
6
|
+
import { rootPathStr } from "../../path";
|
|
9
7
|
import { css } from "typesafecss";
|
|
10
|
-
import { cacheArgsEqual,
|
|
8
|
+
import { cacheArgsEqual, cacheLimited, cacheShallowConfigArgEqual, lazy } from "socket-function/src/caching";
|
|
11
9
|
import { atomic, atomicObjectWrite, atomicObjectWriteNoFreeze, doAtomicWrites, proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
|
|
12
10
|
import { Querysub } from "../../4-querysub/QuerysubController";
|
|
13
11
|
import { formatDate, formatDateTime, formatNumber, formatPercent, formatTime } from "socket-function/src/formatting/format";
|
|
14
|
-
import { getSeededRandom } from "../../misc/random";
|
|
15
12
|
import { Histogram, XYValue } from "../../library-components/Histogram";
|
|
16
13
|
import { Button } from "../../library-components/Button";
|
|
17
14
|
import { createURLSync } from "../../library-components/URLParam";
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import { PathValue, Time, authorityStorage, epochTime } from "../../0-path-value-core/pathValueCore";
|
|
15
|
+
import { sort, throttleFunction } from "socket-function/src/misc";
|
|
16
|
+
import { isEmpty } from "../../misc";
|
|
17
|
+
import { PathValue, epochTime } from "../../0-path-value-core/pathValueCore";
|
|
22
18
|
import { pathValueSerializer } from "../../-h-path-value-serialize/PathValueSerializer";
|
|
23
|
-
import { InputLabel
|
|
24
|
-
import { addEpsilons
|
|
25
|
-
import {
|
|
19
|
+
import { InputLabel } from "../../library-components/InputLabel";
|
|
20
|
+
import { addEpsilons } from "../../bits";
|
|
21
|
+
import { measureWrap } from "socket-function/src/profiling/measure";
|
|
26
22
|
import { ArchiveViewerTable } from "./ArchiveViewerTable";
|
|
27
23
|
import { liveWatchPaths, skipStringsURL, skipValuesURL, startTime, endTime, addFileCount, archiveFileValue, filtersURL, viewMode, archiveViewerDiskCache } from "./archiveViewerShared";
|
|
28
24
|
import { ArchiveViewerFilterUI, filterDatums } from "./archiveViewerFilter";
|
|
@@ -33,6 +29,8 @@ import { t } from "../../4-querysub/Querysub";
|
|
|
33
29
|
import { Anchor } from "../../library-components/ATag";
|
|
34
30
|
import { assertIsManagementUser } from "../../diagnostics/managementPages";
|
|
35
31
|
import { ButtonSelector } from "../../library-components/ButtonSelector";
|
|
32
|
+
import { AuthoritySpec } from "../../0-path-value-core/PathRouter";
|
|
33
|
+
import { getAllAuthoritySpec } from "../../0-path-value-core/PathRouterServerAuthoritySpec";
|
|
36
34
|
|
|
37
35
|
// NOTE: We can load up to about 1M PathValues (although it really depends on their contents,
|
|
38
36
|
// specifically how many unique paths there are). After that the browser will likely stop
|
|
@@ -103,11 +101,7 @@ export class ArchiveViewer extends qreact.Component {
|
|
|
103
101
|
return;
|
|
104
102
|
}
|
|
105
103
|
try {
|
|
106
|
-
let newValuePaths = await controller.getValuePaths(
|
|
107
|
-
//pathPrefix: getPathStr1(getDomain()),
|
|
108
|
-
// More paths is probably better, as it might catch corrupted or missing domain files?
|
|
109
|
-
pathPrefix: rootPathStr,
|
|
110
|
-
});
|
|
104
|
+
let newValuePaths = await controller.getValuePaths(getAllAuthoritySpec());
|
|
111
105
|
Querysub.localCommit(() => this.state.valuePaths = newValuePaths);
|
|
112
106
|
} catch (e: any) {
|
|
113
107
|
Querysub.localCommit(() => {
|
|
@@ -190,8 +184,7 @@ export class ArchiveViewer extends qreact.Component {
|
|
|
190
184
|
this.state.parsedValues = atomicObjectWrite({});
|
|
191
185
|
});
|
|
192
186
|
|
|
193
|
-
|
|
194
|
-
const MAX_FILES = 1;
|
|
187
|
+
const MAX_FILES = 50;
|
|
195
188
|
const MAX_SIZE = 500 * 1024 * 1024;
|
|
196
189
|
let bufferObjs: { path: string; buffer: Buffer }[] = [];
|
|
197
190
|
let newParsedValues: { [path: string]: PathValue[] } = {};
|
|
@@ -249,14 +242,15 @@ export class ArchiveViewer extends qreact.Component {
|
|
|
249
242
|
this.state.progress.filesParsed.max = remainingFiles.size + bufferObjs.length;
|
|
250
243
|
});
|
|
251
244
|
|
|
252
|
-
|
|
253
|
-
|
|
245
|
+
for (let group of fileGroups) {
|
|
246
|
+
let time = Date.now();
|
|
247
|
+
console.log(`Downloading ${group.file.length} files, ${formatNumber(group.size)}B, at ${formatDateTime(time)}`);
|
|
254
248
|
let newBuffers = await controller.getRawValueFiles(group.file, {
|
|
255
249
|
// 1) Allows viewing old archives
|
|
256
250
|
// 2) Allows viewing data that is being updated frequently more easily?
|
|
257
251
|
includeRecycleBin: true
|
|
258
252
|
});
|
|
259
|
-
|
|
253
|
+
console.log(`Downloaded ${formatNumber(newBuffers.map(x => x.length).reduce((a, b) => a + b, 0))}B in ${formatTime(Date.now() - time)}`);
|
|
260
254
|
if (newBuffers.length !== group.file.length) {
|
|
261
255
|
throw new Error(`Expected ${group.file.length} buffers, got ${newBuffers.length}`);
|
|
262
256
|
}
|
|
@@ -269,7 +263,7 @@ export class ArchiveViewer extends qreact.Component {
|
|
|
269
263
|
console.log(`Missing buffer for ${file}, skipping`);
|
|
270
264
|
continue;
|
|
271
265
|
}
|
|
272
|
-
console.log(`Downloaded ${file}, ${formatNumber(buffer.length)}B`);
|
|
266
|
+
//console.log(`Downloaded ${file}, ${formatNumber(buffer.length)}B`);
|
|
273
267
|
archiveViewerDiskCache().set(file, buffer);
|
|
274
268
|
bufferObjs.push({ path: file, buffer });
|
|
275
269
|
}
|
|
@@ -277,7 +271,7 @@ export class ArchiveViewer extends qreact.Component {
|
|
|
277
271
|
this.state.progress.filesDownloaded.value += group.file.length;
|
|
278
272
|
this.state.progress.bytesDownloaded += newBuffers.reduce((sum, buffer) => sum + (buffer?.buffer.byteLength || 0), 0);
|
|
279
273
|
});
|
|
280
|
-
}
|
|
274
|
+
}
|
|
281
275
|
Querysub.commitLocal(() => {
|
|
282
276
|
this.state.progress.filesDownloaded.endTime = Date.now();
|
|
283
277
|
});
|
|
@@ -720,7 +714,7 @@ class ProcessProgress extends qreact.Component<{
|
|
|
720
714
|
|
|
721
715
|
const EMPTY_BUFFER = Buffer.alloc(0);
|
|
722
716
|
class ArchiveViewerControllerBase {
|
|
723
|
-
public async getValuePaths(authority:
|
|
717
|
+
public async getValuePaths(authority: AuthoritySpec): Promise<string[]> {
|
|
724
718
|
return pathValueArchives.getValuePaths(authority);
|
|
725
719
|
}
|
|
726
720
|
public async getValuePathSizes(files: string[]): Promise<number[]> {
|
|
@@ -17,7 +17,7 @@ import { pathValueSerializer } from "../../-h-path-value-serialize/PathValueSeri
|
|
|
17
17
|
import { getSingleSizeEstimate } from "../../5-diagnostics/shared";
|
|
18
18
|
import { ButtonSelector } from "../../library-components/ButtonSelector";
|
|
19
19
|
|
|
20
|
-
const archiveTreeScale = createURLSync("archiveTreeScale", "
|
|
20
|
+
const archiveTreeScale = createURLSync("archiveTreeScale", "count" as "count" | "equal" | "size");
|
|
21
21
|
const archiveTreeSelected = createURLSync("archiveTreeSelected", "");
|
|
22
22
|
const archiveMinPx = createURLSync("archiveMinPx", 50);
|
|
23
23
|
const forceWildcard = createURLSync("forceWildcard", "");
|
|
@@ -82,9 +82,6 @@ export class ArchiveViewerTree extends qreact.Component<{
|
|
|
82
82
|
if (scaleType === "size") {
|
|
83
83
|
total = node.totalSizeEstimate;
|
|
84
84
|
}
|
|
85
|
-
if (node.values.length === 1 && total > 1000 * 1000 * 1000) {
|
|
86
|
-
debugger;
|
|
87
|
-
}
|
|
88
85
|
countInfo += `(${formatNumber(node.values.length)} /// ${sumFormatter(total)}) `;
|
|
89
86
|
} else {
|
|
90
87
|
countInfo += `(${formatNumber(node.values.length)}) `;
|
|
@@ -423,6 +420,11 @@ export class ArchiveViewerTree extends qreact.Component<{
|
|
|
423
420
|
<ATag values={[{ param: viewMode, value: "table" }, { param: filtersURL, value: exactFilter }]}>
|
|
424
421
|
View Exact Path
|
|
425
422
|
</ATag>
|
|
423
|
+
<Button onClick={() => {
|
|
424
|
+
void navigator.clipboard.writeText(archiveTreeSelected.value);
|
|
425
|
+
}}>
|
|
426
|
+
Copy To Clipboard
|
|
427
|
+
</Button>
|
|
426
428
|
{lastFocused && !lite && <ATag values={[{ param: viewMode, value: "inspect" }, getSetArchiveSelected(lastFocused)]}>
|
|
427
429
|
Inspect Latest
|
|
428
430
|
</ATag>}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ExternalRenderClass, qreact } from "../../4-dom/qreact";
|
|
2
2
|
import { throttleRerender } from "../../functional/throttleRerender";
|
|
3
|
-
import { pathWatcher } from "../../0-path-value-core/
|
|
3
|
+
import { pathWatcher } from "../../0-path-value-core/PathWatcher";
|
|
4
4
|
import { css } from "../../4-dom/css";
|
|
5
5
|
import { TimeRangeView } from "./TimeRangeView";
|
|
6
6
|
import { URLParam } from "../../library-components/URLParam";
|
|
@@ -13,7 +13,7 @@ import { managementPageURL, showingManagementURL } from "../../diagnostics/manag
|
|
|
13
13
|
import { watchShowLocalPaths, watchValueType } from "./LocalWatchViewer";
|
|
14
14
|
import { filtersURL, viewMode } from "./archiveViewerShared";
|
|
15
15
|
import { proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
|
|
16
|
-
import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/
|
|
16
|
+
import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/PathRouter";
|
|
17
17
|
|
|
18
18
|
const historySecondsURL = new URLParam("historySeconds", 10);
|
|
19
19
|
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
import { PathValue, ReadLock, authorityStorage, epochTime
|
|
2
|
-
import {
|
|
1
|
+
import { PathValue, ReadLock, authorityStorage, epochTime } from "../../0-path-value-core/pathValueCore";
|
|
2
|
+
import { pathWatcher } from "../../0-path-value-core/PathWatcher";
|
|
3
|
+
import { PathValueProxyWatcher, proxyWatcher } from "../../2-proxy/PathValueProxyWatcher";
|
|
3
4
|
import { ExternalRenderClass, qreact } from "../../4-dom/qreact";
|
|
4
|
-
import { Querysub } from "../../4-querysub/Querysub";
|
|
5
5
|
import { Table } from "../../5-diagnostics/Table";
|
|
6
6
|
import { appendToPathStr, getPathFromStr, getPathStr } from "../../path";
|
|
7
7
|
import { css } from "typesafecss";
|
|
8
8
|
import { filtersURL, formatPath, viewMode } from "./archiveViewerShared";
|
|
9
|
-
import { nextId
|
|
9
|
+
import { nextId } from "socket-function/src/misc";
|
|
10
10
|
import { ArchiveViewerFilterUI, filterDatums } from "./archiveViewerFilter";
|
|
11
11
|
import { ArchiveViewerTree } from "./ArchiveViewerTree";
|
|
12
12
|
import { Button } from "../../library-components/Button";
|
|
@@ -15,12 +15,10 @@ import { Anchor } from "../../library-components/ATag";
|
|
|
15
15
|
import { ButtonSelector } from "../../library-components/ButtonSelector";
|
|
16
16
|
import { throttleRerender } from "../../functional/throttleRerender";
|
|
17
17
|
import { InputLabel, InputLabelURL } from "../../library-components/InputLabel";
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import { TimeRange, TimeRangeView, getTimeRangeStats, getUniqueTimeByPath, normalizeTimeRanges } from "./TimeRangeView";
|
|
21
|
-
import { ComponentSyncStatsBase } from "./ComponentSyncStats";
|
|
18
|
+
import { formatTime } from "socket-function/src/formatting/format";
|
|
19
|
+
import { TimeRange, TimeRangeView, getUniqueTimeByPath, normalizeTimeRanges } from "./TimeRangeView";
|
|
22
20
|
import { addEpsilons } from "../../bits";
|
|
23
|
-
import {
|
|
21
|
+
import { LOCAL_DOMAIN_PATH } from "../../0-path-value-core/PathRouter";
|
|
24
22
|
|
|
25
23
|
export const watchValueType = new URLParam("valueType", "watch" as "watch" | "component");
|
|
26
24
|
export const watchShowLocalPaths = new URLParam("showLocalPaths", false);
|
|
@@ -3,11 +3,10 @@ module.allowclient = true;
|
|
|
3
3
|
import { ArchiveSnapshotOverview, ArchiveSnapshotRead, getSnapshot, getSnapshotList, loadSnapshot, saveSnapshot } from "../../0-path-value-core/archiveLocks/archiveSnapshots";
|
|
4
4
|
import { qreact } from "../../4-dom/qreact";
|
|
5
5
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
6
|
-
import { ValueAuditController } from "../../5-diagnostics/memoryValueAudit";
|
|
7
6
|
import { getBrowserUrlNode } from "../../-f-node-discovery/NodeDiscovery";
|
|
8
7
|
import { Table } from "../../5-diagnostics/Table";
|
|
9
8
|
import { css } from "typesafecss";
|
|
10
|
-
import { t } from "../../4-querysub/Querysub";
|
|
9
|
+
import { Querysub, t } from "../../4-querysub/Querysub";
|
|
11
10
|
import { ATag, Anchor } from "../../library-components/ATag";
|
|
12
11
|
import { URLParam } from "../../library-components/URLParam";
|
|
13
12
|
import { formatDateTime, formatNumber } from "socket-function/src/formatting/format";
|
|
@@ -29,7 +28,8 @@ const selectedSnapshot = new URLParam("selectedSnapshot", "");
|
|
|
29
28
|
export class SnapshotViewer extends qreact.Component {
|
|
30
29
|
state = t.state({
|
|
31
30
|
// file => true
|
|
32
|
-
expanded: t.lookup(t.boolean)
|
|
31
|
+
expanded: t.lookup(t.boolean),
|
|
32
|
+
saving: t.boolean(false),
|
|
33
33
|
});
|
|
34
34
|
render() {
|
|
35
35
|
let controller = SnapshotViewerSynced(getBrowserUrlNode());
|
|
@@ -39,7 +39,7 @@ export class SnapshotViewer extends qreact.Component {
|
|
|
39
39
|
if (!file || typeof file !== "string") {
|
|
40
40
|
return (
|
|
41
41
|
<div class={css.pad2(10).vbox(10)}>
|
|
42
|
-
<h1>Snapshots are taken
|
|
42
|
+
<h1>Snapshots are taken before every change. "zombie" means the file exists, but it has no confirm (so it will be ignored).</h1>
|
|
43
43
|
<Table
|
|
44
44
|
rows={snapshotList}
|
|
45
45
|
columns={{
|
|
@@ -56,10 +56,16 @@ export class SnapshotViewer extends qreact.Component {
|
|
|
56
56
|
</ATag>
|
|
57
57
|
{context?.row?.file === "live" &&
|
|
58
58
|
<Button onClick={async () => {
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
this.state.saving = true;
|
|
60
|
+
try {
|
|
61
|
+
await SnapshotViewerSynced(getBrowserUrlNode()).saveLiveSnapshot.promise();
|
|
62
|
+
} finally {
|
|
63
|
+
Querysub.localCommit(() => {
|
|
64
|
+
this.state.saving = false;
|
|
65
|
+
});
|
|
66
|
+
}
|
|
61
67
|
}}>
|
|
62
|
-
Save
|
|
68
|
+
{this.state.saving ? "Saving..." : "Save"}
|
|
63
69
|
</Button>
|
|
64
70
|
}
|
|
65
71
|
</div>;
|
|
@@ -155,9 +161,9 @@ async function seriousConfirm(config: {
|
|
|
155
161
|
</pre>}
|
|
156
162
|
onChangeValue={value => {
|
|
157
163
|
if (value === "confirm") {
|
|
158
|
-
modal.close();
|
|
159
164
|
resolve(true);
|
|
160
165
|
}
|
|
166
|
+
modal.close();
|
|
161
167
|
}}
|
|
162
168
|
/>
|
|
163
169
|
</FullscreenModal>
|
|
@@ -175,9 +181,6 @@ class SnapshotViewerControllerBase {
|
|
|
175
181
|
}): Promise<void> {
|
|
176
182
|
await loadSnapshot({
|
|
177
183
|
overview: config.overview,
|
|
178
|
-
async audit(nodeId) {
|
|
179
|
-
await ValueAuditController.nodes[nodeId].diskAuditNow();
|
|
180
|
-
},
|
|
181
184
|
});
|
|
182
185
|
}
|
|
183
186
|
|
|
@@ -207,4 +210,12 @@ export const SnapshotViewerController = SocketFunction.register(
|
|
|
207
210
|
noAutoExpose: true,
|
|
208
211
|
}
|
|
209
212
|
);
|
|
210
|
-
const SnapshotViewerSynced = getSyncedController(SnapshotViewerController
|
|
213
|
+
const SnapshotViewerSynced = getSyncedController(SnapshotViewerController, {
|
|
214
|
+
reads: {
|
|
215
|
+
getSnapshotList: ["snapshots"],
|
|
216
|
+
getSnapshot: ["snapshots"],
|
|
217
|
+
},
|
|
218
|
+
writes: {
|
|
219
|
+
saveLiveSnapshot: ["snapshots"],
|
|
220
|
+
}
|
|
221
|
+
});
|
|
@@ -12,7 +12,7 @@ import { css } from "typesafecss";
|
|
|
12
12
|
|
|
13
13
|
export const startTime = createURLSync("archiveStartTime", 0);
|
|
14
14
|
export const endTime = createURLSync("archiveEndTime", +new Date("2100/01/01 00:00:00"));
|
|
15
|
-
export const archiveFileValue = createURLSync("archiveFileValue", "
|
|
15
|
+
export const archiveFileValue = createURLSync("archiveFileValue", "datums" as "files" | "datums" | "bytes");
|
|
16
16
|
export const filtersURL = createURLSync("archivefilters2", {} as { [key: string]: FilterPart });
|
|
17
17
|
|
|
18
18
|
export const liveWatchPaths = createURLSync("liveWatchPaths", {} as { [path: string]: true });
|