querysub 0.725.0 → 0.727.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/backfill-log-categories.js +7 -0
- package/package.json +8 -2
- package/src/-a-archives/archiveCache.ts +7 -7
- package/src/-a-archives/archivesDisk.ts +20 -7
- package/src/0-path-value-core/AuthorityLookup.ts +113 -19
- package/src/deployManager/components/DeployPage.tsx +2 -2
- package/src/deployManager/components/ServiceDetailPage.tsx +17 -4
- package/src/deployManager/components/ServicesListPage.tsx +61 -8
- package/src/deployManager/components/deployButtons.tsx +166 -101
- package/src/deployManager/machineSchema.ts +13 -2
- package/src/deployManager/publishQuerysub.ts +121 -49
- package/src/deployManager/serviceCategories.ts +90 -1
- package/src/diagnostics/logs/IndexedLogs/BufferIndexLogsOptimizationConstants.ts +2 -0
- package/src/diagnostics/logs/IndexedLogs/FindProgressTracker.ts +2 -1
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +40 -1
- package/src/diagnostics/logs/IndexedLogs/LogStreamer.ts +6 -2
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +2 -6
- package/src/diagnostics/logs/IndexedLogs/TimeFileTree.ts +34 -0
- package/src/diagnostics/logs/IndexedLogs/limitPublicLogsSize.ts +119 -0
- package/src/diagnostics/logs/IndexedLogs/moveIndexLogsToPublic.ts +13 -5
- package/src/diagnostics/logs/IndexedLogs/moveLogsEntry.ts +2 -0
- package/src/diagnostics/logs/errorNotifications2/ErrorNotificationPage.tsx +93 -21
- package/src/diagnostics/logs/errorNotifications2/openRouterHelper.ts +4 -2
- package/src/diagnostics/logs/errorNotifications2/suppressionPatternMatcher.ts +1 -1
- package/src/diagnostics/logs/logCategories/LogBreakdownPage.tsx +421 -0
- package/src/diagnostics/logs/logCategories/logCategories.ts +273 -0
- package/src/diagnostics/managementPages.tsx +6 -0
- package/src/diagnostics/network/backfillLogCategoriesMain.ts +128 -0
- package/src/diagnostics/network/deleteOldLogsMain.ts +57 -0
- package/src/diagnostics/network/listLogFilesMain.ts +60 -0
- package/src/diagnostics/network/logFilesMain.ts +13 -15
- package/src/diagnostics/network/probeLogDeleteMain.ts +71 -0
- package/src/diagnostics/network/watchServiceMain.ts +106 -0
- package/src/fs.ts +29 -0
- package/src/misc/moveFolderContents.ts +67 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.727.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -33,6 +33,11 @@
|
|
|
33
33
|
"port-mappings": "yarn typenode ./src/diagnostics/network/portMappingsMain.ts",
|
|
34
34
|
"log-search": "yarn typenode ./src/diagnostics/network/logSearchMain.ts",
|
|
35
35
|
"log-files": "yarn typenode ./src/diagnostics/network/logFilesMain.ts",
|
|
36
|
+
"list-log-files": "yarn typenode ./src/diagnostics/network/listLogFilesMain.ts",
|
|
37
|
+
"backfill-log-categories": "yarn typenode ./src/diagnostics/network/backfillLogCategoriesMain.ts --public",
|
|
38
|
+
"probe-log-delete": "yarn typenode ./src/diagnostics/network/probeLogDeleteMain.ts --public",
|
|
39
|
+
"delete-old-logs": "yarn typenode ./src/diagnostics/network/deleteOldLogsMain.ts --public",
|
|
40
|
+
"watch-service": "yarn typenode ./src/diagnostics/network/watchServiceMain.ts",
|
|
36
41
|
"remote-flushlogs": "yarn typenode ./src/diagnostics/network/remoteFlushLogsMain.ts",
|
|
37
42
|
"publish-querysub": "yarn typenode ./src/deployManager/publishQuerysubMain.ts",
|
|
38
43
|
"parameters-timeline": "yarn typenode ./src/deployManager/parametersTimelineMain.ts",
|
|
@@ -58,6 +63,7 @@
|
|
|
58
63
|
"join-public": "./bin/join-public.js",
|
|
59
64
|
"movelogs": "./bin/movelogs.js",
|
|
60
65
|
"flushlogs": "./bin/flushlogs.js",
|
|
66
|
+
"backfill-log-categories": "./bin/backfill-log-categories.js",
|
|
61
67
|
"addsuperuser": "./bin/addsuperuser.js",
|
|
62
68
|
"error-watch": "./bin/error-watch.js",
|
|
63
69
|
"error-watch-public": "./bin/error-watch-public.js",
|
|
@@ -89,7 +95,7 @@
|
|
|
89
95
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
90
96
|
"pako": "^2.1.0",
|
|
91
97
|
"peggy": "^5.0.6",
|
|
92
|
-
"sliftutils": "^1.9.
|
|
98
|
+
"sliftutils": "^1.9.20",
|
|
93
99
|
"socket-function": "^1.3.1",
|
|
94
100
|
"terser": "^5.31.0",
|
|
95
101
|
"typenode": "^6.6.3",
|
|
@@ -9,6 +9,9 @@ import { sha256 } from "js-sha256";
|
|
|
9
9
|
import { Args } from "socket-function/src/types";
|
|
10
10
|
import { SizeLimiter } from "../diagnostics/SizeLimiter";
|
|
11
11
|
import { isPublic } from "../config";
|
|
12
|
+
import { getSharedFolderPath, moveIntoFolder } from "../fs";
|
|
13
|
+
|
|
14
|
+
const CACHE_FOLDER = "archive-cache";
|
|
12
15
|
|
|
13
16
|
const SIZE_LIMIT = lazy(() => new SizeLimiter({
|
|
14
17
|
diskRoot: getCacheDir(),
|
|
@@ -37,13 +40,8 @@ function bumpAtimeIfStale(path: string) {
|
|
|
37
40
|
});
|
|
38
41
|
}
|
|
39
42
|
|
|
40
|
-
const getCacheDir = lazy(() =>
|
|
41
|
-
|
|
42
|
-
if (!fs.existsSync(dir)) {
|
|
43
|
-
fs.mkdirSync(dir, { recursive: true });
|
|
44
|
-
}
|
|
45
|
-
return dir;
|
|
46
|
-
});
|
|
43
|
+
const getCacheDir = lazy(() => getSharedFolderPath(CACHE_FOLDER));
|
|
44
|
+
const moveCacheDir = () => moveIntoFolder({ oldFolder: os.homedir() + "/.querysub-cache/", folder: getCacheDir() });
|
|
47
45
|
|
|
48
46
|
const TEMP_SUFFIX = ".tmp";
|
|
49
47
|
const TEMP_THRESHOLD = timeInHour * 3;
|
|
@@ -68,6 +66,8 @@ export function getArchiveCachePath(archives: CacheKeySource, key: string): stri
|
|
|
68
66
|
}
|
|
69
67
|
|
|
70
68
|
const getDiskMetricsBase = async () => {
|
|
69
|
+
await moveCacheDir();
|
|
70
|
+
await fs.promises.mkdir(getCacheDir(), { recursive: true });
|
|
71
71
|
ensureDiskMetricsUpdated();
|
|
72
72
|
|
|
73
73
|
let fileSizes: {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { isNode } from "socket-function/src/misc";
|
|
2
2
|
import os from "os";
|
|
3
|
-
import { fsExistsAsync } from "../fs";
|
|
3
|
+
import { fsExistsAsync, getSharedFolderPath, moveIntoFolder } from "../fs";
|
|
4
4
|
import fs from "fs";
|
|
5
5
|
import { blue, red, yellow } from "socket-function/src/formatting/logColors";
|
|
6
6
|
import { delay } from "socket-function/src/batching";
|
|
@@ -15,13 +15,19 @@ export function disableStorage() {
|
|
|
15
15
|
storageDisabled = true;
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
const ARCHIVES_HOME_FOLDER = "archives";
|
|
19
|
+
|
|
18
20
|
let maxFileNameLength = 260;
|
|
19
21
|
let maxFilePartLength = 255;
|
|
20
22
|
|
|
21
23
|
class ArchivesDisk {
|
|
22
24
|
public LAG = 0;
|
|
23
25
|
// Must end with "/"
|
|
24
|
-
|
|
26
|
+
private LOCAL_ARCHIVE_FOLDER: string;
|
|
27
|
+
|
|
28
|
+
public constructor(private config: { folder: string; beforeFirstAccess?: () => Promise<void> }) {
|
|
29
|
+
this.LOCAL_ARCHIVE_FOLDER = config.folder;
|
|
30
|
+
}
|
|
25
31
|
|
|
26
32
|
private logging = false;
|
|
27
33
|
public enableLogging() {
|
|
@@ -37,8 +43,8 @@ class ArchivesDisk {
|
|
|
37
43
|
}
|
|
38
44
|
|
|
39
45
|
private initFinished = false;
|
|
40
|
-
|
|
41
|
-
|
|
46
|
+
private init = lazy(async () => {
|
|
47
|
+
await this.config.beforeFirstAccess?.();
|
|
42
48
|
if (isNode() && process.platform === "win32") {
|
|
43
49
|
try {
|
|
44
50
|
let longPart = "LONG_FILE_PATH_TEST_" + Array(210 - "LONG_FILE_PATH_TEST_".length).fill("_").join("");
|
|
@@ -306,6 +312,7 @@ class ArchivesDisk {
|
|
|
306
312
|
writeTime: number;
|
|
307
313
|
size: number;
|
|
308
314
|
} | undefined> {
|
|
315
|
+
await this.init();
|
|
309
316
|
let path = escapeFileName(pathInput);
|
|
310
317
|
try {
|
|
311
318
|
let stats = await fs.promises.stat(this.LOCAL_ARCHIVE_FOLDER + path);
|
|
@@ -333,6 +340,7 @@ class ArchivesDisk {
|
|
|
333
340
|
target: Archives;
|
|
334
341
|
targetPath: string;
|
|
335
342
|
}) {
|
|
343
|
+
await this.init();
|
|
336
344
|
let { path, target, targetPath } = config;
|
|
337
345
|
while (true) {
|
|
338
346
|
let targetUnwrapped = target.getBaseArchives?.();
|
|
@@ -342,6 +350,7 @@ class ArchivesDisk {
|
|
|
342
350
|
}
|
|
343
351
|
|
|
344
352
|
if (target instanceof ArchivesDisk) {
|
|
353
|
+
await target.init();
|
|
345
354
|
path = escapeFileName(path);
|
|
346
355
|
targetPath = escapeFileName(targetPath);
|
|
347
356
|
await this.ensureDirsExist(path);
|
|
@@ -363,6 +372,7 @@ class ArchivesDisk {
|
|
|
363
372
|
target: Archives;
|
|
364
373
|
targetPath: string;
|
|
365
374
|
}) {
|
|
375
|
+
await this.init();
|
|
366
376
|
let { path, target, targetPath } = config;
|
|
367
377
|
while (true) {
|
|
368
378
|
let targetUnwrapped = target.getBaseArchives?.();
|
|
@@ -372,6 +382,7 @@ class ArchivesDisk {
|
|
|
372
382
|
}
|
|
373
383
|
|
|
374
384
|
if (target instanceof ArchivesDisk) {
|
|
385
|
+
await target.init();
|
|
375
386
|
path = escapeFileName(path);
|
|
376
387
|
targetPath = escapeFileName(targetPath);
|
|
377
388
|
await this.ensureDirsExist(path);
|
|
@@ -425,11 +436,13 @@ function unescapeFileName(fileName: string): string {
|
|
|
425
436
|
}
|
|
426
437
|
|
|
427
438
|
export const getArchivesHome = cache((domain: string): Archives => {
|
|
428
|
-
|
|
439
|
+
let folder = "";
|
|
440
|
+
let beforeFirstAccess: (() => Promise<void>) | undefined;
|
|
429
441
|
|
|
430
442
|
if (isNode()) {
|
|
431
|
-
|
|
443
|
+
folder = getSharedFolderPath(ARCHIVES_HOME_FOLDER + "/" + domain);
|
|
444
|
+
beforeFirstAccess = () => moveIntoFolder({ oldFolder: os.homedir() + "/querysub/" + domain + "/", folder });
|
|
432
445
|
}
|
|
433
446
|
|
|
434
|
-
return
|
|
447
|
+
return new ArchivesDisk({ folder, beforeFirstAccess });
|
|
435
448
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
1
|
+
import { sha256Hash, sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
2
2
|
import { cache, lazy } from "socket-function/src/caching";
|
|
3
3
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
4
4
|
import { delay, runInSerial, runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
@@ -10,7 +10,7 @@ import { getPathStr1 } from "../path";
|
|
|
10
10
|
import { logErrors, timeoutToError } from "../errors";
|
|
11
11
|
import { AuthoritySpec, parsePrefixMatcher } from "./PathRouter";
|
|
12
12
|
|
|
13
|
-
import { formatTime } from "socket-function/src/formatting/format";
|
|
13
|
+
import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
|
|
14
14
|
import { getAllAuthoritySpec, getEmptyAuthoritySpec } from "./PathRouterServerAuthoritySpec";
|
|
15
15
|
import { getScheduledShutdownTime, watchScheduledShutdown } from "../-g-core-values/scheduledShutdown";
|
|
16
16
|
|
|
@@ -29,9 +29,12 @@ const SYNC_JITTER_WINDOW = timeInSecond * 10;
|
|
|
29
29
|
// Nodes this close to their scheduled shutdown are avoided as sources (only used if nothing else can satisfy the request).
|
|
30
30
|
const SHUTDOWN_AVOID_WINDOW = timeInMinute * 5;
|
|
31
31
|
|
|
32
|
+
const MISSING_FUNCTION_ERROR_TEXT = "Function getSetNetworkSyncHash not exposed in";
|
|
33
|
+
|
|
32
34
|
export type AuthorityEntry = {
|
|
33
35
|
nodeId: string;
|
|
34
36
|
authoritySpec: AuthoritySpec;
|
|
37
|
+
specHash?: string;
|
|
35
38
|
entryReceivedTime: number;
|
|
36
39
|
isReady: boolean;
|
|
37
40
|
// This can be far in the future (a switchover scheduled weeks out), so routing doesn't avoid the node immediately — only once within SHUTDOWN_AVOID_WINDOW (see nodeIsShuttingDownSoon).
|
|
@@ -41,6 +44,11 @@ type AuthorityTopology = {
|
|
|
41
44
|
// nodeId =>
|
|
42
45
|
nodes: Map<string, AuthorityEntry>;
|
|
43
46
|
};
|
|
47
|
+
export type AuthorityHashSync = {
|
|
48
|
+
specHash: string;
|
|
49
|
+
isReady: boolean;
|
|
50
|
+
scheduledShutdownTime?: number;
|
|
51
|
+
};
|
|
44
52
|
|
|
45
53
|
class AuthorityLookup {
|
|
46
54
|
private topology: AuthorityTopology = {
|
|
@@ -87,14 +95,14 @@ class AuthorityLookup {
|
|
|
87
95
|
this.setSpec = true;
|
|
88
96
|
|
|
89
97
|
this.ourSpec = spec;
|
|
90
|
-
this.updatePaths(getOwnNodeId(), spec, false);
|
|
98
|
+
this.updatePaths({ nodeId: getOwnNodeId(), spec, specHash: this.getOurSpecHash(), isReady: false });
|
|
91
99
|
|
|
92
100
|
await this.syncAllNow();
|
|
93
101
|
}
|
|
94
102
|
public async setIsReady() {
|
|
95
103
|
if (!this.setSpec) throw new Error("Cannot call setIsReady without calling setOurPaths first.");
|
|
96
104
|
this.ourIsReady = true;
|
|
97
|
-
this.updatePaths(getOwnNodeId(), this.ourSpec, true);
|
|
105
|
+
this.updatePaths({ nodeId: getOwnNodeId(), spec: this.ourSpec, specHash: this.getOurSpecHash(), isReady: true });
|
|
98
106
|
await this.syncAllNow();
|
|
99
107
|
}
|
|
100
108
|
|
|
@@ -146,20 +154,48 @@ class AuthorityLookup {
|
|
|
146
154
|
});
|
|
147
155
|
|
|
148
156
|
|
|
149
|
-
private updatePaths(
|
|
157
|
+
private updatePaths(config: {
|
|
158
|
+
nodeId: string;
|
|
159
|
+
spec: AuthoritySpec;
|
|
160
|
+
specHash?: string;
|
|
161
|
+
isReady: boolean;
|
|
162
|
+
scheduledShutdownTime?: number;
|
|
163
|
+
}) {
|
|
164
|
+
let { nodeId, spec, specHash, isReady, scheduledShutdownTime } = config;
|
|
165
|
+
let prevEntry = this.topology.nodes.get(nodeId);
|
|
166
|
+
let specUnchanged = !!prevEntry && (prevEntry.authoritySpec === spec || JSON.stringify(prevEntry.authoritySpec) === JSON.stringify(spec));
|
|
150
167
|
let entry: AuthorityEntry = {
|
|
151
168
|
nodeId,
|
|
152
|
-
|
|
169
|
+
// Try to preserve === property where possible, so we can use it as a cache key
|
|
170
|
+
authoritySpec: prevEntry && specUnchanged ? prevEntry.authoritySpec : spec,
|
|
171
|
+
specHash,
|
|
153
172
|
entryReceivedTime: Date.now(),
|
|
154
173
|
isReady,
|
|
155
174
|
scheduledShutdownTime,
|
|
156
175
|
};
|
|
157
|
-
let prevEntry = this.topology.nodes.get(nodeId);
|
|
158
|
-
// Try to preserve === property where possible, so we can use it as a cache key
|
|
159
|
-
if (prevEntry && JSON.stringify(prevEntry.authoritySpec) === JSON.stringify(spec)) {
|
|
160
|
-
entry.authoritySpec = prevEntry.authoritySpec;
|
|
161
|
-
}
|
|
162
176
|
this.topology.nodes.set(nodeId, entry);
|
|
177
|
+
if (prevEntry && specUnchanged && prevEntry.isReady === isReady && prevEntry.scheduledShutdownTime === scheduledShutdownTime) return;
|
|
178
|
+
console.log(`Authority topology entry ${prevEntry ? "changed" : "added"} for ${nodeId} (${this.topology.nodes.size} entries)`, {
|
|
179
|
+
prevSpecHash: prevEntry?.specHash,
|
|
180
|
+
specHash,
|
|
181
|
+
prevPrefixes: prevEntry?.authoritySpec.prefixes.map(x => x.originalPrefix),
|
|
182
|
+
prefixes: specUnchanged ? "unchanged" : spec.prefixes.map(x => x.originalPrefix),
|
|
183
|
+
prevIsReady: prevEntry?.isReady,
|
|
184
|
+
isReady,
|
|
185
|
+
prevScheduledShutdownTime: prevEntry?.scheduledShutdownTime && formatDateTime(prevEntry.scheduledShutdownTime),
|
|
186
|
+
scheduledShutdownTime: scheduledShutdownTime && formatDateTime(scheduledShutdownTime),
|
|
187
|
+
});
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
private getOurSpecHash(): string {
|
|
191
|
+
return sha256Hash(JSON.stringify(this.getOurSpec()));
|
|
192
|
+
}
|
|
193
|
+
private getOurHashSync(): AuthorityHashSync {
|
|
194
|
+
return {
|
|
195
|
+
specHash: this.getOurSpecHash(),
|
|
196
|
+
isReady: this.ourIsReady,
|
|
197
|
+
scheduledShutdownTime: getScheduledShutdownTime(),
|
|
198
|
+
};
|
|
163
199
|
}
|
|
164
200
|
|
|
165
201
|
public getScheduledShutdownTime(nodeId: string): number | undefined {
|
|
@@ -180,7 +216,7 @@ class AuthorityLookup {
|
|
|
180
216
|
scheduledShutdownTime?: number;
|
|
181
217
|
}> {
|
|
182
218
|
let nodeId = await IdentityController_getCurrentReconnectNodeIdAssert();
|
|
183
|
-
this.updatePaths(nodeId, config.spec, config.isReady, config.scheduledShutdownTime);
|
|
219
|
+
this.updatePaths({ nodeId, spec: config.spec, isReady: config.isReady, scheduledShutdownTime: config.scheduledShutdownTime });
|
|
184
220
|
console.info(`Received network sync, returning our data`, {
|
|
185
221
|
nodeId,
|
|
186
222
|
otherSpec: config.spec,
|
|
@@ -195,8 +231,31 @@ class AuthorityLookup {
|
|
|
195
231
|
};
|
|
196
232
|
}
|
|
197
233
|
|
|
234
|
+
public async getSetNetworkSyncHash(config: AuthorityHashSync): Promise<AuthorityHashSync> {
|
|
235
|
+
let nodeId = await IdentityController_getCurrentReconnectNodeIdAssert();
|
|
236
|
+
await this.applyHashSync(nodeId, config);
|
|
237
|
+
return this.getOurHashSync();
|
|
238
|
+
}
|
|
198
239
|
|
|
240
|
+
public async getAuthoritySpec(): Promise<AuthorityHashSync & { spec: AuthoritySpec }> {
|
|
241
|
+
return { spec: this.getOurSpec(), ...this.getOurHashSync() };
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
private async applyHashSync(nodeId: string, other: AuthorityHashSync) {
|
|
245
|
+
let prevEntry = this.topology.nodes.get(nodeId);
|
|
246
|
+
if (prevEntry && prevEntry.specHash === other.specHash) {
|
|
247
|
+
this.updatePaths({ nodeId, spec: prevEntry.authoritySpec, specHash: other.specHash, isReady: other.isReady, scheduledShutdownTime: other.scheduledShutdownTime });
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
let full = await timeoutToError(
|
|
251
|
+
CALL_TIMEOUT,
|
|
252
|
+
AuthorityLookupController.nodes[nodeId].getAuthoritySpec(),
|
|
253
|
+
() => new Error(`Timeout calling getAuthoritySpec for ${nodeId}`)
|
|
254
|
+
);
|
|
255
|
+
this.updatePaths({ nodeId, spec: full.spec, specHash: full.specHash, isReady: full.isReady, scheduledShutdownTime: full.scheduledShutdownTime });
|
|
256
|
+
}
|
|
199
257
|
|
|
258
|
+
private legacySyncNodeIds = new Set<string>();
|
|
200
259
|
|
|
201
260
|
private connectTo = cache((nodeId: string) => {
|
|
202
261
|
let stopObj = { stop: false };
|
|
@@ -212,6 +271,7 @@ class AuthorityLookup {
|
|
|
212
271
|
const onNodeRemoved = () => {
|
|
213
272
|
this.removeNode(nodeId, "node removed from node discovery");
|
|
214
273
|
stopObj.stop = true;
|
|
274
|
+
this.legacySyncNodeIds.delete(nodeId);
|
|
215
275
|
this.connectTo.clear(nodeId);
|
|
216
276
|
this.disconnectWatch.clear(nodeId);
|
|
217
277
|
};
|
|
@@ -254,15 +314,47 @@ class AuthorityLookup {
|
|
|
254
314
|
await new Promise(r => setImmediate(r));
|
|
255
315
|
await delay(1);
|
|
256
316
|
let prefixes = await this.getPrefixesForDeploy();
|
|
257
|
-
this.updatePaths(
|
|
258
|
-
nodeId
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
317
|
+
this.updatePaths({
|
|
318
|
+
nodeId,
|
|
319
|
+
spec: {
|
|
320
|
+
nodeId: nodeId,
|
|
321
|
+
prefixes: prefixes.map(p => parsePrefixMatcher(p)),
|
|
322
|
+
routeStart: 0,
|
|
323
|
+
routeEnd: 1,
|
|
324
|
+
},
|
|
325
|
+
isReady: true,
|
|
326
|
+
});
|
|
263
327
|
return;
|
|
264
328
|
}
|
|
265
329
|
|
|
330
|
+
if (!this.legacySyncNodeIds.has(nodeId)) {
|
|
331
|
+
let synced = await this.syncNodeHashed(nodeId);
|
|
332
|
+
if (synced) return;
|
|
333
|
+
this.legacySyncNodeIds.add(nodeId);
|
|
334
|
+
}
|
|
335
|
+
await this.syncNodeLegacy(nodeId);
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
private async syncNodeHashed(nodeId: string): Promise<boolean> {
|
|
339
|
+
let other: AuthorityHashSync;
|
|
340
|
+
try {
|
|
341
|
+
other = await timeoutToError(
|
|
342
|
+
CALL_TIMEOUT,
|
|
343
|
+
AuthorityLookupController.nodes[nodeId].getSetNetworkSyncHash(this.getOurHashSync()),
|
|
344
|
+
() => new Error(`Timeout calling getSetNetworkSyncHash for ${nodeId}`)
|
|
345
|
+
);
|
|
346
|
+
} catch (e: any) {
|
|
347
|
+
let message = String(e?.message || e);
|
|
348
|
+
if (!message.includes(MISSING_FUNCTION_ERROR_TEXT)) throw e;
|
|
349
|
+
console.log(`${nodeId} runs an older querysub without getSetNetworkSyncHash, falling back to networkSyncPaths for it. Error: ${message}`);
|
|
350
|
+
return false;
|
|
351
|
+
}
|
|
352
|
+
await this.applyHashSync(nodeId, other);
|
|
353
|
+
this.disconnectWatch(nodeId);
|
|
354
|
+
return true;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
private async syncNodeLegacy(nodeId: string) {
|
|
266
358
|
let otherPaths = await timeoutToError(
|
|
267
359
|
CALL_TIMEOUT,
|
|
268
360
|
AuthorityLookupController.nodes[nodeId].networkSyncPaths({
|
|
@@ -279,7 +371,7 @@ class AuthorityLookup {
|
|
|
279
371
|
ourSpec: this.ourSpec,
|
|
280
372
|
ourIsReady: this.ourIsReady,
|
|
281
373
|
});
|
|
282
|
-
this.updatePaths(nodeId, otherPaths.spec, otherPaths.isReady, otherPaths.scheduledShutdownTime);
|
|
374
|
+
this.updatePaths({ nodeId, spec: otherPaths.spec, isReady: otherPaths.isReady, scheduledShutdownTime: otherPaths.scheduledShutdownTime });
|
|
283
375
|
this.disconnectWatch(nodeId);
|
|
284
376
|
}
|
|
285
377
|
|
|
@@ -296,6 +388,8 @@ const AuthorityLookupController = SocketFunction.register(
|
|
|
296
388
|
authorityLookup,
|
|
297
389
|
() => ({
|
|
298
390
|
networkSyncPaths: {},
|
|
391
|
+
getSetNetworkSyncHash: {},
|
|
392
|
+
getAuthoritySpec: {},
|
|
299
393
|
}),
|
|
300
394
|
() => ({
|
|
301
395
|
hooks: [requiresNetworkTrustHook],
|
|
@@ -5,7 +5,7 @@ import { deployGetFunctions } from "../../4-deploy/deployFunctions";
|
|
|
5
5
|
import { MachineServiceController, deployFunctionsWithProgress } from "../machineSchema";
|
|
6
6
|
import { functionSchema, FunctionSpec } from "../../3-path-functions/PathFunctionRunner";
|
|
7
7
|
import { getDomain, isPublic } from "../../config";
|
|
8
|
-
import { RenderGitRefInfo,
|
|
8
|
+
import { RenderGitRefInfo, CommitPushButton, bigEmoji, buttonStyle } from "./deployButtons";
|
|
9
9
|
import { css } from "typesafecss";
|
|
10
10
|
import { timeInHour, timeInMinute } from "socket-function/src/misc";
|
|
11
11
|
import { t } from "../../2-proxy/schema2";
|
|
@@ -270,7 +270,7 @@ export class DeployPage extends qreact.Component {
|
|
|
270
270
|
return <div className={css.vbox(40).marginTop(20)}>
|
|
271
271
|
<div className={css.hbox(10)}>
|
|
272
272
|
{isPublic() && <h1>Don't deploy on public servers! It won't work, it will just deploy what that server is already running. TODO: Fix this.</h1>}
|
|
273
|
-
<
|
|
273
|
+
<CommitPushButton />
|
|
274
274
|
<button
|
|
275
275
|
disabled={controller.isAnyLoading()}
|
|
276
276
|
className={buttonStyle.alignSelf("stretch").center.hsl(240, 70, 50).colorhsl(0, 0, 100)
|
|
@@ -17,7 +17,8 @@ import { getPathStr2 } from "../../path";
|
|
|
17
17
|
import { ATag, Anchor } from "../../library-components/ATag";
|
|
18
18
|
import { ScrollOnMount } from "../../library-components/ScrollOnMount";
|
|
19
19
|
import { PrimitiveDisplay } from "../../diagnostics/logs/ObjectDisplay";
|
|
20
|
-
import { RenderGitRefInfo, UpdateServiceButtons, bigEmoji, buttonStyle,
|
|
20
|
+
import { RenderGitRefInfo, UpdateServiceButtons, bigEmoji, buttonStyle, confirmDeploy } from "./deployButtons";
|
|
21
|
+
import { getServiceDeployGroup } from "../serviceCategories";
|
|
21
22
|
import { TypedConfigEditor } from "../../library-components/TypedConfigEditor";
|
|
22
23
|
import { managementPageURL } from "../../diagnostics/managementPages";
|
|
23
24
|
import { getLogViewerParams } from "../../diagnostics/logs/IndexedLogs/LogViewerParams";
|
|
@@ -663,7 +664,14 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
663
664
|
disabled={this.state.isDeploying}
|
|
664
665
|
title="Deploys now; the old instances keep running for the overlap before shutting down"
|
|
665
666
|
onClick={() => {
|
|
666
|
-
|
|
667
|
+
confirmDeploy({
|
|
668
|
+
actionLabel: "🚀 Deploy",
|
|
669
|
+
group: getServiceDeployGroup(config),
|
|
670
|
+
noOverlap: false,
|
|
671
|
+
services: [config],
|
|
672
|
+
latestRef: config.parameters.gitRef,
|
|
673
|
+
onConfirm: () => this.deployConfig(makeDeployConfig()),
|
|
674
|
+
});
|
|
667
675
|
}}>
|
|
668
676
|
{this.state.isDeploying && "⏳ Deploying..." || "🚀 Deploy"}
|
|
669
677
|
</button>
|
|
@@ -672,8 +680,13 @@ export class ServiceDetailPage extends qreact.Component {
|
|
|
672
680
|
disabled={this.state.isDeploying}
|
|
673
681
|
title="Deploys now AND shuts the old instances down as soon as the new ones are up (overlap of 0)"
|
|
674
682
|
onClick={() => {
|
|
675
|
-
|
|
676
|
-
|
|
683
|
+
confirmDeploy({
|
|
684
|
+
actionLabel: "Deploy, No Overlap",
|
|
685
|
+
group: getServiceDeployGroup(config),
|
|
686
|
+
noOverlap: true,
|
|
687
|
+
services: [config],
|
|
688
|
+
latestRef: config.parameters.gitRef,
|
|
689
|
+
onConfirm: () => this.deployConfig(makeDeployConfig(0)),
|
|
677
690
|
});
|
|
678
691
|
}}>
|
|
679
692
|
Deploy, No Overlap
|
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
import { SocketFunction } from "socket-function/SocketFunction";
|
|
2
2
|
import { qreact } from "../../4-dom/qreact";
|
|
3
|
-
import { DEFAULT_OVERLAP_TIME, MachineServiceController, ServiceConfig, getLiveServiceParameters, getMachineIdList, getMachineInfoLive } from "../machineSchema";
|
|
3
|
+
import { DEFAULT_OVERLAP_TIME, MachineConfig, MachineInfo, MachineServiceController, ServiceConfig, getLiveServiceParameters, getMachineIdList, getMachineInfoLive } from "../machineSchema";
|
|
4
4
|
import { css } from "typesafecss";
|
|
5
5
|
import { t } from "../../2-proxy/schema2";
|
|
6
6
|
import { Querysub } from "../../4-querysub/Querysub";
|
|
7
7
|
import { currentViewParam, selectedServiceIdParam, selectedMachineIdParam } from "../urlParams";
|
|
8
8
|
import { managementPageURL } from "../../diagnostics/managementPages";
|
|
9
|
-
import { getServiceCategory } from "../serviceCategories";
|
|
9
|
+
import { DEPLOY_GROUPS, ServiceDeployGroup, getDeployGroupInfo, getInferredDeployGroup, getServiceCategory, getServiceDeployGroup } from "../serviceCategories";
|
|
10
|
+
import { DropdownSelector } from "../../library-components/DropdownSelector";
|
|
10
11
|
import { formatDateTimeDetailed, formatNiceDateTime, formatTime, formatVeryNiceDateTime } from "socket-function/src/formatting/format";
|
|
11
12
|
import { sort, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
12
13
|
import { cache } from "socket-function/src/caching";
|
|
13
14
|
import { Anchor } from "../../library-components/ATag";
|
|
14
15
|
import { isPublic } from "../../config";
|
|
15
16
|
// ResyncMachinesButton is still exported from deployButtons, for whenever forcing a resync becomes useful
|
|
16
|
-
import {
|
|
17
|
+
import { CommitPushButton, DeployGroupButtons, PendingDeployInfo, UpdateServiceButtons } from "./deployButtons";
|
|
17
18
|
import { isDefined, unique } from "../../misc";
|
|
18
19
|
import { formatDateJSX } from "../../misc/formatJSX";
|
|
19
20
|
import { Tools } from "./Tools";
|
|
@@ -65,6 +66,18 @@ export class ServicesListPage extends qreact.Component {
|
|
|
65
66
|
let getMachineInfo = cache((machineId: string) => getMachineInfoLive(machineId));
|
|
66
67
|
let getMachineConfig = cache((machineId: string) => controller.getMachineConfig(machineId));
|
|
67
68
|
|
|
69
|
+
let servicesByGroup = new Map<ServiceDeployGroup, typeof services[number][]>();
|
|
70
|
+
for (let entry of services) {
|
|
71
|
+
let config = entry[1];
|
|
72
|
+
let group: ServiceDeployGroup = config && getServiceDeployGroup(config) || "background";
|
|
73
|
+
let list = servicesByGroup.get(group);
|
|
74
|
+
if (!list) {
|
|
75
|
+
list = [];
|
|
76
|
+
servicesByGroup.set(group, list);
|
|
77
|
+
}
|
|
78
|
+
list.push(entry);
|
|
79
|
+
}
|
|
80
|
+
|
|
68
81
|
let keyCounts = new Map<string, number>();
|
|
69
82
|
for (let [serviceId, config] of services) {
|
|
70
83
|
let key = config?.parameters.key || "";
|
|
@@ -124,7 +137,13 @@ export class ServicesListPage extends qreact.Component {
|
|
|
124
137
|
}}>
|
|
125
138
|
+ New Service
|
|
126
139
|
</button>
|
|
127
|
-
<
|
|
140
|
+
<CommitPushButton />
|
|
141
|
+
</div>
|
|
142
|
+
<div className={css.vbox(6)}>
|
|
143
|
+
{DEPLOY_GROUPS.map(groupInfo => <DeployGroupButtons
|
|
144
|
+
group={groupInfo.group}
|
|
145
|
+
services={(servicesByGroup.get(groupInfo.group) || []).map(x => x[1]).filter(isDefined)}
|
|
146
|
+
/>)}
|
|
128
147
|
</div>
|
|
129
148
|
<div className={css.hbox(8)}>
|
|
130
149
|
<div
|
|
@@ -151,7 +170,31 @@ export class ServicesListPage extends qreact.Component {
|
|
|
151
170
|
})()}
|
|
152
171
|
</div>
|
|
153
172
|
<Tools />
|
|
154
|
-
|
|
173
|
+
{DEPLOY_GROUPS.map(groupInfo => {
|
|
174
|
+
let groupServices = servicesByGroup.get(groupInfo.group) || [];
|
|
175
|
+
if (groupServices.length === 0) return undefined;
|
|
176
|
+
return <div className={css.vbox(8)}>
|
|
177
|
+
<div className={css.vbox(2).pad2(8).fillWidth.hsl(groupInfo.hue, 40, 90).bord2(groupInfo.hue, 40, 60)}>
|
|
178
|
+
<div className={css.fontSize(18).boldStyle}>{groupInfo.emoji} {groupInfo.label} ({groupServices.length})</div>
|
|
179
|
+
<div className={css.fontSize(12).colorhsl(0, 0, 35)}>{groupInfo.description}</div>
|
|
180
|
+
</div>
|
|
181
|
+
{this.renderServices({ services: groupServices, keyCounts, serviceNodeIds, getMachineInfo, getMachineConfig, now })}
|
|
182
|
+
</div>;
|
|
183
|
+
})}
|
|
184
|
+
</div>;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
private renderServices(config: {
|
|
188
|
+
services: (readonly [string, ServiceConfig | undefined])[];
|
|
189
|
+
keyCounts: Map<string, number>;
|
|
190
|
+
serviceNodeIds: Map<string, string[]>;
|
|
191
|
+
getMachineInfo: (machineId: string) => MachineInfo | undefined;
|
|
192
|
+
getMachineConfig: (machineId: string) => MachineConfig | undefined;
|
|
193
|
+
now: number;
|
|
194
|
+
}) {
|
|
195
|
+
let { services, keyCounts, serviceNodeIds, getMachineInfo, getMachineConfig, now } = config;
|
|
196
|
+
let controller = MachineServiceController(SocketFunction.browserNodeId());
|
|
197
|
+
return <div className={css.vbox(8)}>
|
|
155
198
|
{services.map(([serviceId, config]) => {
|
|
156
199
|
if (!config) return <div key={serviceId}>Config is broken? Missing value for service? Is the file corrupted?</div>;
|
|
157
200
|
|
|
@@ -268,6 +311,17 @@ export class ServicesListPage extends qreact.Component {
|
|
|
268
311
|
}}
|
|
269
312
|
/>}
|
|
270
313
|
</div>
|
|
314
|
+
<div className={css.vbox(2).flexShrink0} title={config.info.deployGroup && "Set by hand" || `Never set, so it is worked out from the command (${getDeployGroupInfo(getInferredDeployGroup(config.parameters)).label})`}>
|
|
315
|
+
<DropdownSelector
|
|
316
|
+
title="Category"
|
|
317
|
+
value={getServiceDeployGroup(config)}
|
|
318
|
+
options={DEPLOY_GROUPS.map(groupInfo => ({ value: groupInfo.group, label: `${groupInfo.emoji} ${groupInfo.label}` }))}
|
|
319
|
+
onChange={deployGroup => {
|
|
320
|
+
Querysub.onCommitFinished(() => controller.setServiceDeployGroup.promise({ serviceId, deployGroup }));
|
|
321
|
+
}}
|
|
322
|
+
/>
|
|
323
|
+
{!config.info.deployGroup && <span className={css.fontSize(11).colorhsl(0, 0, 45)}>from command</span>}
|
|
324
|
+
</div>
|
|
271
325
|
<UpdateServiceButtons service={config} />
|
|
272
326
|
<PendingDeployInfo service={config} />
|
|
273
327
|
</div>
|
|
@@ -277,7 +331,6 @@ export class ServicesListPage extends qreact.Component {
|
|
|
277
331
|
</div>
|
|
278
332
|
;
|
|
279
333
|
})}
|
|
280
|
-
</div
|
|
281
|
-
</div>;
|
|
334
|
+
</div>;
|
|
282
335
|
}
|
|
283
|
-
}
|
|
336
|
+
}
|