querysub 0.727.0 → 0.728.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/debugging.md +13 -0
- package/package.json +2 -2
- package/src/-a-archives/archiveCache.ts +27 -0
- package/src/-a-archives/archiveCache2.ts +5 -1
- package/src/-f-node-discovery/LatencyTracking.ts +1 -3
- package/src/0-path-value-core/AuthorityLookup.ts +12 -4
- package/src/deployManager/components/deployButtons.tsx +14 -11
- package/src/deployManager/processManager.ts +11 -11
- package/src/deployManager/serviceCategories.ts +2 -1
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +64 -30
- package/src/diagnostics/network/logSearchMain.ts +43 -8
package/debugging.md
CHANGED
|
@@ -4,6 +4,19 @@ When the site won't load, the debug pages are unreachable, or nodes keep dying o
|
|
|
4
4
|
|
|
5
5
|
Every script takes `--cwd <project root>` so it runs with that project's domain, keys, and certs (for the cyoa network: `--cwd D:/repos/qs-cyoa/`). Both scripts are read-only, except that, like every server-mode querysub process, they run the node-discovery audit, which deletes node list entries whose port has provably been taken by a newer thread.
|
|
6
6
|
|
|
7
|
+
## The processes
|
|
8
|
+
|
|
9
|
+
Every service runs as its own process in its own tmux session, `<service key>-<instance index>-dply`, on the machines its config lists. `overview.md` explains the roles; these are the processes that implement them, in the order a request flows through them:
|
|
10
|
+
|
|
11
|
+
- **bootstrapper** (`bootstrapper-0-dply`, port 443 behind Cloudflare): the front-end proxy. It serves the JavaScript payload that boots the site and nothing else, so it rarely needs updating. It binds a fixed port, so it must be deployed with no overlap: if the old instance still holds 443 when the new one starts, the new one fails with `Port 443 is already in use`, never retries, and the site is down once the old one exits.
|
|
12
|
+
- **cyoa** (`cyoa-N-dply`): the client-facing querysub server. The browser connects to it; it handles the front end, security and login, and adds the client's function calls as writes. Several instances run per machine and clients pick the closest.
|
|
13
|
+
- **serverall** (`serverall-N-dply`): the path value servers. This is where the data lives. Each instance owns a slice of the paths (see sharding in `overview.md`), and every other process reads and writes data through them. If these are unhealthy, everything reads as missing or stale.
|
|
14
|
+
- **shardall** (`shardall-N-dply`): the function runners. They read the function call writes and run the functions against the data in serverall. "The function runner isn't receiving data" means shardall cannot reach or route to serverall.
|
|
15
|
+
- **storageserve** (`storageserve-0-dply`, ports 5234 and 5235): the storage core, the archive store behind the node list, service configs, trust, logs and everything else that is not a live path value. Everything depends on it and it is deployed separately and rarely.
|
|
16
|
+
- **machine-alwaysup**: each machine's deploy process. It re-reads the service configs, launches and verifies the instances, and writes the machine heartbeat. A stale machine heartbeat with services still running means this process is stuck, not the machine.
|
|
17
|
+
|
|
18
|
+
The rest are background tasks: `movelogs` (moves pending logs to public storage and categorizes them), `errorwatch`, `gc`, `thin`, `join`, `ssh-trust`, `ping`, and the app's `queue` and `gpuworker` workers. They can go down without taking the site down.
|
|
19
|
+
|
|
7
20
|
## 1. What nodes exist
|
|
8
21
|
|
|
9
22
|
```
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "querysub",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.728.0",
|
|
4
4
|
"main": "index.js",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
|
|
@@ -95,7 +95,7 @@
|
|
|
95
95
|
"node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
|
|
96
96
|
"pako": "^2.1.0",
|
|
97
97
|
"peggy": "^5.0.6",
|
|
98
|
-
"sliftutils": "^1.9.
|
|
98
|
+
"sliftutils": "^1.9.21",
|
|
99
99
|
"socket-function": "^1.3.1",
|
|
100
100
|
"terser": "^5.31.0",
|
|
101
101
|
"typenode": "^6.6.3",
|
|
@@ -65,6 +65,33 @@ export function getArchiveCachePath(archives: CacheKeySource, key: string): stri
|
|
|
65
65
|
return getCacheDir() + hash.slice(0, 16) + "." + name.slice(0, 128) + CACHE_SUFFIX;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
+
/** Serves an immutable cache hit straight from disk, without waiting for the cache folder scan that getDiskMetrics does on first use. Misses (and empty files, which are deletions) return undefined so the caller falls back to the full path. */
|
|
69
|
+
export async function readCacheFileFast(
|
|
70
|
+
archives: CacheKeySource,
|
|
71
|
+
key: string,
|
|
72
|
+
config?: { range?: { start: number; end: number } }
|
|
73
|
+
): Promise<Buffer | undefined> {
|
|
74
|
+
let path = getArchiveCachePath(archives, key);
|
|
75
|
+
let handle: fs.promises.FileHandle | undefined;
|
|
76
|
+
try {
|
|
77
|
+
handle = await fs.promises.open(path, "r");
|
|
78
|
+
let stats = await handle.stat();
|
|
79
|
+
if (stats.size === 0) return undefined;
|
|
80
|
+
let start = config?.range?.start ?? 0;
|
|
81
|
+
let end = config?.range?.end ?? stats.size;
|
|
82
|
+
let readBuffer = Buffer.alloc(end - start);
|
|
83
|
+
let read = await handle.read(readBuffer, 0, end - start, start);
|
|
84
|
+
bumpAtimeIfStale(path);
|
|
85
|
+
return readBuffer.slice(0, read.bytesRead);
|
|
86
|
+
} catch {
|
|
87
|
+
return undefined;
|
|
88
|
+
} finally {
|
|
89
|
+
if (handle) {
|
|
90
|
+
await handle.close();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
68
95
|
const getDiskMetricsBase = async () => {
|
|
69
96
|
await moveCacheDir();
|
|
70
97
|
await fs.promises.mkdir(getCacheDir(), { recursive: true });
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from "fs";
|
|
2
2
|
import { GetConfig, IArchives, SetConfig } from "sliftutils/storage/IArchives";
|
|
3
|
-
import { getArchiveCachePath, getDiskMetrics, getTempFilePath, LARGE_FILE_CHUNK } from "./archiveCache";
|
|
3
|
+
import { getArchiveCachePath, getDiskMetrics, getTempFilePath, LARGE_FILE_CHUNK, readCacheFileFast } from "./archiveCache";
|
|
4
4
|
|
|
5
5
|
let cacheArchives2Symbol = Symbol("cacheArchives2");
|
|
6
6
|
|
|
@@ -89,6 +89,10 @@ export function wrapArchivesWithCache2(archives: IArchives, rootConfig?: {
|
|
|
89
89
|
getSyncStatus: archives.getSyncStatus && (() => archives.getSyncStatus!()),
|
|
90
90
|
|
|
91
91
|
get: async (fileName: string, config?: GetConfig) => {
|
|
92
|
+
if (rootConfig?.immutable) {
|
|
93
|
+
let fast = await readCacheFileFast(archives, fileName, config);
|
|
94
|
+
if (fast) return fast;
|
|
95
|
+
}
|
|
92
96
|
let metrics = await getDiskMetrics();
|
|
93
97
|
// Get info to at least see if it is removed or not. This makes sure we don't get in
|
|
94
98
|
// a bad state, at least for immutable files (for mutable files we don't have an easy
|
|
@@ -17,7 +17,8 @@ import { getScheduledShutdownTime, watchScheduledShutdown } from "../-g-core-val
|
|
|
17
17
|
setImmediate(() => import("../3-path-functions/syncSchema"));
|
|
18
18
|
|
|
19
19
|
let NETWORK_POLL_INTERVAL = timeInMinute * 1;
|
|
20
|
-
let
|
|
20
|
+
let INITIAL_CALL_TIMEOUT = timeInSecond * 5;
|
|
21
|
+
const CONNECTED_CALL_TIMEOUT = timeInMinute;
|
|
21
22
|
|
|
22
23
|
// Every node's per-node poll is created together at discovery, so without jitter all of their
|
|
23
24
|
// networkSyncPaths calls fire in lockstep each NETWORK_POLL_INTERVAL — and when many nodes are
|
|
@@ -162,6 +163,7 @@ class AuthorityLookup {
|
|
|
162
163
|
scheduledShutdownTime?: number;
|
|
163
164
|
}) {
|
|
164
165
|
let { nodeId, spec, specHash, isReady, scheduledShutdownTime } = config;
|
|
166
|
+
this.syncedNodeIds.add(nodeId);
|
|
165
167
|
let prevEntry = this.topology.nodes.get(nodeId);
|
|
166
168
|
let specUnchanged = !!prevEntry && (prevEntry.authoritySpec === spec || JSON.stringify(prevEntry.authoritySpec) === JSON.stringify(spec));
|
|
167
169
|
let entry: AuthorityEntry = {
|
|
@@ -248,7 +250,7 @@ class AuthorityLookup {
|
|
|
248
250
|
return;
|
|
249
251
|
}
|
|
250
252
|
let full = await timeoutToError(
|
|
251
|
-
|
|
253
|
+
this.getCallTimeout(nodeId),
|
|
252
254
|
AuthorityLookupController.nodes[nodeId].getAuthoritySpec(),
|
|
253
255
|
() => new Error(`Timeout calling getAuthoritySpec for ${nodeId}`)
|
|
254
256
|
);
|
|
@@ -256,6 +258,11 @@ class AuthorityLookup {
|
|
|
256
258
|
}
|
|
257
259
|
|
|
258
260
|
private legacySyncNodeIds = new Set<string>();
|
|
261
|
+
private syncedNodeIds = new Set<string>();
|
|
262
|
+
|
|
263
|
+
private getCallTimeout(nodeId: string): number {
|
|
264
|
+
return this.syncedNodeIds.has(nodeId) ? CONNECTED_CALL_TIMEOUT : INITIAL_CALL_TIMEOUT;
|
|
265
|
+
}
|
|
259
266
|
|
|
260
267
|
private connectTo = cache((nodeId: string) => {
|
|
261
268
|
let stopObj = { stop: false };
|
|
@@ -272,6 +279,7 @@ class AuthorityLookup {
|
|
|
272
279
|
this.removeNode(nodeId, "node removed from node discovery");
|
|
273
280
|
stopObj.stop = true;
|
|
274
281
|
this.legacySyncNodeIds.delete(nodeId);
|
|
282
|
+
this.syncedNodeIds.delete(nodeId);
|
|
275
283
|
this.connectTo.clear(nodeId);
|
|
276
284
|
this.disconnectWatch.clear(nodeId);
|
|
277
285
|
};
|
|
@@ -339,7 +347,7 @@ class AuthorityLookup {
|
|
|
339
347
|
let other: AuthorityHashSync;
|
|
340
348
|
try {
|
|
341
349
|
other = await timeoutToError(
|
|
342
|
-
|
|
350
|
+
this.getCallTimeout(nodeId),
|
|
343
351
|
AuthorityLookupController.nodes[nodeId].getSetNetworkSyncHash(this.getOurHashSync()),
|
|
344
352
|
() => new Error(`Timeout calling getSetNetworkSyncHash for ${nodeId}`)
|
|
345
353
|
);
|
|
@@ -356,7 +364,7 @@ class AuthorityLookup {
|
|
|
356
364
|
|
|
357
365
|
private async syncNodeLegacy(nodeId: string) {
|
|
358
366
|
let otherPaths = await timeoutToError(
|
|
359
|
-
|
|
367
|
+
this.getCallTimeout(nodeId),
|
|
360
368
|
AuthorityLookupController.nodes[nodeId].networkSyncPaths({
|
|
361
369
|
spec: this.ourSpec,
|
|
362
370
|
isReady: this.ourIsReady,
|
|
@@ -89,12 +89,15 @@ class DeployConfirmModal extends qreact.Component<{
|
|
|
89
89
|
let warning = groupInfo.deployWarning;
|
|
90
90
|
let count = services.length;
|
|
91
91
|
return <div className={css.vbox(14).maxWidth(620)}>
|
|
92
|
-
{warning && <div className={css.vbox(8).pad2(12).fillWidth.bord2(groupInfo.hue,
|
|
93
|
-
<div className={css.fontSize(18).boldStyle}>⚠️ {groupInfo.emoji} {warning.title}</div>
|
|
92
|
+
{warning && <div className={css.vbox(8).pad2(12).fillWidth.bord2(groupInfo.hue, 70, 50, 2).hsl(groupInfo.hue, 50, 18)}>
|
|
93
|
+
<div className={css.fontSize(18).boldStyle.colorhsl(groupInfo.hue, 90, 75)}>⚠️ {groupInfo.emoji} {warning.title}</div>
|
|
94
94
|
{warning.paragraphs.map(paragraph => <div>{paragraph}</div>)}
|
|
95
95
|
</div>}
|
|
96
96
|
<div className={css.fontSize(18).boldStyle}>Deploy {count} {groupInfo.label} service{count === 1 ? "" : "s"} onto {latestRef.slice(0, 6)}?</div>
|
|
97
|
-
{noOverlap && <div>
|
|
97
|
+
{groupInfo.noOverlap && <div>
|
|
98
|
+
{groupInfo.label} services are always deployed without overlap: the old instances are shut down as soon as the new ones are up, because two copies must never run at once.
|
|
99
|
+
</div>}
|
|
100
|
+
{!groupInfo.noOverlap && noOverlap && <div>
|
|
98
101
|
The old instances are shut down as soon as the new ones are up. This does not give you a clean cut over: the new instances are already up before the old ones are told to stop, and stopping is not instant, so there is still a few seconds of overlap either way. Deploying normally, letting the old instances run for {formatTime(DEFAULT_OVERLAP_TIME)}, is usually the better choice.
|
|
99
102
|
</div>}
|
|
100
103
|
{!noOverlap && <div>
|
|
@@ -103,12 +106,12 @@ class DeployConfirmModal extends qreact.Component<{
|
|
|
103
106
|
<div className={css.vbox(2).fillWidth.maxHeight(CONFIRM_LIST_MAX_HEIGHT).overflowAuto}>
|
|
104
107
|
{services.map(service => <div className={css.hbox(8)}>
|
|
105
108
|
<span className={css.boldStyle}>{service.title}</span>
|
|
106
|
-
<span className={css.colorhsl(0, 0,
|
|
109
|
+
<span className={css.colorhsl(0, 0, 65)}>currently {service.gitRef.slice(0, 6)}</span>
|
|
107
110
|
</div>)}
|
|
108
111
|
</div>
|
|
109
112
|
<div className={css.hbox(10)}>
|
|
110
113
|
<button
|
|
111
|
-
className={buttonStyle.hsl(noOverlap ? 0 : 120, 70, noOverlap ? 70 : 90)}
|
|
114
|
+
className={buttonStyle.hsl(noOverlap && !groupInfo.noOverlap ? 0 : 120, 70, noOverlap && !groupInfo.noOverlap ? 70 : 90)}
|
|
112
115
|
onClick={() => {
|
|
113
116
|
closeAllModals();
|
|
114
117
|
this.props.onConfirm();
|
|
@@ -211,14 +214,14 @@ export class DeployGroupButtons extends qreact.Component<{
|
|
|
211
214
|
return <div className={css.hbox(10).pad2(8).hsl(groupInfo.hue, 40, 95).bord2(groupInfo.hue, 40, 70)}>
|
|
212
215
|
{label}
|
|
213
216
|
<button
|
|
214
|
-
className={buttonStyle.hsl(120, 70, 90) + (isDeploying && css.opacity(0.6))}
|
|
217
|
+
className={buttonStyle.hsl(noOverlapOnly ? 210 : 120, 70, 90) + (isDeploying && css.opacity(0.6))}
|
|
215
218
|
title={noOverlapOnly
|
|
216
219
|
? `Deploys every outdated ${groupInfo.label} service now; the old instances are shut down as soon as the new ones are up, so two copies never run at once`
|
|
217
220
|
: `Asks first, then deploys every outdated ${groupInfo.label} service; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
|
|
218
221
|
onClick={() => {
|
|
219
222
|
if (isDeploying) return;
|
|
220
223
|
confirmDeploy({
|
|
221
|
-
actionLabel: `Update ${groupInfo.label} to New & Restart (${outdatedServices.length})`,
|
|
224
|
+
actionLabel: `Update ${groupInfo.label} to New${noOverlapOnly ? ", No Overlap" : " & Restart"} (${outdatedServices.length})`,
|
|
222
225
|
group,
|
|
223
226
|
noOverlap: noOverlapOnly,
|
|
224
227
|
services: outdatedServices,
|
|
@@ -229,7 +232,7 @@ export class DeployGroupButtons extends qreact.Component<{
|
|
|
229
232
|
}}
|
|
230
233
|
>
|
|
231
234
|
<div>
|
|
232
|
-
{bigEmoji("🎯", -4)} <span>{isDeploying && "⏳ Deploying..." || `Update ${outdatedServices.length} to New${noOverlapOnly ? "" : " & Restart"}`}</span>
|
|
235
|
+
{bigEmoji(noOverlapOnly ? "⚡" : "🎯", -4)} <span>{isDeploying && "⏳ Deploying..." || `Update ${outdatedServices.length} to New${noOverlapOnly ? ", No Overlap" : " & Restart"}`}</span>
|
|
233
236
|
</div>
|
|
234
237
|
<div className={css.hbox(5)}>
|
|
235
238
|
<b>New</b>
|
|
@@ -351,14 +354,14 @@ export class UpdateServiceButtons extends qreact.Component<{
|
|
|
351
354
|
|
|
352
355
|
return <>
|
|
353
356
|
<button
|
|
354
|
-
className={buttonStyle.hsl(120, 70, 90).alignSelf("stretch")}
|
|
357
|
+
className={buttonStyle.hsl(noOverlapOnly ? 210 : 120, 70, 90).alignSelf("stretch")}
|
|
355
358
|
disabled={this.state.isDeploying}
|
|
356
359
|
title={noOverlapOnly
|
|
357
360
|
? "Deploys now; the old instance is shut down as soon as the new one is up, so two copies never run at once"
|
|
358
361
|
: `Deploys now; the old instances keep running for ${formatTime(DEFAULT_OVERLAP_TIME)} before shutting down`}
|
|
359
362
|
onClick={() => {
|
|
360
363
|
confirmDeploy({
|
|
361
|
-
actionLabel: "Update to New & Restart",
|
|
364
|
+
actionLabel: noOverlapOnly ? "Update to New, No Overlap" : "Update to New & Restart",
|
|
362
365
|
group,
|
|
363
366
|
noOverlap: noOverlapOnly,
|
|
364
367
|
services: [service],
|
|
@@ -368,7 +371,7 @@ export class UpdateServiceButtons extends qreact.Component<{
|
|
|
368
371
|
}}
|
|
369
372
|
>
|
|
370
373
|
<div>
|
|
371
|
-
{bigEmoji("🎯", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || `Update to New${noOverlapOnly ? "" : " & Restart"}`}</span>
|
|
374
|
+
{bigEmoji(noOverlapOnly ? "⚡" : "🎯", -4)} <span>{this.state.isDeploying && "⏳ Deploying..." || `Update to New${noOverlapOnly ? ", No Overlap" : " & Restart"}`}</span>
|
|
372
375
|
</div>
|
|
373
376
|
<div className={css.hbox(5)}>
|
|
374
377
|
<b>New</b>
|
|
@@ -13,7 +13,7 @@ import { forceRemoveNode } from "../-f-node-discovery/NodeDiscovery";
|
|
|
13
13
|
import { fsExistsAsync } from "../fs";
|
|
14
14
|
import { PromiseObj } from "../promise";
|
|
15
15
|
import { SERVICE_FOLDER, SERVICE_NODE_FILE_NAME } from "./machineSchema";
|
|
16
|
-
import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord, markProcessDead } from "./processLogs";
|
|
16
|
+
import { ProcessRecord, getLogPipeScript, getLogTailScript, getProcessLogPath, getPipeScriptPath, getTailScriptPath, findProcessRecordByPid, writeProcessRecord, markProcessDead, listProcessRecords } from "./processLogs";
|
|
17
17
|
import { setPreciseTimeout } from "../misc";
|
|
18
18
|
|
|
19
19
|
// Running, inspecting and killing the tmux screens services run in. This layer only knows "here is a configuration, run it" - which version should be running when is the deploy logic's problem.
|
|
@@ -48,7 +48,13 @@ export const retireScreen = cache(async (screenName: string): Promise<void> => {
|
|
|
48
48
|
console.log(green(`${screenName} is retiring, and is due to be killed at ${new Date(dueAt).toISOString()}`));
|
|
49
49
|
await new Promise<void>(resolve => setPreciseTimeout({ time: dueAt, callback: () => resolve() }));
|
|
50
50
|
}
|
|
51
|
+
let records = (await listProcessRecords()).filter(record => record.screenName === screenName && record.deadTime === undefined);
|
|
51
52
|
await killScreen({ screenName });
|
|
53
|
+
for (let record of records) {
|
|
54
|
+
if (!record.nodeId) continue;
|
|
55
|
+
console.log(green(`Removing the node registration ${record.nodeId} of the killed retiring screen ${screenName}`));
|
|
56
|
+
await forceRemoveNode(record.nodeId);
|
|
57
|
+
}
|
|
52
58
|
} finally {
|
|
53
59
|
// Dropped either way: a failure should be retried by the next resync rather than remembered as done
|
|
54
60
|
retireScreen.clear(screenName);
|
|
@@ -340,17 +346,11 @@ export async function retireCurrentScreen(config: {
|
|
|
340
346
|
let outgoingPid = await getScreenPanePid(canonicalScreenName);
|
|
341
347
|
if (!outgoingPid) return;
|
|
342
348
|
let retiringScreenName = getRetiringScreenName(canonicalScreenName, killTime);
|
|
343
|
-
console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}`));
|
|
349
|
+
console.log(green(`Retiring ${canonicalScreenName} as ${retiringScreenName}, it keeps its node registration ${nodeId || "(unknown)"} until it is killed at ${new Date(killTime).toISOString()}`));
|
|
344
350
|
await runPromise(`${getTmuxPrefix()}tmux rename-session -t ${canonicalScreenName} ${retiringScreenName}`);
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
await writeProcessRecord({ ...record, screenName: retiringScreenName });
|
|
349
|
-
}
|
|
350
|
-
})();
|
|
351
|
-
if (nodeId) {
|
|
352
|
-
// The file in the folder belongs to its replacement now, so only the registration goes
|
|
353
|
-
void removeServiceNode({ folder, nodeId, ownsFile: false });
|
|
351
|
+
let record = await findProcessRecordByPid(folder, outgoingPid);
|
|
352
|
+
if (record) {
|
|
353
|
+
await writeProcessRecord({ ...record, screenName: retiringScreenName, nodeId: nodeId || record.nodeId });
|
|
354
354
|
}
|
|
355
355
|
// Started here rather than waiting for a resync to notice the screen: with a killTime already past - a zero overlap - this takes it down right now
|
|
356
356
|
void retireScreen(retiringScreenName);
|
|
@@ -60,7 +60,7 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
|
|
|
60
60
|
label: "Front-end Proxy",
|
|
61
61
|
emoji: "🚪",
|
|
62
62
|
hue: 35,
|
|
63
|
-
description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating.",
|
|
63
|
+
description: "The HTTP bootstrapper. If it goes down users can't load the site, but it does very little, so it rarely needs updating. It binds port 443, so it is always deployed without overlap: an overlapped deploy leaves the new instance unable to bind the port, and the site goes down when the old one exits.",
|
|
64
64
|
deployWarning: {
|
|
65
65
|
title: "You probably don't need to update the front-end proxy",
|
|
66
66
|
paragraphs: [
|
|
@@ -69,6 +69,7 @@ export const DEPLOY_GROUPS: DeployGroupInfo[] = [
|
|
|
69
69
|
"It's usually best not to touch it.",
|
|
70
70
|
],
|
|
71
71
|
},
|
|
72
|
+
noOverlap: true,
|
|
72
73
|
},
|
|
73
74
|
{
|
|
74
75
|
group: "storageCore",
|
|
@@ -1,5 +1,9 @@
|
|
|
1
|
-
import
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import { sort, timeInDay, timeInHour, timeInMinute, timeInSecond, timeoutToError, timeoutToUndefined, timeoutToUndefinedSilent } from "socket-function/src/misc";
|
|
2
4
|
import { lazy } from "socket-function/src/caching";
|
|
5
|
+
import { runInSerial } from "socket-function/src/batching";
|
|
6
|
+
import { getSharedFolderPath } from "../../../fs";
|
|
3
7
|
import { getMachineId } from "sliftutils/misc/https/certs";
|
|
4
8
|
import { getDomain } from "../../../config";
|
|
5
9
|
import { getAllNodeIds, getOwnMachineId, isOwnNodeId } from "../../../-f-node-discovery/NodeDiscovery";
|
|
@@ -22,6 +26,9 @@ const END_TIME_MIN_AGE = timeInMinute;
|
|
|
22
26
|
const MOVE_GRACE = timeInMinute;
|
|
23
27
|
// TTL for the path-cache (the slow TimeFileTree.findAllPaths walk).
|
|
24
28
|
const PATHS_CACHE_TTL = timeInMinute * 15;
|
|
29
|
+
const PATHS_STATE_MAX_AGE = timeInDay * 30;
|
|
30
|
+
export const SEARCH_STATE_FOLDER = "log-search-cache";
|
|
31
|
+
const SEARCH_STATE_FILE = "search-state.json";
|
|
25
32
|
// During the read/scan loops, emit a progress line at most this often.
|
|
26
33
|
const PROGRESS_LOG_INTERVAL = timeInSecond * 5;
|
|
27
34
|
const FLUSH_CALL_TIMEOUT = timeInSecond * 30;
|
|
@@ -101,6 +108,44 @@ export type SearchResult = {
|
|
|
101
108
|
|
|
102
109
|
type Direction = "fromStart" | "fromEnd";
|
|
103
110
|
|
|
111
|
+
type SearchState = {
|
|
112
|
+
movedThrough: { [machineId: string]: { through: number; at: number } };
|
|
113
|
+
paths: { [key: string]: { at: number; paths: TimeFilePath[] } };
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
const getSearchStatePath = lazy(() => getSharedFolderPath(SEARCH_STATE_FOLDER) + SEARCH_STATE_FILE);
|
|
117
|
+
|
|
118
|
+
const loadSearchState = lazy(async (): Promise<SearchState> => {
|
|
119
|
+
let statePath = getSearchStatePath();
|
|
120
|
+
try {
|
|
121
|
+
let state: SearchState = JSON.parse(await fs.promises.readFile(statePath, "utf8"));
|
|
122
|
+
return { movedThrough: state.movedThrough || {}, paths: state.paths || {} };
|
|
123
|
+
} catch (e: any) {
|
|
124
|
+
if (e?.code !== "ENOENT") {
|
|
125
|
+
console.warn(`Ignoring unreadable log search state ${statePath}, starting with an empty one: ${String(e?.stack ?? e)}`);
|
|
126
|
+
}
|
|
127
|
+
return { movedThrough: {}, paths: {} };
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const saveSearchState = runInSerial(async (state: SearchState) => {
|
|
132
|
+
let statePath = getSearchStatePath();
|
|
133
|
+
await fs.promises.mkdir(path.dirname(statePath), { recursive: true });
|
|
134
|
+
let tempPath = statePath + ".tmp";
|
|
135
|
+
await fs.promises.writeFile(tempPath, JSON.stringify(state));
|
|
136
|
+
await fs.promises.rename(tempPath, statePath);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
export function assertSearchTimeRange(startTime: number, endTime: number) {
|
|
140
|
+
let now = Date.now();
|
|
141
|
+
if (endTime > now - END_TIME_MIN_AGE) {
|
|
142
|
+
throw new Error(`endTime must be at least ${formatTime(END_TIME_MIN_AGE)} in the past (got ${formatTime(now - endTime)} ago)`);
|
|
143
|
+
}
|
|
144
|
+
if (startTime >= endTime) {
|
|
145
|
+
throw new Error(`startTime (${formatDateTime(startTime)}) must be < endTime (${formatDateTime(endTime)})`);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
104
149
|
// One log file within a specific logger's archive, as gathered for searching.
|
|
105
150
|
type FileEntry = {
|
|
106
151
|
path: TimeFilePath;
|
|
@@ -144,13 +189,6 @@ function createProgressLogger(): (message: string) => void {
|
|
|
144
189
|
}
|
|
145
190
|
|
|
146
191
|
export class MCPIndexedLogs {
|
|
147
|
-
// machineId -> latest timestamp guaranteed to already be moved-to-public.
|
|
148
|
-
private movedThroughByMachine = new Map<string, number>();
|
|
149
|
-
|
|
150
|
-
// Cache: `${type}|${loggerName}|${startBucket}|${endBucket}` -> { time, paths }.
|
|
151
|
-
// Buckets are hour-aligned start/end so adjacent searches reuse work.
|
|
152
|
-
private pathsCache = new Map<string, { time: number; paths: TimeFilePath[] }>();
|
|
153
|
-
|
|
154
192
|
public async search(config: {
|
|
155
193
|
query: string;
|
|
156
194
|
machine: string | "local";
|
|
@@ -168,19 +206,16 @@ export class MCPIndexedLogs {
|
|
|
168
206
|
// meaningfully read a log row without knowing when it happened.
|
|
169
207
|
let columns = config.columns.includes("time") ? config.columns : ["time", ...config.columns];
|
|
170
208
|
console.log(`[search] query=${JSON.stringify(config.query)} | machine=${config.machine} | startTime=${formatDateTime(startTime)} | endTime=${formatDateTime(endTime)} | direction=${config.direction} | columns=[${config.columns.join(",")}] | limit=${config.limit ?? "(default)"}`);
|
|
171
|
-
|
|
172
|
-
if (endTime > now - END_TIME_MIN_AGE) {
|
|
173
|
-
throw new Error(`endTime must be at least ${formatTime(END_TIME_MIN_AGE)} in the past (got ${formatTime(now - endTime)} ago)`);
|
|
174
|
-
}
|
|
175
|
-
if (startTime >= endTime) {
|
|
176
|
-
throw new Error(`startTime (${formatDateTime(startTime)}) must be < endTime (${formatDateTime(endTime)})`);
|
|
177
|
-
}
|
|
209
|
+
assertSearchTimeRange(startTime, endTime);
|
|
178
210
|
|
|
179
211
|
let machineId = config.machine === "local" ? getOwnMachineId() : config.machine;
|
|
180
212
|
|
|
181
213
|
let moveStart = Date.now();
|
|
182
214
|
let moveOutcome = await this.ensureMovedThrough(machineId, endTime, config.forceRefresh);
|
|
183
215
|
console.log(`[search] ensureMovedThrough ${moveOutcome} in ${formatTime(Date.now() - moveStart)}`);
|
|
216
|
+
let state = await loadSearchState();
|
|
217
|
+
let flushedAt = state.movedThrough[machineId]?.at;
|
|
218
|
+
let minListingTime = flushedAt ?? Date.now() - PATHS_CACHE_TTL;
|
|
184
219
|
|
|
185
220
|
let loggers = await getLoggers2Async();
|
|
186
221
|
let useType: "local" | "public" = config.machine === "local" ? "local" : "public";
|
|
@@ -207,6 +242,7 @@ export class MCPIndexedLogs {
|
|
|
207
242
|
loggerName,
|
|
208
243
|
startTime,
|
|
209
244
|
endTime,
|
|
245
|
+
minListingTime,
|
|
210
246
|
forceRefresh: config.forceRefresh,
|
|
211
247
|
});
|
|
212
248
|
totalPathsSeen += paths.length;
|
|
@@ -485,8 +521,9 @@ export class MCPIndexedLogs {
|
|
|
485
521
|
// now - MOVE_GRACE so we skip this on subsequent calls covering the same
|
|
486
522
|
// window.
|
|
487
523
|
private async ensureMovedThrough(machineId: string, endTime: number, forceRefresh?: boolean): Promise<"cached" | "no-node" | "unreachable" | "moved"> {
|
|
488
|
-
let
|
|
489
|
-
|
|
524
|
+
let state = await loadSearchState();
|
|
525
|
+
let moved = state.movedThrough[machineId];
|
|
526
|
+
if (!forceRefresh && moved && moved.through >= endTime) return "cached";
|
|
490
527
|
|
|
491
528
|
let nodeIds = await this.findRemoteNodesOnMachine(machineId);
|
|
492
529
|
if (nodeIds.length === 0) {
|
|
@@ -537,7 +574,8 @@ export class MCPIndexedLogs {
|
|
|
537
574
|
console.warn(`MCPIndexedLogs: machine ${machineId} could not be flushed for ${unflushed.join(", ")}. Results only include what it had already moved to public storage on its own.`);
|
|
538
575
|
return "unreachable";
|
|
539
576
|
}
|
|
540
|
-
|
|
577
|
+
state.movedThrough[machineId] = { through: Date.now() - MOVE_GRACE, at: Date.now() };
|
|
578
|
+
await saveSearchState(state);
|
|
541
579
|
return "moved";
|
|
542
580
|
}
|
|
543
581
|
|
|
@@ -576,32 +614,27 @@ export class MCPIndexedLogs {
|
|
|
576
614
|
throw new Error(`Unknown logger ${name}`);
|
|
577
615
|
}
|
|
578
616
|
|
|
579
|
-
// Caches TimeFileTree.findAllPaths by hour-aligned bucket so repeated
|
|
580
|
-
// searches over similar windows reuse the slow folder walk.
|
|
581
617
|
private async getCachedPaths(config: {
|
|
582
618
|
archives: LogArchives;
|
|
583
619
|
type: "local" | "public";
|
|
584
620
|
loggerName: LoggerName;
|
|
585
621
|
startTime: number;
|
|
586
622
|
endTime: number;
|
|
623
|
+
minListingTime: number;
|
|
587
624
|
forceRefresh?: boolean;
|
|
588
625
|
}): Promise<TimeFilePath[]> {
|
|
589
626
|
let bucketStart = Math.floor(config.startTime / timeInHour) * timeInHour;
|
|
590
627
|
let bucketEnd = Math.ceil(config.endTime / timeInHour) * timeInHour;
|
|
591
628
|
let key = `${config.type}|${config.loggerName}|${bucketStart}|${bucketEnd}`;
|
|
592
629
|
let now = Date.now();
|
|
630
|
+
let state = await loadSearchState();
|
|
593
631
|
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
if (now - v.time > PATHS_CACHE_TTL) this.pathsCache.delete(k);
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
if (config.forceRefresh) {
|
|
600
|
-
this.pathsCache.delete(key);
|
|
632
|
+
for (let [k, v] of Object.entries(state.paths)) {
|
|
633
|
+
if (now - v.at > PATHS_STATE_MAX_AGE) delete state.paths[k];
|
|
601
634
|
}
|
|
602
635
|
|
|
603
|
-
let cached =
|
|
604
|
-
if (
|
|
636
|
+
let cached = state.paths[key];
|
|
637
|
+
if (!config.forceRefresh && cached && cached.at >= config.minListingTime) {
|
|
605
638
|
return cached.paths;
|
|
606
639
|
}
|
|
607
640
|
|
|
@@ -609,7 +642,8 @@ export class MCPIndexedLogs {
|
|
|
609
642
|
startTime: bucketStart,
|
|
610
643
|
endTime: bucketEnd,
|
|
611
644
|
});
|
|
612
|
-
|
|
645
|
+
state.paths[key] = { at: now, paths };
|
|
646
|
+
await saveSearchState(state);
|
|
613
647
|
return paths;
|
|
614
648
|
}
|
|
615
649
|
|
|
@@ -11,17 +11,22 @@ module.hotreload = false;
|
|
|
11
11
|
|
|
12
12
|
import "../../inject";
|
|
13
13
|
|
|
14
|
+
import fs from "fs";
|
|
15
|
+
import path from "path";
|
|
14
16
|
import yargs from "yargs";
|
|
15
17
|
import { formatDateTime } from "socket-function/src/formatting/format";
|
|
16
18
|
import { blue, magenta, yellow } from "socket-function/src/formatting/logColors";
|
|
17
|
-
import { timeInMinute } from "socket-function/src/misc";
|
|
18
|
-
import { MCPIndexedLogs, normalizeTime } from "../logs/IndexedLogs/MCPIndexedLogs";
|
|
19
|
-
import { machineNames } from "../../deployManager/machineSchema";
|
|
19
|
+
import { timeInDay, timeInMinute } from "socket-function/src/misc";
|
|
20
|
+
import { MCPIndexedLogs, SEARCH_STATE_FOLDER, assertSearchTimeRange, normalizeTime } from "../logs/IndexedLogs/MCPIndexedLogs";
|
|
21
|
+
import { MachineName, machineNames } from "../../deployManager/machineSchema";
|
|
22
|
+
import { getSharedFolderPath } from "../../fs";
|
|
20
23
|
import { disablePathAuditer } from "../pathAuditerCallback";
|
|
21
24
|
|
|
22
25
|
const DEFAULT_LIMIT = 200;
|
|
23
26
|
const DEFAULT_COLUMNS = ["__entry", "__threadId", "param0", "param1", "param2", "param3"];
|
|
24
27
|
const DEFAULT_END_AGE = timeInMinute * 2;
|
|
28
|
+
const MACHINE_NAMES_CACHE_TTL = timeInDay;
|
|
29
|
+
const MACHINE_NAMES_CACHE_FILE = "machine-names.json";
|
|
25
30
|
|
|
26
31
|
let args = yargs(process.argv.slice(2))
|
|
27
32
|
.option("machine", { type: "string", demandOption: true, describe: "machine id, a substring of a machine name (ex \"5600x\"), or \"local\"" })
|
|
@@ -31,14 +36,34 @@ let args = yargs(process.argv.slice(2))
|
|
|
31
36
|
.option("direction", { choices: ["fromStart", "fromEnd"] as const, default: "fromEnd" as const })
|
|
32
37
|
.option("columns", { type: "array", string: true, describe: `fields to print (default ${DEFAULT_COLUMNS.join(", ")})` })
|
|
33
38
|
.option("limit", { type: "number", default: DEFAULT_LIMIT })
|
|
34
|
-
.option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders instead of using the cached
|
|
39
|
+
.option("refresh", { type: "boolean", default: false, describe: "re-walk the log folders and re-flush the machine instead of using the cached state" })
|
|
35
40
|
.option("every", { type: "number", default: 1, describe: "print only every Nth row (oldest first), to see a trend over many rows" })
|
|
36
41
|
.strict()
|
|
37
42
|
.argv;
|
|
38
43
|
|
|
39
|
-
|
|
40
|
-
|
|
44
|
+
type MachineNamesCache = { at: number; names: MachineName[] };
|
|
45
|
+
|
|
46
|
+
async function getMachineNames(): Promise<MachineName[]> {
|
|
47
|
+
let cachePath = getSharedFolderPath(SEARCH_STATE_FOLDER) + MACHINE_NAMES_CACHE_FILE;
|
|
48
|
+
let cached: MachineNamesCache | undefined;
|
|
49
|
+
try {
|
|
50
|
+
cached = JSON.parse(await fs.promises.readFile(cachePath, "utf8"));
|
|
51
|
+
} catch (e: any) {
|
|
52
|
+
if (e?.code !== "ENOENT") {
|
|
53
|
+
console.warn(yellow(`Ignoring unreadable machine name cache ${cachePath}: ${String(e?.stack ?? e)}`));
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
if (cached && !args.refresh && Date.now() - cached.at < MACHINE_NAMES_CACHE_TTL) {
|
|
57
|
+
return cached.names;
|
|
58
|
+
}
|
|
41
59
|
let names = await machineNames.values();
|
|
60
|
+
let fresh: MachineNamesCache = { at: Date.now(), names };
|
|
61
|
+
await fs.promises.mkdir(path.dirname(cachePath), { recursive: true });
|
|
62
|
+
await fs.promises.writeFile(cachePath, JSON.stringify(fresh));
|
|
63
|
+
return names;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function findMachine(names: MachineName[], machine: string): string | undefined {
|
|
42
67
|
if (names.some(x => x.machineId === machine)) return machine;
|
|
43
68
|
let matches = names.filter(x => x.name.toLowerCase().includes(machine.toLowerCase()));
|
|
44
69
|
if (matches.length === 1) {
|
|
@@ -48,19 +73,29 @@ async function resolveMachineId(machine: string): Promise<string> {
|
|
|
48
73
|
if (matches.length > 1) {
|
|
49
74
|
throw new Error(`Machine "${machine}" matches several machines: ${matches.map(x => `${x.name} (${x.machineId})`).join(", ")}`);
|
|
50
75
|
}
|
|
76
|
+
return undefined;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async function resolveMachineId(machine: string): Promise<string> {
|
|
80
|
+
if (machine === "local") return machine;
|
|
81
|
+
let names = await getMachineNames();
|
|
82
|
+
let found = findMachine(names, machine);
|
|
83
|
+
if (found) return found;
|
|
51
84
|
console.warn(yellow(`No machine is named like "${machine}", treating it as a machine id`));
|
|
52
85
|
return machine;
|
|
53
86
|
}
|
|
54
87
|
|
|
55
88
|
async function main() {
|
|
56
89
|
disablePathAuditer();
|
|
90
|
+
let startTime = normalizeTime(args.start, "start");
|
|
91
|
+
let endTime = args.end && normalizeTime(args.end, "end") || Date.now() - DEFAULT_END_AGE;
|
|
92
|
+
assertSearchTimeRange(startTime, endTime);
|
|
57
93
|
let machine = await resolveMachineId(args.machine);
|
|
58
94
|
let columns = (args.columns || DEFAULT_COLUMNS).map(String);
|
|
59
|
-
let endTime = args.end && normalizeTime(args.end, "end") || Date.now() - DEFAULT_END_AGE;
|
|
60
95
|
let result = await new MCPIndexedLogs().search({
|
|
61
96
|
query: args.query,
|
|
62
97
|
machine,
|
|
63
|
-
startTime
|
|
98
|
+
startTime,
|
|
64
99
|
endTime,
|
|
65
100
|
direction: args.direction,
|
|
66
101
|
columns,
|