querysub 0.451.0 → 0.452.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +6 -1
- package/bin/join-public.js +1 -0
- package/package.json +1 -1
- package/src/-a-archives/archiveCache.ts +53 -597
- package/src/-g-core-values/NodeCapabilities.ts +29 -28
- package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts +24 -0
- package/src/0-path-value-core/pathValueCore.ts +1 -1
- package/src/2-proxy/PathValueProxyWatcher.ts +6 -6
- package/src/archiveapps/archiveGCEntry.tsx +1 -0
- package/src/archiveapps/archiveJoinEntry.ts +8 -2
- package/src/deployManager/LaunchTrackingHeader.tsx +69 -0
- package/src/deployManager/machineApplyMainCode.ts +46 -15
- package/src/deployManager/machineSchema.ts +82 -1
- package/src/diagnostics/NodeConnectionsPage.tsx +1 -1
- package/src/diagnostics/NodeViewer.tsx +15 -25
- package/src/diagnostics/debugger/mcp-server.ts +3 -3
- package/src/diagnostics/logs/IndexedLogs/IndexedLogs.ts +2 -2
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogs.ts +64 -22
- package/src/diagnostics/logs/IndexedLogs/MCPIndexedLogsEntry.ts +32 -1
- package/src/diagnostics/managementPages.tsx +8 -0
- package/src/diagnostics/misc-pages/AuthoritySpecPage.tsx +112 -0
- package/src/diagnostics/pathAuditer.ts +0 -6
- package/test.ts +2 -1
- package/src/misc/getParentProcessId.cs +0 -53
- package/src/misc/getParentProcessId.ts +0 -53
|
@@ -12,6 +12,7 @@ import { delay } from "socket-function/src/batching";
|
|
|
12
12
|
import debugbreak from "debugbreak";
|
|
13
13
|
import { formatTime } from "socket-function/src/formatting/format";
|
|
14
14
|
import type { DebugFunctionShardInfo } from "../3-path-functions/PathFunctionRunner";
|
|
15
|
+
import type { AuthoritySpec } from "../0-path-value-core/PathRouter";
|
|
15
16
|
import { requiresNetworkTrustHook } from "../-d-trust/NetworkTrust2";
|
|
16
17
|
import { isNoNetwork } from "../config";
|
|
17
18
|
import { getDebuggerUrl } from "../diagnostics/listenOnDebugger";
|
|
@@ -94,10 +95,10 @@ export async function getControllerNodeIdList(
|
|
|
94
95
|
await Promise.all(nodeIdsToTest.map(async nodeId => {
|
|
95
96
|
let result = await doesNodeExposeController(nodeId, controller);
|
|
96
97
|
if (result) {
|
|
97
|
-
let
|
|
98
|
+
let metadata = await NodeCapabilitiesController.nodes[nodeId].getMetadata();
|
|
98
99
|
passedNodeIds.set(nodeId, {
|
|
99
100
|
machineId: getMachineId(nodeId),
|
|
100
|
-
entryPoint,
|
|
101
|
+
entryPoint: metadata.entryPoint,
|
|
101
102
|
});
|
|
102
103
|
}
|
|
103
104
|
}));
|
|
@@ -117,38 +118,43 @@ export async function getControllerNodeIdList(
|
|
|
117
118
|
|
|
118
119
|
|
|
119
120
|
export async function doesNodeExposeController(reconnectNodeId: string, controller: SocketRegistered<{}>): Promise<boolean> {
|
|
120
|
-
let
|
|
121
|
+
let metadata = await timeoutToUndefinedSilent(10_000, NodeCapabilitiesController.nodes[reconnectNodeId].getMetadata());
|
|
121
122
|
|
|
122
|
-
if (exposedControllers
|
|
123
|
+
if (metadata?.exposedControllers.includes(controller._classGuid)) {
|
|
123
124
|
return true;
|
|
124
125
|
}
|
|
125
126
|
return false;
|
|
126
127
|
}
|
|
127
128
|
|
|
128
129
|
const startupTime = Date.now();
|
|
130
|
+
|
|
131
|
+
export type NodeMetadata = {
|
|
132
|
+
entryPoint: string;
|
|
133
|
+
startupTime: number;
|
|
134
|
+
processId: number;
|
|
135
|
+
functionRunnerShards: DebugFunctionShardInfo[];
|
|
136
|
+
authoritySpec: AuthoritySpec;
|
|
137
|
+
exposedControllers: string[];
|
|
138
|
+
trueTimeOffset: number;
|
|
139
|
+
};
|
|
140
|
+
|
|
129
141
|
class NodeCapabilitiesControllerBase {
|
|
130
|
-
public async
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
142
|
+
public async getMetadata(): Promise<NodeMetadata> {
|
|
143
|
+
let { authorityLookup } = await import("../0-path-value-core/AuthorityLookup");
|
|
144
|
+
return {
|
|
145
|
+
entryPoint: process.argv[1],
|
|
146
|
+
startupTime,
|
|
147
|
+
processId: process.pid,
|
|
148
|
+
functionRunnerShards: getFunctionRunnerShards(),
|
|
149
|
+
authoritySpec: authorityLookup.getOurSpec(),
|
|
150
|
+
exposedControllers: Array.from(SocketFunction.exposedClasses),
|
|
151
|
+
trueTimeOffset: getTrueTimeOffset(),
|
|
152
|
+
};
|
|
138
153
|
}
|
|
154
|
+
|
|
139
155
|
public async getMemoryUsage() {
|
|
140
156
|
return process.memoryUsage();
|
|
141
157
|
}
|
|
142
|
-
public async getProcessId() {
|
|
143
|
-
return process.pid;
|
|
144
|
-
}
|
|
145
|
-
public async getFunctionRunnerShards() {
|
|
146
|
-
return getFunctionRunnerShards();
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
public async getTrueTimeOffset() {
|
|
150
|
-
return getTrueTimeOffset();
|
|
151
|
-
}
|
|
152
158
|
|
|
153
159
|
public async getInspectURL() {
|
|
154
160
|
return await getDebuggerUrl();
|
|
@@ -188,13 +194,8 @@ export const NodeCapabilitiesController = SocketFunction.register(
|
|
|
188
194
|
"NodeCapabilitiesController-399b7067-75c3-4d92-8be3-8470bde32d3c",
|
|
189
195
|
new NodeCapabilitiesControllerBase(),
|
|
190
196
|
() => ({
|
|
191
|
-
|
|
192
|
-
getEntryPoint: {},
|
|
193
|
-
getStartupTime: {},
|
|
197
|
+
getMetadata: {},
|
|
194
198
|
getMemoryUsage: {},
|
|
195
|
-
getProcessId: {},
|
|
196
|
-
getFunctionRunnerShards: {},
|
|
197
|
-
getTrueTimeOffset: {},
|
|
198
199
|
getInspectURL: { hooks: [requiresNetworkTrustHook] },
|
|
199
200
|
exposeExternalDebugPortOnce: { hooks: [requiresNetworkTrustHook] },
|
|
200
201
|
}),
|
|
@@ -245,7 +245,31 @@ export function createArchiveLocker2(config: {
|
|
|
245
245
|
newTransaction.ops.push({ type: "delete", key: obj.file });
|
|
246
246
|
}
|
|
247
247
|
}
|
|
248
|
+
// Per-op trace so a "written too slowly" crash on a specific file can be
|
|
249
|
+
// correlated against the exact transaction that touched it. The aggregate
|
|
250
|
+
// "Joining N => M" log doesn't name the files. We log before the attempt and
|
|
251
|
+
// again after with the resulting status (including "rejected").
|
|
252
|
+
const describeOp = (op: typeof newTransaction.ops[number]): string => {
|
|
253
|
+
let detail = "";
|
|
254
|
+
if (op.key.endsWith(".data") || op.key.endsWith(".data.locked")) {
|
|
255
|
+
try {
|
|
256
|
+
detail = ` source=${pathValueArchives.decodeDataPath(op.key).sourceType}`;
|
|
257
|
+
} catch (e) {
|
|
258
|
+
detail = ` decodeError=${(e as Error).message ?? e}`;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return `${op.type} ${op.key}${detail}`;
|
|
262
|
+
};
|
|
263
|
+
for (let op of newTransaction.ops) {
|
|
264
|
+
console.info(`Transaction attempt: ${describeOp(op)}`);
|
|
265
|
+
}
|
|
266
|
+
|
|
248
267
|
let status = await locker.addTransaction(newTransaction);
|
|
268
|
+
|
|
269
|
+
for (let op of newTransaction.ops) {
|
|
270
|
+
console.info(`Transaction ${status}: ${describeOp(op)}`);
|
|
271
|
+
}
|
|
272
|
+
|
|
249
273
|
if (status === "accepted") {
|
|
250
274
|
let newFiles = new Set<string>();
|
|
251
275
|
for (let file of files) {
|
|
@@ -68,7 +68,7 @@ export const MAX_CHANGE_AGE = MAX_ACCEPTED_CHANGE_AGE * 2;
|
|
|
68
68
|
/** Extra time we keep clientside prediction rejections for, to give us time to receive the actual values. */
|
|
69
69
|
export const CLIENTSIDE_PREDICT_LEEWAY = 500;
|
|
70
70
|
|
|
71
|
-
/** Any PathValues which take longer than this to
|
|
71
|
+
/** Any PathValues which take longer than this to write should be rejected, so... we have
|
|
72
72
|
* to write well before this time.
|
|
73
73
|
* - This has to be at least MAX_CHANGE_AGE * 4.5 + the time to serialize and
|
|
74
74
|
* send our data to remote storage.
|
|
@@ -1472,8 +1472,8 @@ export class PathValueProxyWatcher {
|
|
|
1472
1472
|
let notWatchingUnsyncedParent = reallyUnsyncedParentAccesses.filter(x => !remoteWatcher.debugIsWatchingPath(x));
|
|
1473
1473
|
if (notWatchingUnsynced.length !== 0 || notWatchingUnsyncedParent.length !== 0) {
|
|
1474
1474
|
console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("NOT REMOTE WATCHING REQUIRED PATHS")}. This means our sync or unsync (likely unsync) logic is broken, in remoteWatcher/clientWatcher. OR, there were no read nodes when we tried to sync (we don't handle missing read nodes correctly at the moment)`), { notWatchingUnsynced, notWatchingUnsyncedParent }, watcher.options.watchFunction);
|
|
1475
|
-
debugbreak(2);
|
|
1476
|
-
debugger;
|
|
1475
|
+
// debugbreak(2);
|
|
1476
|
+
// debugger;
|
|
1477
1477
|
} else {
|
|
1478
1478
|
console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT RECEIVE PATH VALUES")}. This means PathValueServer is not responding to watches, either to specific paths, or for all paths`), { reallyUnsyncedAccesses, reallyUnsyncedParentAccesses }, watcher.options.watchFunction);
|
|
1479
1479
|
// debugbreak(2);
|
|
@@ -1481,12 +1481,12 @@ export class PathValueProxyWatcher {
|
|
|
1481
1481
|
}
|
|
1482
1482
|
} else if (watcher.lastSpecialPromiseUnsynced) {
|
|
1483
1483
|
console.warn((`${yellow("WATCHER SLOW TO SYNC")} ${watcher.debugName} ${magenta("DEPENDENT PROMISE NEVER RESOLVED")}. This promise might resolve, but it probably won't. Slow promises should be detached from the watcher system and use multiple watchers/writes, instead of blocking on a promise.`), watcher.lastSpecialPromiseUnsyncedReason, watcher.options.watchFunction);
|
|
1484
|
-
debugbreak(2);
|
|
1485
|
-
debugger;
|
|
1484
|
+
// debugbreak(2);
|
|
1485
|
+
// debugger;
|
|
1486
1486
|
} else {
|
|
1487
1487
|
console.error((`${red("WATCHER FAILED TO SYNC")} ${watcher.debugName} ${magenta("DID NOT TRIGGER WATCHER")}. This means either ProxyWatcher is broken (and isn't triggering when it should, or isn't watching when it should), or ClientWatcher/PathWatcher are broken and are not properly informing callers of watchers.`), { lastUnsyncedAccesses: watcher.lastUnsyncedAccesses, lastUnsyncedParentAccesses: watcher.lastUnsyncedParentAccesses }, watcher.options.watchFunction);
|
|
1488
|
-
debugbreak(2);
|
|
1489
|
-
debugger;
|
|
1488
|
+
// debugbreak(2);
|
|
1489
|
+
// debugger;
|
|
1490
1490
|
}
|
|
1491
1491
|
}, 60000);
|
|
1492
1492
|
}
|
|
@@ -2,7 +2,7 @@ import "../inject";
|
|
|
2
2
|
|
|
3
3
|
import { logErrors } from "../errors";
|
|
4
4
|
import { PathValueArchives, pathValueArchives } from "../0-path-value-core/pathValueArchives";
|
|
5
|
-
import { PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
|
|
5
|
+
import { ARCHIVE_FLUSH_LIMIT, PathValue, VALUE_GC_THRESHOLD } from "../0-path-value-core/pathValueCore";
|
|
6
6
|
import { runInfinitePollCallAtStart } from "socket-function/src/batching";
|
|
7
7
|
import { measureBlock } from "socket-function/src/profiling/measure";
|
|
8
8
|
import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
|
|
@@ -41,7 +41,12 @@ async function runGenesisJoinIteration(config?: { force?: boolean }) {
|
|
|
41
41
|
valueFiles = valueFiles.filter(x => {
|
|
42
42
|
let obj = pathValueArchives.decodeDataPath(x.file);
|
|
43
43
|
if (!obj.minTime) return false;
|
|
44
|
-
|
|
44
|
+
if (obj.sourceType === "genesis") {
|
|
45
|
+
// Has to be old enough
|
|
46
|
+
return Date.now() - obj.time < ARCHIVE_FLUSH_LIMIT;
|
|
47
|
+
}
|
|
48
|
+
// Anything else can be merged immediately.
|
|
49
|
+
return true;
|
|
45
50
|
});
|
|
46
51
|
}
|
|
47
52
|
let withinTimeRangeCount = valueFiles.length;
|
|
@@ -123,6 +128,7 @@ async function main() {
|
|
|
123
128
|
await Querysub.hostService("join");
|
|
124
129
|
|
|
125
130
|
if (yargObj.watch) {
|
|
131
|
+
console.log("Running in watch mode.");
|
|
126
132
|
await runInfinitePollCallAtStart(VALUE_GC_THRESHOLD * 0.8, runGenesisJoinIteration);
|
|
127
133
|
} else {
|
|
128
134
|
try {
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
module.allowclient = true;
|
|
2
|
+
|
|
3
|
+
import { qreact } from "../4-dom/qreact";
|
|
4
|
+
import { css } from "typesafecss";
|
|
5
|
+
import { getBrowserUrlNode } from "../-f-node-discovery/NodeDiscovery";
|
|
6
|
+
import { MachineServiceController } from "./machineSchema";
|
|
7
|
+
|
|
8
|
+
const SINCE_DAYS = 7;
|
|
9
|
+
const CRASH_HUE = 0;
|
|
10
|
+
const OTHER_HUE = 210;
|
|
11
|
+
|
|
12
|
+
export class LaunchTrackingHeader extends qreact.Component {
|
|
13
|
+
render() {
|
|
14
|
+
let summaries = MachineServiceController(getBrowserUrlNode()).getRecentLaunches(SINCE_DAYS);
|
|
15
|
+
if (!summaries) return undefined;
|
|
16
|
+
if (summaries.length === 0) return undefined;
|
|
17
|
+
|
|
18
|
+
let totalCrashes = 0;
|
|
19
|
+
let totalOther = 0;
|
|
20
|
+
let perKey = new Map<string, { crashes: number; other: number }>();
|
|
21
|
+
for (let s of summaries) {
|
|
22
|
+
let isCrash = s.reason === "crashed";
|
|
23
|
+
if (isCrash) totalCrashes++;
|
|
24
|
+
else totalOther++;
|
|
25
|
+
let entry = perKey.get(s.serviceKey);
|
|
26
|
+
if (!entry) {
|
|
27
|
+
entry = { crashes: 0, other: 0 };
|
|
28
|
+
perKey.set(s.serviceKey, entry);
|
|
29
|
+
}
|
|
30
|
+
if (isCrash) entry.crashes++;
|
|
31
|
+
else entry.other++;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
let top: { serviceKey: string; crashes: number; other: number } | undefined;
|
|
35
|
+
for (let [serviceKey, c] of perKey) {
|
|
36
|
+
let candidate = { serviceKey, crashes: c.crashes, other: c.other };
|
|
37
|
+
if (!top) {
|
|
38
|
+
top = candidate;
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
if (candidate.crashes > top.crashes) {
|
|
42
|
+
top = candidate;
|
|
43
|
+
} else if (candidate.crashes === top.crashes && (candidate.crashes + candidate.other) > (top.crashes + top.other)) {
|
|
44
|
+
top = candidate;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
let title = `Launches in last ${SINCE_DAYS} days: ${totalCrashes} crashed, ${totalOther} other`;
|
|
49
|
+
if (top && top.crashes > 0) {
|
|
50
|
+
title += ` -- top by crashes: ${top.serviceKey} (${top.crashes} crashed, ${top.other} other)`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return <div title={title} className={css.hbox(6).colorhsl(0, 0, 20)}>
|
|
54
|
+
<span>🚀</span>
|
|
55
|
+
<span>
|
|
56
|
+
<span className={css.colorhsl(CRASH_HUE, 70, 35).boldStyle}>{totalCrashes}</span>
|
|
57
|
+
<span className={css.colorhsl(0, 0, 55)}>|</span>
|
|
58
|
+
<span className={css.colorhsl(OTHER_HUE, 65, 35).boldStyle}>{totalOther}</span>
|
|
59
|
+
</span>
|
|
60
|
+
{top && top.crashes > 0 &&
|
|
61
|
+
<span className={css.colorhsl(0, 0, 40)}>
|
|
62
|
+
(<span className={css.colorhsl(CRASH_HUE, 70, 35)}>{top.crashes}</span>
|
|
63
|
+
<span className={css.colorhsl(0, 0, 55)}>|</span>
|
|
64
|
+
<span className={css.colorhsl(OTHER_HUE, 65, 35)}>{top.other}</span>)
|
|
65
|
+
</span>
|
|
66
|
+
}
|
|
67
|
+
</div>;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
@@ -4,7 +4,7 @@ import { measureWrap } from "socket-function/src/profiling/measure";
|
|
|
4
4
|
import { getOwnMachineId } from "../-a-auth/certs";
|
|
5
5
|
import { forceRemoveNode, getOurNodeId, getOurNodeIdAssert } from "../-f-node-discovery/NodeDiscovery";
|
|
6
6
|
import { Querysub } from "../4-querysub/QuerysubController";
|
|
7
|
-
import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs } from "./machineSchema";
|
|
7
|
+
import { MACHINE_RESYNC_INTERVAL, MachineServiceControllerBase, MachineInfo, ServiceConfig, serviceConfigs, SERVICE_FOLDER, machineInfos, SERVICE_NODE_FILE_NAME, getEffectiveServiceConfigs, recordLaunch } from "./machineSchema";
|
|
8
8
|
import { runPromise } from "../functional/runCommand";
|
|
9
9
|
import { getExternalIP } from "socket-function/src/networking";
|
|
10
10
|
import { errorToUndefined, errorToUndefinedSilent } from "../errors";
|
|
@@ -183,25 +183,21 @@ export async function streamScreenOutput(config: {
|
|
|
183
183
|
const pipeFile = `${root}${screenName}/pipe.txt`;
|
|
184
184
|
const tailScript = `${root}${screenName}/smart_tail.sh`;
|
|
185
185
|
|
|
186
|
-
// Create a smart tail script that handles file truncation
|
|
186
|
+
// Create a smart tail script that handles file truncation. It does
|
|
187
|
+
// NOT emit the initial backlog - that is read and sent directly by
|
|
188
|
+
// this process (as a single message) below. The script is given a
|
|
189
|
+
// start byte offset and only emits content appended after it.
|
|
187
190
|
await fs.promises.writeFile(tailScript, `#!/bin/bash
|
|
188
191
|
PIPE_FILE="$1"
|
|
192
|
+
START_POS="$2"
|
|
189
193
|
|
|
190
|
-
#
|
|
191
|
-
CURRENT_POS
|
|
194
|
+
# Position tracking starts at the offset we already read directly.
|
|
195
|
+
CURRENT_POS=$START_POS
|
|
196
|
+
CURRENT_SIZE=$START_POS
|
|
192
197
|
LAST_MTIME=""
|
|
193
198
|
|
|
194
|
-
# Read initial content and get file size
|
|
195
199
|
if [ -f "$PIPE_FILE" ]; then
|
|
196
|
-
CURRENT_SIZE=$(stat -c%s "$PIPE_FILE" 2>/dev/null || wc -c < "$PIPE_FILE")
|
|
197
200
|
LAST_MTIME=$(stat -c%Y "$PIPE_FILE" 2>/dev/null || stat -f%m "$PIPE_FILE" 2>/dev/null || echo "0")
|
|
198
|
-
# Output initial content like tail would
|
|
199
|
-
if [ $CURRENT_SIZE -gt 0 ]; then
|
|
200
|
-
cat "$PIPE_FILE"
|
|
201
|
-
CURRENT_POS=$CURRENT_SIZE
|
|
202
|
-
fi
|
|
203
|
-
else
|
|
204
|
-
CURRENT_SIZE=0
|
|
205
201
|
fi
|
|
206
202
|
|
|
207
203
|
# Poll for file changes every 250ms
|
|
@@ -237,8 +233,21 @@ done`);
|
|
|
237
233
|
|
|
238
234
|
await runPromise(`chmod +x ${tailScript}`);
|
|
239
235
|
|
|
240
|
-
//
|
|
241
|
-
|
|
236
|
+
// Read the existing backlog ourselves and deliver it as a single
|
|
237
|
+
// onData call. If we let the tail script cat it, the stream chunks
|
|
238
|
+
// it and runInSerial dribbles it out one round-trip at a time.
|
|
239
|
+
let initialContent = "";
|
|
240
|
+
let initialByteSize = 0;
|
|
241
|
+
try {
|
|
242
|
+
initialContent = await fs.promises.readFile(pipeFile, "utf8");
|
|
243
|
+
initialByteSize = Buffer.byteLength(initialContent, "utf8");
|
|
244
|
+
} catch {
|
|
245
|
+
// pipe.txt may not exist yet; the tail script will pick it up.
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// The tail script emits only content appended after initialByteSize,
|
|
249
|
+
// so the backlog we just read is never sent twice.
|
|
250
|
+
childProcess = spawn("bash", [tailScript, pipeFile, String(initialByteSize)], {
|
|
242
251
|
stdio: "pipe",
|
|
243
252
|
});
|
|
244
253
|
|
|
@@ -262,6 +271,13 @@ done`);
|
|
|
262
271
|
started.reject(err);
|
|
263
272
|
});
|
|
264
273
|
|
|
274
|
+
if (initialContent) {
|
|
275
|
+
// Queued synchronously here, before any stdout 'data' event can
|
|
276
|
+
// fire, so the backlog is always delivered ahead of new output.
|
|
277
|
+
started.resolve();
|
|
278
|
+
void onDataWrapped(initialContent);
|
|
279
|
+
}
|
|
280
|
+
|
|
265
281
|
await started.promise;
|
|
266
282
|
} catch (e) {
|
|
267
283
|
void stop();
|
|
@@ -725,6 +741,21 @@ const resyncServicesBase = runInSerial(measureWrap(async function resyncServices
|
|
|
725
741
|
|
|
726
742
|
await fs.promises.writeFile(parameterPath, newParametersString);
|
|
727
743
|
|
|
744
|
+
let launchReason: "crashed" | "update";
|
|
745
|
+
if (!sameParameters) {
|
|
746
|
+
launchReason = "update";
|
|
747
|
+
} else {
|
|
748
|
+
launchReason = "crashed";
|
|
749
|
+
}
|
|
750
|
+
void recordLaunch({
|
|
751
|
+
serviceId: config.serviceId,
|
|
752
|
+
serviceKey: config.parameters.key,
|
|
753
|
+
screenName,
|
|
754
|
+
machineId,
|
|
755
|
+
reason: launchReason,
|
|
756
|
+
time: Date.now(),
|
|
757
|
+
});
|
|
758
|
+
|
|
728
759
|
await runScreenCommand({
|
|
729
760
|
screenName,
|
|
730
761
|
command: config.parameters.command,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { isNodeTrue, list, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
1
|
+
import { isNodeTrue, list, timeInDay, timeInMinute, timeInSecond } from "socket-function/src/misc";
|
|
2
2
|
import { nestArchives } from "../-a-archives/archives";
|
|
3
3
|
import { getArchivesBackblaze } from "../-a-archives/archivesBackBlaze";
|
|
4
4
|
import { getDomain } from "../config";
|
|
@@ -103,6 +103,69 @@ export const machineInfos = archiveJSONT<MachineInfo>(() => nestArchives("machin
|
|
|
103
103
|
export const serviceConfigs = archiveJSONT<ServiceConfig>(() => nestArchives("machines/service-configs/", getArchivesBackblaze(getDomain())));
|
|
104
104
|
export const machineConfigs = archiveJSONT<MachineConfig>(() => nestArchives("machines/machine-configs/", getArchivesBackblaze(getDomain())));
|
|
105
105
|
|
|
106
|
+
export type LaunchRecord = {
|
|
107
|
+
serviceId: string;
|
|
108
|
+
serviceKey: string;
|
|
109
|
+
screenName: string;
|
|
110
|
+
machineId: string;
|
|
111
|
+
reason: "crashed" | "update";
|
|
112
|
+
time: number;
|
|
113
|
+
};
|
|
114
|
+
|
|
115
|
+
export type LaunchSummary = {
|
|
116
|
+
time: number;
|
|
117
|
+
reason: string;
|
|
118
|
+
serviceKey: string;
|
|
119
|
+
key: string;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
const launches = lazy(() => nestArchives("machines/launches/", getArchivesBackblaze(getDomain())));
|
|
123
|
+
|
|
124
|
+
function formatLaunchDay(time: number): string {
|
|
125
|
+
let date = new Date(time);
|
|
126
|
+
let y = date.getUTCFullYear();
|
|
127
|
+
let m = String(date.getUTCMonth() + 1).padStart(2, "0");
|
|
128
|
+
let d = String(date.getUTCDate()).padStart(2, "0");
|
|
129
|
+
return `${y}-${m}-${d}`;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export async function recordLaunch(record: LaunchRecord) {
|
|
133
|
+
let day = formatLaunchDay(record.time);
|
|
134
|
+
let key = `${day}/${record.time}_${record.reason}_${record.serviceKey}`;
|
|
135
|
+
await launches().set(key, Buffer.from(JSON.stringify(record)));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function parseLaunchKey(key: string): LaunchSummary {
|
|
139
|
+
let rest = key;
|
|
140
|
+
let slash = key.indexOf("/");
|
|
141
|
+
if (slash >= 0) {
|
|
142
|
+
rest = key.slice(slash + 1);
|
|
143
|
+
}
|
|
144
|
+
let timeStr = "";
|
|
145
|
+
let afterTime = rest;
|
|
146
|
+
let firstUnderscore = rest.indexOf("_");
|
|
147
|
+
if (firstUnderscore >= 0) {
|
|
148
|
+
timeStr = rest.slice(0, firstUnderscore);
|
|
149
|
+
afterTime = rest.slice(firstUnderscore + 1);
|
|
150
|
+
}
|
|
151
|
+
let reason = afterTime;
|
|
152
|
+
let serviceKey = "";
|
|
153
|
+
let secondUnderscore = afterTime.indexOf("_");
|
|
154
|
+
if (secondUnderscore >= 0) {
|
|
155
|
+
reason = afterTime.slice(0, secondUnderscore);
|
|
156
|
+
serviceKey = afterTime.slice(secondUnderscore + 1);
|
|
157
|
+
}
|
|
158
|
+
let time = Number(timeStr);
|
|
159
|
+
if (!Number.isFinite(time)) {
|
|
160
|
+
console.warn(`Unparseable launch key (bad time): ${key}`);
|
|
161
|
+
time = Date.now();
|
|
162
|
+
}
|
|
163
|
+
if (reason !== "crashed" && reason !== "update") {
|
|
164
|
+
console.warn(`Launch key has unexpected reason "${reason}": ${key}`);
|
|
165
|
+
}
|
|
166
|
+
return { time, reason, serviceKey, key };
|
|
167
|
+
}
|
|
168
|
+
|
|
106
169
|
export function doRegisterNodeForMachineCleanup() {
|
|
107
170
|
if (isNode()) {
|
|
108
171
|
void SocketFunction.mountPromise.finally(() => {
|
|
@@ -371,6 +434,22 @@ export class MachineServiceControllerBase {
|
|
|
371
434
|
});
|
|
372
435
|
}
|
|
373
436
|
|
|
437
|
+
public async getRecentLaunches(sinceDays: number): Promise<LaunchSummary[]> {
|
|
438
|
+
let now = Date.now();
|
|
439
|
+
let dayStrings: string[] = [];
|
|
440
|
+
for (let i = 0; i <= sinceDays; i++) {
|
|
441
|
+
dayStrings.push(formatLaunchDay(now - i * timeInDay));
|
|
442
|
+
}
|
|
443
|
+
let keyLists = await Promise.all(dayStrings.map(day => launches().find(`${day}/`)));
|
|
444
|
+
let summaries: LaunchSummary[] = [];
|
|
445
|
+
for (let keys of keyLists) {
|
|
446
|
+
for (let key of keys) {
|
|
447
|
+
summaries.push(parseLaunchKey(key));
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
return summaries;
|
|
451
|
+
}
|
|
452
|
+
|
|
374
453
|
public async deployFunctions(config: {
|
|
375
454
|
functionSpecs: FunctionSpec[];
|
|
376
455
|
prefixes: string[];
|
|
@@ -469,6 +548,7 @@ export const MachineServiceController = getSyncedController(
|
|
|
469
548
|
getPendingFunctions: {},
|
|
470
549
|
deployFunctions: {},
|
|
471
550
|
getLiveFunctions: {},
|
|
551
|
+
getRecentLaunches: {},
|
|
472
552
|
}),
|
|
473
553
|
() => ({
|
|
474
554
|
hooks: [assertIsManagementUser],
|
|
@@ -497,6 +577,7 @@ export const MachineServiceController = getSyncedController(
|
|
|
497
577
|
getGitInfo: ["gitInfo"],
|
|
498
578
|
getPendingFunctions: ["gitInfo"],
|
|
499
579
|
getLiveFunctions: ["gitInfo"],
|
|
580
|
+
getRecentLaunches: ["launches"],
|
|
500
581
|
}
|
|
501
582
|
}
|
|
502
583
|
);
|
|
@@ -142,7 +142,7 @@ class NodeConnectionsControllerBase {
|
|
|
142
142
|
}
|
|
143
143
|
|
|
144
144
|
public async getEntryPoint_forBrowser(nodeId: string) {
|
|
145
|
-
return await NodeCapabilitiesController.nodes[nodeId].
|
|
145
|
+
return (await NodeCapabilitiesController.nodes[nodeId].getMetadata()).entryPoint;
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
public async getAllNodeIds() {
|
|
@@ -79,10 +79,11 @@ export class NodeViewer extends qreact.Component {
|
|
|
79
79
|
let data: NodeData = { nodeId };
|
|
80
80
|
try {
|
|
81
81
|
data.table = await controller.getMiscInfo(nodeId);
|
|
82
|
-
|
|
83
|
-
data.
|
|
84
|
-
data.
|
|
85
|
-
data.
|
|
82
|
+
let metadata = await controller.live_getMetadata(nodeId);
|
|
83
|
+
data.live_isAlive = !!metadata;
|
|
84
|
+
data.live_exposedControllers = metadata?.exposedControllers;
|
|
85
|
+
data.live_entryPoint = metadata?.entryPoint;
|
|
86
|
+
data.live_trueTimeOffset = metadata?.trueTimeOffset;
|
|
86
87
|
data.live_authorityPaths = await controller.live_getAuthorityPaths(nodeId);
|
|
87
88
|
data.ip = await controller.getNodeIP(nodeId);
|
|
88
89
|
if (data.ip === ourExternalIP || data.ip === ourIP) {
|
|
@@ -484,27 +485,18 @@ class NodeViewerControllerBase {
|
|
|
484
485
|
await syncNodesNow();
|
|
485
486
|
}
|
|
486
487
|
|
|
487
|
-
public async
|
|
488
|
-
return await NodeCapabilitiesController.nodes[nodeId].
|
|
488
|
+
public async live_getMetadata(nodeId: string) {
|
|
489
|
+
return await errorToUndefinedSilent(NodeCapabilitiesController.nodes[nodeId].getMetadata());
|
|
489
490
|
}
|
|
490
491
|
|
|
491
492
|
public async getControllerNodeIdList(controller: SocketRegistered<{}>) {
|
|
492
493
|
return await getControllerNodeIdList(controller);
|
|
493
494
|
}
|
|
494
495
|
|
|
495
|
-
public async live_isAlive(nodeId: string) {
|
|
496
|
-
return !!await errorToUndefinedSilent(NodeCapabilitiesController.nodes[nodeId].getEntryPoint());
|
|
497
|
-
}
|
|
498
496
|
public async live_getAuthorityPaths(nodeId: string) {
|
|
499
497
|
let topo = await NodeMetadataController.nodes[nodeId].debugGetTopologyEntry(nodeId);
|
|
500
498
|
return topo?.authoritySpec;
|
|
501
499
|
}
|
|
502
|
-
public async live_getEntryPoint(nodeId: string) {
|
|
503
|
-
return NodeCapabilitiesController.nodes[nodeId].getEntryPoint();
|
|
504
|
-
}
|
|
505
|
-
public async live_getTrueTimeOffset(nodeId: string) {
|
|
506
|
-
return NodeCapabilitiesController.nodes[nodeId].getTrueTimeOffset();
|
|
507
|
-
}
|
|
508
500
|
|
|
509
501
|
public async getMiscInfo(nodeId: string): Promise<{
|
|
510
502
|
columns: ColumnsType;
|
|
@@ -521,18 +513,19 @@ class NodeViewerControllerBase {
|
|
|
521
513
|
row[columnName] = "Error: " + e.stack;
|
|
522
514
|
}
|
|
523
515
|
}
|
|
516
|
+
let metadataPromise = NodeCapabilitiesController.nodes[nodeId].getMetadata();
|
|
524
517
|
let promises = [
|
|
525
518
|
wrapAddTableValue("paths|paths", {}, async () => {
|
|
526
519
|
let live_authorityPaths = (await NodeMetadataController.nodes[nodeId].debugGetPathAuthorities(nodeId));
|
|
527
520
|
return JSON.stringify(live_authorityPaths);
|
|
528
521
|
}),
|
|
529
522
|
wrapAddTableValue("paths|functions", {}, async () => {
|
|
530
|
-
let
|
|
531
|
-
return
|
|
523
|
+
let metadata = await metadataPromise;
|
|
524
|
+
return metadata.functionRunnerShards.map(x => `${x.domainName}[${x.shardRange.startFraction}-${x.shardRange.endFraction}]${x.secondaryShardRange && `+[${x.secondaryShardRange?.startFraction || 0}-${x.secondaryShardRange?.endFraction || 0}]` || ""}`).join(" | ");
|
|
532
525
|
}),
|
|
533
526
|
wrapAddTableValue("uptime", { formatter: "timeSpan" }, async () => {
|
|
534
|
-
let
|
|
535
|
-
return Date.now() - startupTime;
|
|
527
|
+
let metadata = await metadataPromise;
|
|
528
|
+
return Date.now() - metadata.startupTime;
|
|
536
529
|
}),
|
|
537
530
|
wrapAddTableValue("port", {}, async () => {
|
|
538
531
|
return nodeId.split(":").at(-1);
|
|
@@ -557,8 +550,8 @@ class NodeViewerControllerBase {
|
|
|
557
550
|
].filter(x => x);
|
|
558
551
|
}),
|
|
559
552
|
wrapAddTableValue("capabilities|capabilities", {}, async () => {
|
|
560
|
-
let
|
|
561
|
-
return
|
|
553
|
+
let metadata = await metadataPromise;
|
|
554
|
+
return metadata.exposedControllers.map(x => x.split("-")[0]);
|
|
562
555
|
}),
|
|
563
556
|
];
|
|
564
557
|
await Promise.allSettled(promises);
|
|
@@ -590,12 +583,9 @@ export const NodeViewerController = SocketFunction.register(
|
|
|
590
583
|
getExternalInspectURL: {},
|
|
591
584
|
verifyAccess: {},
|
|
592
585
|
getAllNodeIds: {},
|
|
593
|
-
getExposedControllers: {},
|
|
594
586
|
getControllerNodeIdList: {},
|
|
595
|
-
|
|
587
|
+
live_getMetadata: {},
|
|
596
588
|
live_getAuthorityPaths: {},
|
|
597
|
-
live_getEntryPoint: {},
|
|
598
|
-
live_getTrueTimeOffset: {},
|
|
599
589
|
getMiscInfo: {},
|
|
600
590
|
|
|
601
591
|
forceRefreshNodes: {},
|
|
@@ -168,17 +168,17 @@ async function getNodeInfos(): Promise<NodeInfo[]> {
|
|
|
168
168
|
const nodes = await getAllNodeIds();
|
|
169
169
|
return Promise.all(
|
|
170
170
|
nodes.map(async (nodeId): Promise<NodeInfo> => {
|
|
171
|
-
const [
|
|
171
|
+
const [metadata, inspectUrl] = await Promise.all([
|
|
172
172
|
timeoutToUndefinedSilent(
|
|
173
173
|
NODE_INFO_TIMEOUT_MS,
|
|
174
|
-
NodeCapabilitiesController.nodes[nodeId].
|
|
174
|
+
NodeCapabilitiesController.nodes[nodeId].getMetadata(),
|
|
175
175
|
),
|
|
176
176
|
timeoutToUndefinedSilent(
|
|
177
177
|
NODE_INFO_TIMEOUT_MS,
|
|
178
178
|
NodeCapabilitiesController.nodes[nodeId].getInspectURL(),
|
|
179
179
|
),
|
|
180
180
|
]);
|
|
181
|
-
return { nodeId, entryPoint, inspectUrl };
|
|
181
|
+
return { nodeId, entryPoint: metadata?.entryPoint, inspectUrl };
|
|
182
182
|
}),
|
|
183
183
|
);
|
|
184
184
|
}
|
|
@@ -290,8 +290,8 @@ export class IndexedLogs<T> {
|
|
|
290
290
|
if (!hasLogger) return false;
|
|
291
291
|
// NOTE: Prefer to do the searching on the move logs service. However, if it's not available, any service can do searching. It just might lag that server...
|
|
292
292
|
if (preferredOnly) {
|
|
293
|
-
let
|
|
294
|
-
if (!entryPoint
|
|
293
|
+
let metadata = await timeoutToUndefinedSilent(2500, NodeCapabilitiesController.nodes[nodeId].getMetadata());
|
|
294
|
+
if (!metadata?.entryPoint.includes("movelogs")) return false;
|
|
295
295
|
}
|
|
296
296
|
added = true;
|
|
297
297
|
|