plugin-cluster-manager 1.1.7 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client.js +1 -0
- package/dist/client/AclCacheManager.d.ts +2 -0
- package/dist/client/CacheMonitor.d.ts +2 -0
- package/dist/client/ClusterManagerLayout.d.ts +2 -0
- package/dist/client/ClusterNodes.d.ts +2 -0
- package/dist/client/ContainerOrchestrator.d.ts +2 -0
- package/dist/client/Doctor.d.ts +2 -0
- package/dist/client/EventQueueMonitor.d.ts +2 -0
- package/dist/client/LockMonitor.d.ts +2 -0
- package/dist/client/NginxCacheManager.d.ts +2 -0
- package/dist/client/PackageInstaller.d.ts +2 -0
- package/dist/client/PluginOperations.d.ts +2 -0
- package/dist/client/RedisMonitor.d.ts +2 -0
- package/dist/client/TaskManager.d.ts +2 -0
- package/dist/client/WorkflowExecutions.d.ts +2 -0
- package/dist/client/index.d.ts +5 -0
- package/dist/client/index.js +1 -1
- package/dist/client/utils/clientSafeCache.d.ts +3 -0
- package/dist/client/utils/requestDedupInterceptor.d.ts +2 -0
- package/dist/client/utils.d.ts +12 -0
- package/dist/externalVersion.js +5 -5
- package/dist/index.d.ts +2 -0
- package/dist/locale/en-US.json +97 -1
- package/dist/locale/vi-VN.json +98 -1
- package/dist/locale/zh-CN.json +98 -1
- package/dist/server/actions/acl-cache.d.ts +53 -0
- package/dist/server/actions/acl-cache.js +1 -1
- package/dist/server/actions/cache-monitor.d.ts +33 -0
- package/dist/server/actions/cache-monitor.js +301 -0
- package/dist/server/actions/cluster-nodes.d.ts +64 -0
- package/dist/server/actions/cluster-nodes.js +394 -10
- package/dist/server/actions/doctor.d.ts +82 -0
- package/dist/server/actions/doctor.js +1250 -0
- package/dist/server/actions/event-queue-monitor.d.ts +13 -0
- package/dist/server/actions/lock-monitor.d.ts +19 -0
- package/dist/server/actions/orchestrator.d.ts +58 -0
- package/dist/server/actions/package-manager.d.ts +6 -0
- package/dist/server/actions/plugin-operations.d.ts +6 -0
- package/dist/server/actions/redis-monitor.d.ts +12 -0
- package/dist/server/actions/tasks.d.ts +7 -0
- package/dist/server/actions/workflow-executions.d.ts +7 -0
- package/dist/server/adapters/redis-lock-adapter.d.ts +15 -0
- package/dist/server/adapters/redis-node-registry.d.ts +12 -0
- package/dist/server/adapters/redis-pubsub-adapter.d.ts +16 -0
- package/dist/server/collections/app.d.ts +8 -0
- package/dist/server/collections/cluster-manager-acl-cache.d.ts +22 -0
- package/dist/server/collections/cluster-manager-cache-mgr.d.ts +22 -0
- package/dist/server/collections/cluster-manager-cluster.d.ts +22 -0
- package/dist/server/collections/cluster-manager-doctor-runs.d.ts +3 -0
- package/dist/server/collections/cluster-manager-doctor-runs.js +52 -0
- package/dist/server/collections/cluster-manager-doctor.d.ts +18 -0
- package/dist/server/collections/cluster-manager-doctor.js +44 -0
- package/dist/server/collections/cluster-manager-lock.d.ts +22 -0
- package/dist/server/collections/cluster-manager-plugins.d.ts +18 -0
- package/dist/server/collections/cluster-manager-queue.d.ts +22 -0
- package/dist/server/collections/cluster-manager-redis.d.ts +22 -0
- package/dist/server/collections/cluster-manager-workflow.d.ts +22 -0
- package/dist/server/collections/cluster-manager.d.ts +22 -0
- package/dist/server/collections/orchestrator-settings.d.ts +59 -0
- package/dist/server/collections/orchestrator-stacks.d.ts +102 -0
- package/dist/server/collections/worker-orchestrator.d.ts +22 -0
- package/dist/server/collections/worker-packages-configs.d.ts +3 -0
- package/dist/server/collections/worker-packages.d.ts +22 -0
- package/dist/server/hooks/cacheInvalidationHooks.d.ts +1 -0
- package/dist/server/hooks/cacheInvalidationHooks.js +81 -0
- package/dist/server/index.d.ts +1 -0
- package/dist/server/middlewares/listMetaCacheMiddleware.d.ts +2 -0
- package/dist/server/middlewares/listMetaCacheMiddleware.js +79 -0
- package/dist/server/orchestrator/PackageManager.d.ts +39 -0
- package/dist/server/orchestrator/PackageManager.js +83 -27
- package/dist/server/orchestrator/docker-adapter.d.ts +41 -0
- package/dist/server/orchestrator/index.d.ts +4 -0
- package/dist/server/orchestrator/k8s-adapter.d.ts +50 -0
- package/dist/server/orchestrator/leader-election.d.ts +48 -0
- package/dist/server/orchestrator/types.d.ts +84 -0
- package/dist/server/plugin.d.ts +26 -0
- package/dist/server/plugin.js +70 -8
- package/dist/server/utils/node.d.ts +6 -0
- package/dist/server/utils/redis.d.ts +29 -0
- package/dist/server/utils/versionManager.d.ts +10 -0
- package/dist/server/utils/versionManager.js +91 -0
- package/dist/shared/packages.d.ts +23 -0
- package/package.json +41 -41
- package/server.js +1 -0
- package/src/client/CacheMonitor.tsx +166 -179
- package/src/client/ClusterManagerLayout.tsx +48 -42
- package/src/client/ClusterNodes.tsx +691 -418
- package/src/client/Doctor.tsx +559 -0
- package/src/client/NginxCacheManager.tsx +415 -0
- package/src/client/PluginOperations.tsx +234 -234
- package/src/client/index.tsx +22 -14
- package/src/client/utils/clientSafeCache.ts +41 -0
- package/src/client/utils/requestDedupInterceptor.ts +213 -0
- package/src/locale/en-US.json +97 -1
- package/src/locale/vi-VN.json +98 -1
- package/src/locale/zh-CN.json +98 -1
- package/src/server/__tests__/doctor.test.ts +53 -0
- package/src/server/actions/acl-cache.ts +272 -272
- package/src/server/actions/cache-monitor.ts +453 -116
- package/src/server/actions/cluster-nodes.ts +882 -378
- package/src/server/actions/doctor.ts +1540 -0
- package/src/server/collections/cluster-manager-doctor-runs.ts +23 -0
- package/src/server/collections/cluster-manager-doctor.ts +19 -0
- package/src/server/hooks/cacheInvalidationHooks.ts +58 -0
- package/src/server/middlewares/listMetaCacheMiddleware.ts +55 -0
- package/src/server/orchestrator/PackageManager.ts +19 -15
- package/src/server/plugin.ts +353 -263
- package/src/server/utils/versionManager.ts +69 -0
|
@@ -0,0 +1,1250 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file is part of the NocoBase (R) project.
|
|
3
|
+
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
|
|
4
|
+
* Authors: NocoBase Team.
|
|
5
|
+
*
|
|
6
|
+
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
|
+
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
var __create = Object.create;
|
|
11
|
+
var __defProp = Object.defineProperty;
|
|
12
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
13
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
14
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
15
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
16
|
+
var __export = (target, all) => {
|
|
17
|
+
for (var name in all)
|
|
18
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
19
|
+
};
|
|
20
|
+
var __copyProps = (to, from, except, desc) => {
|
|
21
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
22
|
+
for (let key of __getOwnPropNames(from))
|
|
23
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
24
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
25
|
+
}
|
|
26
|
+
return to;
|
|
27
|
+
};
|
|
28
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
29
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
30
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
31
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
32
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
33
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
34
|
+
mod
|
|
35
|
+
));
|
|
36
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
37
|
+
var doctor_exports = {};
|
|
38
|
+
__export(doctor_exports, {
|
|
39
|
+
collectLocalDoctorSnapshot: () => collectLocalDoctorSnapshot,
|
|
40
|
+
doctorActions: () => doctorActions
|
|
41
|
+
});
|
|
42
|
+
module.exports = __toCommonJS(doctor_exports);
|
|
43
|
+
var import_crypto = __toESM(require("crypto"));
|
|
44
|
+
var import_fs = require("fs");
|
|
45
|
+
var import_os = __toESM(require("os"));
|
|
46
|
+
var import_path = __toESM(require("path"));
|
|
47
|
+
var import_redis_node_registry = require("../adapters/redis-node-registry");
|
|
48
|
+
var import_node = require("../utils/node");
|
|
49
|
+
var import_redis = require("../utils/redis");
|
|
50
|
+
var import_packages = require("../../shared/packages");
|
|
51
|
+
const ACTIVE_RUN_KEY = "cluster-manager:doctor:active";
|
|
52
|
+
const RESPONSE_KEY_PREFIX = "cluster-manager:doctor-response:";
|
|
53
|
+
const FINISH_LOCK_PREFIX = "cluster-manager:doctor:finish-lock:";
|
|
54
|
+
const DEFAULT_DURATION_MS = 12e4;
|
|
55
|
+
const MAX_DURATION_MS = 12e4;
|
|
56
|
+
const MIN_DURATION_MS = 1e4;
|
|
57
|
+
const LOCK_BUFFER_MS = 3e4;
|
|
58
|
+
const SNAPSHOT_RESPONSE_TTL_SECONDS = 90;
|
|
59
|
+
const FINISH_LOCK_TTL_SECONDS = 90;
|
|
60
|
+
const MAX_NODE_LOG_LINES = 800;
|
|
61
|
+
const MAX_CONTAINER_LOG_LINES = 200;
|
|
62
|
+
const LOG_PREFIXES = ["system", "system_error", "request"];
|
|
63
|
+
class ActiveDoctorRunError extends Error {
|
|
64
|
+
constructor(activeRun) {
|
|
65
|
+
super("A diagnostic session is already running.");
|
|
66
|
+
this.activeRun = activeRun;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
const timers = /* @__PURE__ */ new Map();
|
|
70
|
+
let localActiveRun = null;
|
|
71
|
+
function getApp(app) {
|
|
72
|
+
return app;
|
|
73
|
+
}
|
|
74
|
+
function getPayload(ctx) {
|
|
75
|
+
var _a;
|
|
76
|
+
return ctx.action.params.values || ((_a = ctx.request) == null ? void 0 : _a.body) || {};
|
|
77
|
+
}
|
|
78
|
+
function clampDuration(value) {
|
|
79
|
+
const duration = Number(value) || DEFAULT_DURATION_MS;
|
|
80
|
+
return Math.min(Math.max(duration, MIN_DURATION_MS), MAX_DURATION_MS);
|
|
81
|
+
}
|
|
82
|
+
function modelToJSON(model) {
|
|
83
|
+
if (!model) return null;
|
|
84
|
+
if (typeof model.toJSON === "function") {
|
|
85
|
+
return model.toJSON();
|
|
86
|
+
}
|
|
87
|
+
return {};
|
|
88
|
+
}
|
|
89
|
+
function getModelValue(model, key) {
|
|
90
|
+
if ("get" in model && typeof model.get === "function") {
|
|
91
|
+
return model.get(key);
|
|
92
|
+
}
|
|
93
|
+
return model[key];
|
|
94
|
+
}
|
|
95
|
+
function getRepository(app, name) {
|
|
96
|
+
return getApp(app).db.getRepository(name);
|
|
97
|
+
}
|
|
98
|
+
function getUserLabel(ctx) {
|
|
99
|
+
const state = ctx.state;
|
|
100
|
+
const currentUser = state == null ? void 0 : state.currentUser;
|
|
101
|
+
return String((currentUser == null ? void 0 : currentUser.nickname) || (currentUser == null ? void 0 : currentUser.username) || (currentUser == null ? void 0 : currentUser.id) || "unknown");
|
|
102
|
+
}
|
|
103
|
+
function getErrorMessage(error) {
|
|
104
|
+
return error instanceof Error ? error.message : String(error);
|
|
105
|
+
}
|
|
106
|
+
function parseJson(value, fallback) {
|
|
107
|
+
if (!value) return fallback;
|
|
108
|
+
if (typeof value !== "string") return value;
|
|
109
|
+
try {
|
|
110
|
+
return JSON.parse(value);
|
|
111
|
+
} catch {
|
|
112
|
+
return fallback;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
function normalizeList(value) {
|
|
116
|
+
if (!Array.isArray(value)) return [];
|
|
117
|
+
return Array.from(
|
|
118
|
+
new Set(
|
|
119
|
+
value.filter((item) => typeof item === "string").map((item) => item.trim()).filter(Boolean)
|
|
120
|
+
)
|
|
121
|
+
);
|
|
122
|
+
}
|
|
123
|
+
function normalizePackageMap(packages) {
|
|
124
|
+
return {
|
|
125
|
+
apt: normalizeList(packages == null ? void 0 : packages.apt),
|
|
126
|
+
npm: normalizeList(packages == null ? void 0 : packages.npm),
|
|
127
|
+
python: normalizeList(packages == null ? void 0 : packages.python)
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
function parseCustomPackages(value) {
|
|
131
|
+
const custom = parseJson(value, { python: [], node: [], npm: [] });
|
|
132
|
+
return {
|
|
133
|
+
python: normalizeList(custom.python),
|
|
134
|
+
node: normalizeList(custom.node),
|
|
135
|
+
npm: normalizeList(custom.npm)
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
function parsePackageWhitelist(value) {
|
|
139
|
+
const whitelist = parseJson(value, {});
|
|
140
|
+
return {
|
|
141
|
+
apt: normalizeList(whitelist.apt),
|
|
142
|
+
npm: normalizeList([...whitelist.npm || [], ...whitelist.node || []]),
|
|
143
|
+
python: normalizeList(whitelist.python)
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
function diffPackages(expected, installed) {
|
|
147
|
+
return {
|
|
148
|
+
apt: expected.apt.filter((pkg) => !installed.apt.includes(pkg)),
|
|
149
|
+
npm: expected.npm.filter((pkg) => !installed.npm.includes(pkg)),
|
|
150
|
+
python: expected.python.filter((pkg) => !installed.python.includes(pkg))
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function countPackages(packages) {
|
|
154
|
+
return packages.apt.length + packages.npm.length + packages.python.length;
|
|
155
|
+
}
|
|
156
|
+
function getNodeRole(node) {
|
|
157
|
+
if (node.isSandbox) {
|
|
158
|
+
return "sandbox";
|
|
159
|
+
}
|
|
160
|
+
const workerMode = node.workerMode || "main";
|
|
161
|
+
return workerMode === "worker" || workerMode === "task" || workerMode === "*" ? "worker" : "app";
|
|
162
|
+
}
|
|
163
|
+
function getSafeEnv() {
|
|
164
|
+
return {
|
|
165
|
+
APP_ENV: process.env.APP_ENV,
|
|
166
|
+
APP_NAME: process.env.APP_NAME,
|
|
167
|
+
APP_ROLE: process.env.APP_ROLE,
|
|
168
|
+
APP_PORT: process.env.APP_PORT,
|
|
169
|
+
CLUSTER_MODE: process.env.CLUSTER_MODE,
|
|
170
|
+
WORKER_MODE: process.env.WORKER_MODE,
|
|
171
|
+
LOGGER_LEVEL: process.env.LOGGER_LEVEL,
|
|
172
|
+
LOGGER_FORMAT: process.env.LOGGER_FORMAT,
|
|
173
|
+
LOGGER_TRANSPORT: process.env.LOGGER_TRANSPORT,
|
|
174
|
+
NOCOBASE_VERSION: process.env.NOCOBASE_VERSION,
|
|
175
|
+
DB_DIALECT: process.env.DB_DIALECT
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
function redactText(value) {
|
|
179
|
+
return value.replace(
|
|
180
|
+
/(authorization|cookie|set-cookie|token|secret|password|passwd|pwd|api[-_]?key)=([^,\s&]+)/gi,
|
|
181
|
+
"$1=[REDACTED]"
|
|
182
|
+
).replace(/(Bearer\s+)[A-Za-z0-9._~+/=-]+/gi, "$1[REDACTED]").replace(/:\/\/([^:/\s]+):([^@/\s]+)@/g, "://[REDACTED]:[REDACTED]@");
|
|
183
|
+
}
|
|
184
|
+
function getLogDir(app) {
|
|
185
|
+
const appLike = getApp(app);
|
|
186
|
+
const logBasePath = process.env.LOGGER_BASE_PATH || import_path.default.resolve(process.cwd(), "storage", "logs");
|
|
187
|
+
const appName = process.env.APP_NAME || appLike.name || "main";
|
|
188
|
+
return import_path.default.resolve(logBasePath, appName);
|
|
189
|
+
}
|
|
190
|
+
async function readTailLines(filePath, maxLines) {
|
|
191
|
+
try {
|
|
192
|
+
const stat = await import_fs.promises.stat(filePath);
|
|
193
|
+
const bufferSize = Math.min(stat.size, Math.max(maxLines, 1) * 2048);
|
|
194
|
+
const buffer = Buffer.alloc(bufferSize);
|
|
195
|
+
const fh = await import_fs.promises.open(filePath, "r");
|
|
196
|
+
try {
|
|
197
|
+
await fh.read(buffer, 0, bufferSize, Math.max(0, stat.size - bufferSize));
|
|
198
|
+
} finally {
|
|
199
|
+
await fh.close();
|
|
200
|
+
}
|
|
201
|
+
return buffer.toString("utf8").split(/\r?\n/).filter((line) => line.trim()).slice(-maxLines);
|
|
202
|
+
} catch {
|
|
203
|
+
return [];
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
function parseLogTimestamp(line) {
|
|
207
|
+
try {
|
|
208
|
+
const parsed = JSON.parse(line);
|
|
209
|
+
if (parsed.timestamp) {
|
|
210
|
+
const time2 = Date.parse(parsed.timestamp);
|
|
211
|
+
return Number.isFinite(time2) ? time2 : null;
|
|
212
|
+
}
|
|
213
|
+
} catch {
|
|
214
|
+
}
|
|
215
|
+
const match = line.match(/(\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/);
|
|
216
|
+
if (!match) return null;
|
|
217
|
+
const value = match[1].includes("T") ? match[1] : match[1].replace(" ", "T");
|
|
218
|
+
const time = Date.parse(value);
|
|
219
|
+
return Number.isFinite(time) ? time : null;
|
|
220
|
+
}
|
|
221
|
+
function parseDiagnosticLine(source, rawLine) {
|
|
222
|
+
var _a;
|
|
223
|
+
const redacted = redactText(rawLine);
|
|
224
|
+
try {
|
|
225
|
+
const parsed = JSON.parse(redacted);
|
|
226
|
+
return {
|
|
227
|
+
source,
|
|
228
|
+
line: redacted,
|
|
229
|
+
timestamp: typeof parsed.timestamp === "string" ? parsed.timestamp : void 0,
|
|
230
|
+
level: typeof parsed.level === "string" ? parsed.level.toLowerCase() : void 0,
|
|
231
|
+
message: typeof parsed.message === "string" ? parsed.message : void 0,
|
|
232
|
+
stack: typeof parsed.stack === "string" ? parsed.stack : void 0
|
|
233
|
+
};
|
|
234
|
+
} catch {
|
|
235
|
+
const levelMatch = redacted.match(/\b(?:level=|\[)(error|warn|warning|info|debug|trace)\b/i);
|
|
236
|
+
const timestamp = parseLogTimestamp(redacted);
|
|
237
|
+
return {
|
|
238
|
+
source,
|
|
239
|
+
line: redacted,
|
|
240
|
+
timestamp: timestamp ? new Date(timestamp).toISOString() : void 0,
|
|
241
|
+
level: (_a = levelMatch == null ? void 0 : levelMatch[1]) == null ? void 0 : _a.toLowerCase().replace("warning", "warn"),
|
|
242
|
+
message: redacted.replace(/^\d{4}-\d{2}-\d{2}[^[]*/, "").trim()
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
function normalizeSignature(value) {
|
|
247
|
+
return value.replace(/\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g, "<time>").replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, "<uuid>").replace(/\b[0-9a-f]{16,}\b/gi, "<hex>").replace(/\b\d+\b/g, "<num>").replace(/\s+/g, " ").trim().slice(0, 220);
|
|
248
|
+
}
|
|
249
|
+
function analyzeLogLines(lines) {
|
|
250
|
+
const levels = {};
|
|
251
|
+
const signatures = /* @__PURE__ */ new Map();
|
|
252
|
+
for (const item of lines) {
|
|
253
|
+
const level = item.level || (/error/i.test(item.line) ? "error" : /warn/i.test(item.line) ? "warn" : "info");
|
|
254
|
+
levels[level] = (levels[level] || 0) + 1;
|
|
255
|
+
if (level !== "error" && level !== "warn") {
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
const base = item.stack || item.message || item.line;
|
|
259
|
+
const signature = normalizeSignature(base);
|
|
260
|
+
const key = `${level}:${signature}`;
|
|
261
|
+
const existing = signatures.get(key);
|
|
262
|
+
if (existing) {
|
|
263
|
+
existing.count++;
|
|
264
|
+
existing.lastSeen = item.timestamp || existing.lastSeen;
|
|
265
|
+
if (!existing.sources.includes(item.source)) {
|
|
266
|
+
existing.sources.push(item.source);
|
|
267
|
+
}
|
|
268
|
+
if (existing.samples.length < 3) {
|
|
269
|
+
existing.samples.push(item.line.slice(0, 1e3));
|
|
270
|
+
}
|
|
271
|
+
} else {
|
|
272
|
+
signatures.set(key, {
|
|
273
|
+
signature,
|
|
274
|
+
level,
|
|
275
|
+
count: 1,
|
|
276
|
+
firstSeen: item.timestamp,
|
|
277
|
+
lastSeen: item.timestamp,
|
|
278
|
+
sources: [item.source],
|
|
279
|
+
samples: [item.line.slice(0, 1e3)]
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return {
|
|
284
|
+
totalLines: lines.length,
|
|
285
|
+
levels,
|
|
286
|
+
signatures: [...signatures.values()].sort((a, b) => b.count - a.count).slice(0, 50)
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
async function readDiagnosticLogs(app, options) {
|
|
290
|
+
const logDir = getLogDir(app);
|
|
291
|
+
const maxLines = Math.min(Number(options.maxLines) || MAX_NODE_LOG_LINES, MAX_NODE_LOG_LINES);
|
|
292
|
+
const sinceMs = Number(options.sinceMs) || 0;
|
|
293
|
+
const untilMs = Number(options.untilMs) || Date.now();
|
|
294
|
+
const files = [];
|
|
295
|
+
const lines = [];
|
|
296
|
+
let names = [];
|
|
297
|
+
try {
|
|
298
|
+
names = await import_fs.promises.readdir(logDir);
|
|
299
|
+
} catch {
|
|
300
|
+
return { files, lines, analysis: analyzeLogLines(lines) };
|
|
301
|
+
}
|
|
302
|
+
const candidates = names.filter((name) => name.endsWith(".log") && LOG_PREFIXES.some((prefix) => name.startsWith(prefix))).sort().reverse().slice(0, 12);
|
|
303
|
+
const perFileLimit = Math.max(50, Math.ceil(maxLines / Math.max(candidates.length || 1, 1)));
|
|
304
|
+
for (const file of candidates) {
|
|
305
|
+
const filePath = import_path.default.resolve(logDir, file);
|
|
306
|
+
const rawLines = await readTailLines(filePath, perFileLimit);
|
|
307
|
+
const parsedLines = rawLines.map((line) => parseDiagnosticLine(file, line)).filter((line) => {
|
|
308
|
+
const timestamp = line.timestamp ? Date.parse(line.timestamp) : parseLogTimestamp(line.line);
|
|
309
|
+
if (!timestamp) return true;
|
|
310
|
+
return timestamp >= sinceMs && timestamp <= untilMs;
|
|
311
|
+
});
|
|
312
|
+
if (parsedLines.length > 0) {
|
|
313
|
+
files.push({ file, lineCount: parsedLines.length });
|
|
314
|
+
lines.push(...parsedLines);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
const limitedLines = lines.slice(-maxLines);
|
|
318
|
+
return {
|
|
319
|
+
files,
|
|
320
|
+
lines: limitedLines,
|
|
321
|
+
analysis: analyzeLogLines(limitedLines)
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
async function getApplicationPluginRows(app) {
|
|
325
|
+
try {
|
|
326
|
+
const repo = getRepository(app, "applicationPlugins");
|
|
327
|
+
const rows = await repo.find({ sort: ["name"] });
|
|
328
|
+
return rows.map((row) => modelToJSON(row) || {});
|
|
329
|
+
} catch {
|
|
330
|
+
return [];
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
function getLoadedPlugin(app, name, packageName) {
|
|
334
|
+
const pm = getApp(app).pm;
|
|
335
|
+
if (!(pm == null ? void 0 : pm.get)) return null;
|
|
336
|
+
const pluginName = typeof name === "string" ? name : "";
|
|
337
|
+
const pluginPackageName = typeof packageName === "string" ? packageName : "";
|
|
338
|
+
return pm.get(pluginName) || pm.get(pluginPackageName) || null;
|
|
339
|
+
}
|
|
340
|
+
function getRuntimePluginVersion(instance) {
|
|
341
|
+
var _a, _b;
|
|
342
|
+
const plugin = instance;
|
|
343
|
+
return ((_b = (_a = plugin == null ? void 0 : plugin.options) == null ? void 0 : _a.packageJson) == null ? void 0 : _b.version) || (plugin == null ? void 0 : plugin.version);
|
|
344
|
+
}
|
|
345
|
+
async function getLocalPluginSnapshot(app) {
|
|
346
|
+
const rows = await getApplicationPluginRows(app);
|
|
347
|
+
return rows.map((row) => {
|
|
348
|
+
const instance = getLoadedPlugin(app, row.name, row.packageName);
|
|
349
|
+
return {
|
|
350
|
+
name: typeof row.name === "string" ? row.name : void 0,
|
|
351
|
+
packageName: typeof row.packageName === "string" ? row.packageName : void 0,
|
|
352
|
+
enabled: Boolean(row.enabled),
|
|
353
|
+
dbVersion: typeof row.version === "string" ? row.version : void 0,
|
|
354
|
+
loaded: Boolean(instance),
|
|
355
|
+
runtimeVersion: getRuntimePluginVersion(instance)
|
|
356
|
+
};
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
async function collectLocalDoctorSnapshot(app, options = {}) {
|
|
360
|
+
const workerMode = process.env.WORKER_MODE || "main";
|
|
361
|
+
const node = {
|
|
362
|
+
hostname: import_os.default.hostname(),
|
|
363
|
+
pid: process.pid,
|
|
364
|
+
workerMode,
|
|
365
|
+
role: getNodeRole({ workerMode, isSandbox: process.env.SKILL_HUB_SANDBOX === "true" }),
|
|
366
|
+
appVersion: process.env.NOCOBASE_VERSION || process.version,
|
|
367
|
+
nodeVersion: process.version,
|
|
368
|
+
platform: process.platform,
|
|
369
|
+
arch: process.arch,
|
|
370
|
+
uptime: process.uptime(),
|
|
371
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true"
|
|
372
|
+
};
|
|
373
|
+
return {
|
|
374
|
+
nodeId: (0, import_node.getLocalNodeId)(app),
|
|
375
|
+
node,
|
|
376
|
+
memory: process.memoryUsage(),
|
|
377
|
+
os: {
|
|
378
|
+
totalMemory: import_os.default.totalmem(),
|
|
379
|
+
freeMemory: import_os.default.freemem(),
|
|
380
|
+
cpuCount: import_os.default.cpus().length,
|
|
381
|
+
loadAvg: import_os.default.loadavg()
|
|
382
|
+
},
|
|
383
|
+
env: getSafeEnv(),
|
|
384
|
+
plugins: await getLocalPluginSnapshot(app),
|
|
385
|
+
logs: await readDiagnosticLogs(app, options),
|
|
386
|
+
collectedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
async function getActiveRunState(app) {
|
|
390
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
391
|
+
if (redis) {
|
|
392
|
+
const raw = await redis.sendCommand(["GET", ACTIVE_RUN_KEY]);
|
|
393
|
+
if (typeof raw !== "string" || !raw) return null;
|
|
394
|
+
return parseJson(raw, null);
|
|
395
|
+
}
|
|
396
|
+
if (!localActiveRun) return null;
|
|
397
|
+
if (Date.parse(localActiveRun.deadlineAt) + LOCK_BUFFER_MS < Date.now()) {
|
|
398
|
+
localActiveRun = null;
|
|
399
|
+
return null;
|
|
400
|
+
}
|
|
401
|
+
return localActiveRun;
|
|
402
|
+
}
|
|
403
|
+
async function acquireActiveRun(app, state) {
|
|
404
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
405
|
+
if (redis) {
|
|
406
|
+
const ttlMs = state.durationMs + LOCK_BUFFER_MS;
|
|
407
|
+
const result = await redis.sendCommand(["SET", ACTIVE_RUN_KEY, JSON.stringify(state), "NX", "PX", String(ttlMs)]);
|
|
408
|
+
if (result !== "OK") {
|
|
409
|
+
throw new ActiveDoctorRunError(await getActiveRunState(app));
|
|
410
|
+
}
|
|
411
|
+
return;
|
|
412
|
+
}
|
|
413
|
+
const activeRun = await getActiveRunState(app);
|
|
414
|
+
if (activeRun) {
|
|
415
|
+
throw new ActiveDoctorRunError(activeRun);
|
|
416
|
+
}
|
|
417
|
+
localActiveRun = state;
|
|
418
|
+
}
|
|
419
|
+
async function releaseActiveRun(app, runId) {
|
|
420
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
421
|
+
if (redis) {
|
|
422
|
+
const activeRun = await getActiveRunState(app);
|
|
423
|
+
if ((activeRun == null ? void 0 : activeRun.runId) === runId) {
|
|
424
|
+
await redis.sendCommand(["DEL", ACTIVE_RUN_KEY]);
|
|
425
|
+
}
|
|
426
|
+
return;
|
|
427
|
+
}
|
|
428
|
+
if ((localActiveRun == null ? void 0 : localActiveRun.runId) === runId) {
|
|
429
|
+
localActiveRun = null;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
async function acquireFinishLock(app, runId) {
|
|
433
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
434
|
+
if (!redis) return true;
|
|
435
|
+
const result = await redis.sendCommand([
|
|
436
|
+
"SET",
|
|
437
|
+
`${FINISH_LOCK_PREFIX}${runId}`,
|
|
438
|
+
process.pid.toString(),
|
|
439
|
+
"NX",
|
|
440
|
+
"EX",
|
|
441
|
+
String(FINISH_LOCK_TTL_SECONDS)
|
|
442
|
+
]);
|
|
443
|
+
return result === "OK";
|
|
444
|
+
}
|
|
445
|
+
async function releaseFinishLock(app, runId) {
|
|
446
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
447
|
+
if (!redis) return;
|
|
448
|
+
await redis.sendCommand(["DEL", `${FINISH_LOCK_PREFIX}${runId}`]);
|
|
449
|
+
}
|
|
450
|
+
function clearRunTimer(runId) {
|
|
451
|
+
const timer = timers.get(runId);
|
|
452
|
+
if (timer) {
|
|
453
|
+
clearTimeout(timer);
|
|
454
|
+
timers.delete(runId);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
function scheduleAutoFinish(app, runId, deadlineAt) {
|
|
458
|
+
clearRunTimer(runId);
|
|
459
|
+
const delayMs = Math.max(0, Date.parse(deadlineAt) - Date.now());
|
|
460
|
+
const timer = setTimeout(() => {
|
|
461
|
+
finishDoctorRun(app, runId, "timeout").catch((error) => {
|
|
462
|
+
getApp(app).logger.error(
|
|
463
|
+
`[ClusterDoctor] Failed to auto-finish diagnostic run ${runId}: ${getErrorMessage(error)}`
|
|
464
|
+
);
|
|
465
|
+
});
|
|
466
|
+
}, delayMs);
|
|
467
|
+
timers.set(runId, timer);
|
|
468
|
+
}
|
|
469
|
+
async function getRunById(app, runId) {
|
|
470
|
+
const repo = getRepository(app, "clusterManagerDoctorRuns");
|
|
471
|
+
return repo.findOne({ filter: { runId } });
|
|
472
|
+
}
|
|
473
|
+
async function getLatestRun(app) {
|
|
474
|
+
const repo = getRepository(app, "clusterManagerDoctorRuns");
|
|
475
|
+
const rows = await repo.find({ sort: ["-createdAt"], limit: 1 });
|
|
476
|
+
return rows[0] || null;
|
|
477
|
+
}
|
|
478
|
+
function runJsonToActiveState(run) {
|
|
479
|
+
if (!run.runId || !run.startedAt || !run.deadlineAt) return null;
|
|
480
|
+
return {
|
|
481
|
+
runId: String(run.runId),
|
|
482
|
+
startedAt: new Date(String(run.startedAt)).toISOString(),
|
|
483
|
+
deadlineAt: new Date(String(run.deadlineAt)).toISOString(),
|
|
484
|
+
durationMs: Number(run.durationMs || DEFAULT_DURATION_MS),
|
|
485
|
+
startedBy: typeof run.startedBy === "string" ? run.startedBy : void 0
|
|
486
|
+
};
|
|
487
|
+
}
|
|
488
|
+
async function getBlockingRunStateFromDb(app) {
|
|
489
|
+
const latestRun = modelToJSON(await getLatestRun(app));
|
|
490
|
+
if (!latestRun || latestRun.status !== "running") {
|
|
491
|
+
return null;
|
|
492
|
+
}
|
|
493
|
+
const activeState = runJsonToActiveState(latestRun);
|
|
494
|
+
if (!activeState) return null;
|
|
495
|
+
if (Date.parse(activeState.deadlineAt) + LOCK_BUFFER_MS <= Date.now()) {
|
|
496
|
+
return null;
|
|
497
|
+
}
|
|
498
|
+
return activeState;
|
|
499
|
+
}
|
|
500
|
+
async function updateRun(app, runId, values) {
|
|
501
|
+
const repo = getRepository(app, "clusterManagerDoctorRuns");
|
|
502
|
+
await repo.update({ filter: { runId }, values });
|
|
503
|
+
}
|
|
504
|
+
async function getClusterNodes(app) {
|
|
505
|
+
var _a;
|
|
506
|
+
const plugin = (_a = getApp(app).pm) == null ? void 0 : _a.get("plugin-cluster-manager");
|
|
507
|
+
const registry = (plugin == null ? void 0 : plugin.nodeRegistry) ?? new import_redis_node_registry.RedisNodeRegistry(app);
|
|
508
|
+
return registry.getNodes();
|
|
509
|
+
}
|
|
510
|
+
async function requestRemoteSnapshot(app, node, options) {
|
|
511
|
+
const targetNodeId = node.id;
|
|
512
|
+
if (!targetNodeId) {
|
|
513
|
+
throw new Error("Node does not have an id.");
|
|
514
|
+
}
|
|
515
|
+
if (targetNodeId === (0, import_node.getLocalNodeId)(app)) {
|
|
516
|
+
return collectLocalDoctorSnapshot(app, options);
|
|
517
|
+
}
|
|
518
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
519
|
+
const pubSub = getApp(app).pubSubManager;
|
|
520
|
+
if (!redis || !pubSub) {
|
|
521
|
+
throw new Error("Redis/PubSub is not available for remote diagnostic snapshot collection.");
|
|
522
|
+
}
|
|
523
|
+
const requestId = import_crypto.default.randomBytes(8).toString("hex");
|
|
524
|
+
const responseKey = `${RESPONSE_KEY_PREFIX}${requestId}`;
|
|
525
|
+
await pubSub.publish(
|
|
526
|
+
`cluster-manager:doctor-collect:${targetNodeId}`,
|
|
527
|
+
JSON.stringify({
|
|
528
|
+
requestId,
|
|
529
|
+
targetNodeId,
|
|
530
|
+
runId: options.runId,
|
|
531
|
+
sinceMs: options.sinceMs,
|
|
532
|
+
untilMs: options.untilMs,
|
|
533
|
+
maxLines: options.maxLines
|
|
534
|
+
})
|
|
535
|
+
);
|
|
536
|
+
for (let i = 0; i < 60; i++) {
|
|
537
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
538
|
+
const raw = await redis.sendCommand(["GET", responseKey]);
|
|
539
|
+
if (typeof raw === "string" && raw) {
|
|
540
|
+
await redis.sendCommand(["DEL", responseKey]);
|
|
541
|
+
const snapshot = parseJson(raw, {});
|
|
542
|
+
if (snapshot.node && snapshot.logs) {
|
|
543
|
+
return snapshot;
|
|
544
|
+
}
|
|
545
|
+
throw new Error(snapshot.error || "Invalid diagnostic snapshot response payload.");
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
throw new Error(`Timeout waiting for diagnostic snapshot from ${targetNodeId}.`);
|
|
549
|
+
}
|
|
550
|
+
async function collectNodeSnapshots(app, nodes, options) {
|
|
551
|
+
const targets = nodes.length > 0 ? nodes : [{ id: (0, import_node.getLocalNodeId)(app), hostname: import_os.default.hostname() }];
|
|
552
|
+
return Promise.all(
|
|
553
|
+
targets.map(async (node) => {
|
|
554
|
+
try {
|
|
555
|
+
return await requestRemoteSnapshot(app, node, options);
|
|
556
|
+
} catch (error) {
|
|
557
|
+
const workerMode = node.workerMode || "unknown";
|
|
558
|
+
return {
|
|
559
|
+
nodeId: node.id || `${node.hostname || "unknown"}:${node.pid || "unknown"}`,
|
|
560
|
+
node: {
|
|
561
|
+
hostname: node.hostname || "unknown",
|
|
562
|
+
pid: Number(node.pid || 0),
|
|
563
|
+
workerMode,
|
|
564
|
+
role: getNodeRole({ workerMode, isSandbox: node.isSandbox }),
|
|
565
|
+
appVersion: node.appVersion || "",
|
|
566
|
+
nodeVersion: "",
|
|
567
|
+
platform: "",
|
|
568
|
+
arch: "",
|
|
569
|
+
uptime: 0,
|
|
570
|
+
isSandbox: Boolean(node.isSandbox)
|
|
571
|
+
},
|
|
572
|
+
memory: process.memoryUsage(),
|
|
573
|
+
os: { totalMemory: 0, freeMemory: 0, cpuCount: 0, loadAvg: [] },
|
|
574
|
+
env: {},
|
|
575
|
+
plugins: [],
|
|
576
|
+
logs: { files: [], lines: [], analysis: analyzeLogLines([]) },
|
|
577
|
+
collectedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
578
|
+
error: getErrorMessage(error)
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
})
|
|
582
|
+
);
|
|
583
|
+
}
|
|
584
|
+
function aggregateTopSignatures(snapshots, containerDiagnostics) {
|
|
585
|
+
var _a, _b;
|
|
586
|
+
const signatures = /* @__PURE__ */ new Map();
|
|
587
|
+
const collect = (nodeId, items) => {
|
|
588
|
+
for (const item of items) {
|
|
589
|
+
const key = `${item.level}:${item.signature}`;
|
|
590
|
+
const existing = signatures.get(key);
|
|
591
|
+
if (existing) {
|
|
592
|
+
existing.count += item.count;
|
|
593
|
+
if (!existing.nodes.includes(nodeId)) {
|
|
594
|
+
existing.nodes.push(nodeId);
|
|
595
|
+
}
|
|
596
|
+
for (const source of item.sources) {
|
|
597
|
+
if (!existing.sources.includes(source)) existing.sources.push(source);
|
|
598
|
+
}
|
|
599
|
+
existing.samples.push(...item.samples.slice(0, Math.max(0, 3 - existing.samples.length)));
|
|
600
|
+
} else {
|
|
601
|
+
signatures.set(key, { ...item, nodes: [nodeId], samples: item.samples.slice(0, 3) });
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
};
|
|
605
|
+
snapshots.forEach((snapshot) => collect(snapshot.nodeId, snapshot.logs.analysis.signatures));
|
|
606
|
+
const containerStacks = Array.isArray(containerDiagnostics.stacks) ? containerDiagnostics.stacks : [];
|
|
607
|
+
for (const stack of containerStacks) {
|
|
608
|
+
for (const container of stack.containers || []) {
|
|
609
|
+
collect(`container:${container.id}`, ((_b = (_a = container.logs) == null ? void 0 : _a.analysis) == null ? void 0 : _b.signatures) || []);
|
|
610
|
+
}
|
|
611
|
+
}
|
|
612
|
+
return [...signatures.values()].sort((a, b) => b.count - a.count).slice(0, 30);
|
|
613
|
+
}
|
|
614
|
+
function buildVersionDiagnostics(nodes, snapshots) {
|
|
615
|
+
const nodeVersions = snapshots.map((snapshot) => ({
|
|
616
|
+
nodeId: snapshot.nodeId,
|
|
617
|
+
hostname: snapshot.node.hostname,
|
|
618
|
+
role: snapshot.node.role,
|
|
619
|
+
appVersion: snapshot.node.appVersion,
|
|
620
|
+
nodeVersion: snapshot.node.nodeVersion,
|
|
621
|
+
platform: snapshot.node.platform,
|
|
622
|
+
arch: snapshot.node.arch
|
|
623
|
+
}));
|
|
624
|
+
const appVersions = new Set(nodeVersions.map((node) => node.appVersion).filter(Boolean));
|
|
625
|
+
const runtimeVersions = new Set(
|
|
626
|
+
nodeVersions.map((node) => `${node.nodeVersion}:${node.platform}:${node.arch}`).filter(Boolean)
|
|
627
|
+
);
|
|
628
|
+
return {
|
|
629
|
+
registryNodes: nodes,
|
|
630
|
+
nodeVersions,
|
|
631
|
+
versionDrift: appVersions.size > 1,
|
|
632
|
+
runtimeDrift: runtimeVersions.size > 1
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
function buildPluginDiagnostics(pluginRows, snapshots) {
|
|
636
|
+
const plugins = pluginRows.map((plugin) => {
|
|
637
|
+
const name = typeof plugin.name === "string" ? plugin.name : "";
|
|
638
|
+
const packageName = typeof plugin.packageName === "string" ? plugin.packageName : "";
|
|
639
|
+
const dbVersion = typeof plugin.version === "string" ? plugin.version : void 0;
|
|
640
|
+
const nodeStates = snapshots.map((snapshot) => {
|
|
641
|
+
const nodePlugin = snapshot.plugins.find((item) => item.name === name || item.packageName === packageName);
|
|
642
|
+
return {
|
|
643
|
+
nodeId: snapshot.nodeId,
|
|
644
|
+
hostname: snapshot.node.hostname,
|
|
645
|
+
loaded: Boolean(nodePlugin == null ? void 0 : nodePlugin.loaded),
|
|
646
|
+
runtimeVersion: nodePlugin == null ? void 0 : nodePlugin.runtimeVersion
|
|
647
|
+
};
|
|
648
|
+
});
|
|
649
|
+
const runtimeVersions = Array.from(
|
|
650
|
+
new Set(nodeStates.filter((item) => item.loaded && item.runtimeVersion).map((item) => item.runtimeVersion))
|
|
651
|
+
);
|
|
652
|
+
const hasVersionMismatch = runtimeVersions.length > 1 || Boolean(dbVersion && runtimeVersions.some((version) => version && version !== dbVersion));
|
|
653
|
+
const loadedValues = new Set(nodeStates.map((item) => item.loaded));
|
|
654
|
+
return {
|
|
655
|
+
name,
|
|
656
|
+
packageName,
|
|
657
|
+
enabled: Boolean(plugin.enabled),
|
|
658
|
+
dbVersion,
|
|
659
|
+
runtimeVersions,
|
|
660
|
+
versionDrift: hasVersionMismatch,
|
|
661
|
+
loadDrift: Boolean(plugin.enabled) && loadedValues.size > 1,
|
|
662
|
+
nodes: nodeStates
|
|
663
|
+
};
|
|
664
|
+
});
|
|
665
|
+
return {
|
|
666
|
+
plugins,
|
|
667
|
+
versionDrifts: plugins.filter((plugin) => plugin.versionDrift),
|
|
668
|
+
loadDrifts: plugins.filter((plugin) => plugin.loadDrift)
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
async function getExpectedPackages(app) {
|
|
672
|
+
try {
|
|
673
|
+
const repo = getRepository(app, "workerPackagesConfigs");
|
|
674
|
+
const config = await repo.findOne();
|
|
675
|
+
if (!config) {
|
|
676
|
+
return normalizePackageMap((0, import_packages.packagesFromConfig)({}));
|
|
677
|
+
}
|
|
678
|
+
const configured = (0, import_packages.packagesFromConfig)({
|
|
679
|
+
aptPackages: getModelValue(config, "aptPackages"),
|
|
680
|
+
pythonPackages: getModelValue(config, "pythonPackages"),
|
|
681
|
+
npmPackages: getModelValue(config, "npmPackages")
|
|
682
|
+
});
|
|
683
|
+
const custom = parseCustomPackages(getModelValue(config, "customPackages"));
|
|
684
|
+
return normalizePackageMap({
|
|
685
|
+
apt: configured.apt,
|
|
686
|
+
npm: [...configured.npm || [], ...custom.node || [], ...custom.npm || []],
|
|
687
|
+
python: [...configured.python || [], ...custom.python || []]
|
|
688
|
+
});
|
|
689
|
+
} catch {
|
|
690
|
+
return { apt: [], npm: [], python: [] };
|
|
691
|
+
}
|
|
692
|
+
}
|
|
693
|
+
async function getPackageDiagnostics(app, nodes) {
|
|
694
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
695
|
+
const expectedPackages = await getExpectedPackages(app);
|
|
696
|
+
const nodePackages = [];
|
|
697
|
+
if (!redis) {
|
|
698
|
+
return {
|
|
699
|
+
available: false,
|
|
700
|
+
expectedPackages,
|
|
701
|
+
nodes: [],
|
|
702
|
+
packageDrifts: []
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
for (const node of nodes.filter((item) => getNodeRole(item) !== "app")) {
|
|
706
|
+
const keys = [
|
|
707
|
+
node.id ? `cluster-manager:pkg-status:${node.id}` : null,
|
|
708
|
+
node.hostname ? `orchestrator:pkg-status:${node.hostname}` : null,
|
|
709
|
+
node.name ? `orchestrator:pkg-status:${node.name}` : null
|
|
710
|
+
].filter(Boolean);
|
|
711
|
+
let status = null;
|
|
712
|
+
for (const key of keys) {
|
|
713
|
+
const raw = await redis.sendCommand(["GET", key]);
|
|
714
|
+
if (typeof raw === "string" && raw) {
|
|
715
|
+
status = parseJson(raw, null);
|
|
716
|
+
if (status) break;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
const installedPackages = parsePackageWhitelist(status == null ? void 0 : status.packageWhitelist);
|
|
720
|
+
const missingPackages = diffPackages(expectedPackages, installedPackages);
|
|
721
|
+
nodePackages.push({
|
|
722
|
+
nodeId: node.id,
|
|
723
|
+
hostname: node.hostname,
|
|
724
|
+
role: getNodeRole(node),
|
|
725
|
+
status: (status == null ? void 0 : status.initStatus) || "unknown",
|
|
726
|
+
lastInitAt: (status == null ? void 0 : status.lastInitAt) || null,
|
|
727
|
+
installedPackages,
|
|
728
|
+
missingPackages,
|
|
729
|
+
drift: !status || status.initStatus !== "succeeded" || countPackages(missingPackages) > 0
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
return {
|
|
733
|
+
available: true,
|
|
734
|
+
expectedPackages,
|
|
735
|
+
nodes: nodePackages,
|
|
736
|
+
packageDrifts: nodePackages.filter((item) => item.drift)
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
async function safeCount(app, collection, filter) {
|
|
740
|
+
try {
|
|
741
|
+
if (getApp(app).db.hasCollection && !getApp(app).db.hasCollection(collection)) {
|
|
742
|
+
return null;
|
|
743
|
+
}
|
|
744
|
+
const repo = getRepository(app, collection);
|
|
745
|
+
if (!repo.count) return null;
|
|
746
|
+
return await repo.count(filter ? { filter } : void 0);
|
|
747
|
+
} catch {
|
|
748
|
+
return null;
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
async function getDatabaseDiagnostics(app) {
|
|
752
|
+
var _a;
|
|
753
|
+
const startedAt = Date.now();
|
|
754
|
+
let ping;
|
|
755
|
+
try {
|
|
756
|
+
await ((_a = getApp(app).db.sequelize) == null ? void 0 : _a.query("SELECT 1"));
|
|
757
|
+
ping = { ok: true, latencyMs: Date.now() - startedAt };
|
|
758
|
+
} catch (error) {
|
|
759
|
+
ping = { ok: false, error: getErrorMessage(error) };
|
|
760
|
+
}
|
|
761
|
+
return {
|
|
762
|
+
ping,
|
|
763
|
+
asyncTasks: {
|
|
764
|
+
pending: await safeCount(app, "asyncTasks", { status: null }),
|
|
765
|
+
running: await safeCount(app, "asyncTasks", { status: 0 }),
|
|
766
|
+
failed: await safeCount(app, "asyncTasks", { status: -1 }),
|
|
767
|
+
canceled: await safeCount(app, "asyncTasks", { status: -2 })
|
|
768
|
+
},
|
|
769
|
+
workflowExecutions: {
|
|
770
|
+
queued: await safeCount(app, "executions", { status: null }),
|
|
771
|
+
running: await safeCount(app, "executions", { status: 0 }),
|
|
772
|
+
failed: await safeCount(app, "executions", { status: -1 }),
|
|
773
|
+
canceled: await safeCount(app, "executions", { status: -4 })
|
|
774
|
+
},
|
|
775
|
+
jobs: {
|
|
776
|
+
pending: await safeCount(app, "jobs", { status: 0 }),
|
|
777
|
+
failed: await safeCount(app, "jobs", { status: -1 }),
|
|
778
|
+
canceled: await safeCount(app, "jobs", { status: -4 })
|
|
779
|
+
},
|
|
780
|
+
applicationPlugins: {
|
|
781
|
+
total: await safeCount(app, "applicationPlugins"),
|
|
782
|
+
enabled: await safeCount(app, "applicationPlugins", { enabled: true }),
|
|
783
|
+
disabled: await safeCount(app, "applicationPlugins", { enabled: false })
|
|
784
|
+
}
|
|
785
|
+
};
|
|
786
|
+
}
|
|
787
|
+
function parseRedisInfo(raw) {
|
|
788
|
+
const sections = {};
|
|
789
|
+
let current = "default";
|
|
790
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
791
|
+
const trimmed = line.trim();
|
|
792
|
+
if (!trimmed) continue;
|
|
793
|
+
if (trimmed.startsWith("#")) {
|
|
794
|
+
current = trimmed.replace(/^#\s*/, "").toLowerCase();
|
|
795
|
+
sections[current] = sections[current] || {};
|
|
796
|
+
continue;
|
|
797
|
+
}
|
|
798
|
+
const idx = trimmed.indexOf(":");
|
|
799
|
+
if (idx > 0) {
|
|
800
|
+
sections[current] = sections[current] || {};
|
|
801
|
+
sections[current][trimmed.slice(0, idx)] = trimmed.slice(idx + 1);
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
return sections;
|
|
805
|
+
}
|
|
806
|
+
async function getRedisDiagnostics(app) {
|
|
807
|
+
const redis = (0, import_redis.getRedisClient)(app);
|
|
808
|
+
if (!redis) {
|
|
809
|
+
return { available: false };
|
|
810
|
+
}
|
|
811
|
+
try {
|
|
812
|
+
const startedAt = Date.now();
|
|
813
|
+
await redis.sendCommand(["PING"]);
|
|
814
|
+
const rawInfo = typeof redis.info === "function" ? await redis.info() : String(await redis.sendCommand(["INFO"]));
|
|
815
|
+
const info = parseRedisInfo(rawInfo);
|
|
816
|
+
const memory = info.memory || {};
|
|
817
|
+
const stats = info.stats || {};
|
|
818
|
+
const clients = info.clients || {};
|
|
819
|
+
const dbSize = Number(await redis.sendCommand(["DBSIZE"])) || 0;
|
|
820
|
+
const lockKeys = await (0, import_redis.scanKeys)(redis, "nocobase:lock:*", 200);
|
|
821
|
+
const rawSlowlog = await redis.sendCommand(["SLOWLOG", "GET", "10"]);
|
|
822
|
+
const slowlog = Array.isArray(rawSlowlog) ? rawSlowlog.map((entry) => {
|
|
823
|
+
const item = Array.isArray(entry) ? entry : [];
|
|
824
|
+
const command = Array.isArray(item[3]) ? item[3].join(" ") : String(item[3] || "");
|
|
825
|
+
return {
|
|
826
|
+
id: item[0],
|
|
827
|
+
timestamp: item[1],
|
|
828
|
+
durationUs: item[2],
|
|
829
|
+
command: redactText(command)
|
|
830
|
+
};
|
|
831
|
+
}) : [];
|
|
832
|
+
return {
|
|
833
|
+
available: true,
|
|
834
|
+
latencyMs: Date.now() - startedAt,
|
|
835
|
+
memory: {
|
|
836
|
+
used: memory.used_memory_human,
|
|
837
|
+
usedBytes: Number(memory.used_memory || 0),
|
|
838
|
+
peak: memory.used_memory_peak_human,
|
|
839
|
+
fragmentationRatio: Number(memory.mem_fragmentation_ratio || 0)
|
|
840
|
+
},
|
|
841
|
+
clients: {
|
|
842
|
+
connected: Number(clients.connected_clients || 0),
|
|
843
|
+
blocked: Number(clients.blocked_clients || 0)
|
|
844
|
+
},
|
|
845
|
+
stats: {
|
|
846
|
+
opsPerSec: Number(stats.instantaneous_ops_per_sec || 0),
|
|
847
|
+
evictedKeys: Number(stats.evicted_keys || 0),
|
|
848
|
+
expiredKeys: Number(stats.expired_keys || 0)
|
|
849
|
+
},
|
|
850
|
+
dbSize,
|
|
851
|
+
activeLocks: lockKeys.length,
|
|
852
|
+
slowlog
|
|
853
|
+
};
|
|
854
|
+
} catch (error) {
|
|
855
|
+
return { available: false, error: getErrorMessage(error) };
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
async function getQueueDiagnostics(app) {
|
|
859
|
+
var _a, _b, _c;
|
|
860
|
+
const eventQueue = getApp(app).eventQueue;
|
|
861
|
+
if (!eventQueue) {
|
|
862
|
+
return { available: false };
|
|
863
|
+
}
|
|
864
|
+
const adapter = eventQueue.adapter;
|
|
865
|
+
const channels = [];
|
|
866
|
+
for (const [channel, options] of eventQueue.events || /* @__PURE__ */ new Map()) {
|
|
867
|
+
let pending = null;
|
|
868
|
+
if ((adapter == null ? void 0 : adapter.queues) && eventQueue.getFullChannel) {
|
|
869
|
+
const fullChannel = eventQueue.getFullChannel(channel, options.shared);
|
|
870
|
+
pending = ((_a = adapter.queues.get(fullChannel)) == null ? void 0 : _a.length) || 0;
|
|
871
|
+
}
|
|
872
|
+
channels.push({
|
|
873
|
+
channel,
|
|
874
|
+
concurrency: options.concurrency || 1,
|
|
875
|
+
interval: options.interval || 250,
|
|
876
|
+
pending
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
return {
|
|
880
|
+
available: true,
|
|
881
|
+
connected: ((_b = eventQueue.isConnected) == null ? void 0 : _b.call(eventQueue)) || false,
|
|
882
|
+
adapter: ((_c = adapter == null ? void 0 : adapter.constructor) == null ? void 0 : _c.name) || "unknown",
|
|
883
|
+
channels,
|
|
884
|
+
totalPending: channels.reduce((sum, item) => sum + (item.pending || 0), 0)
|
|
885
|
+
};
|
|
886
|
+
}
|
|
887
|
+
async function getOrchestratorDiagnostics(app, options) {
|
|
888
|
+
var _a;
|
|
889
|
+
const plugin = (_a = getApp(app).pm) == null ? void 0 : _a.get("plugin-cluster-manager");
|
|
890
|
+
const adapter = plugin == null ? void 0 : plugin.orchestrator;
|
|
891
|
+
if (!adapter) {
|
|
892
|
+
return { configured: false, stacks: [] };
|
|
893
|
+
}
|
|
894
|
+
let stacks = [];
|
|
895
|
+
try {
|
|
896
|
+
const repo = getRepository(app, "orchestratorStacks");
|
|
897
|
+
const rows = await repo.find({ sort: ["name"], limit: 10 });
|
|
898
|
+
stacks = rows.map((row) => modelToJSON(row)).filter((stack) => (stack == null ? void 0 : stack.enabled) !== false);
|
|
899
|
+
} catch (error) {
|
|
900
|
+
return { configured: true, adapter: adapter.name, error: getErrorMessage(error), stacks: [] };
|
|
901
|
+
}
|
|
902
|
+
const results = [];
|
|
903
|
+
for (const stack of stacks) {
|
|
904
|
+
try {
|
|
905
|
+
const containers = await adapter.listContainers(stack);
|
|
906
|
+
const enriched = [];
|
|
907
|
+
for (const container of containers.slice(0, 10)) {
|
|
908
|
+
let stats = null;
|
|
909
|
+
let logs = null;
|
|
910
|
+
try {
|
|
911
|
+
stats = container.status === "running" ? await adapter.getStats(container.id) : null;
|
|
912
|
+
} catch (error) {
|
|
913
|
+
stats = { error: getErrorMessage(error) };
|
|
914
|
+
}
|
|
915
|
+
try {
|
|
916
|
+
const rawLogs = await adapter.getLogs(container.id, MAX_CONTAINER_LOG_LINES);
|
|
917
|
+
const parsed = rawLogs.split(/\r?\n/).filter((line) => line.trim()).map((line) => parseDiagnosticLine(`container:${container.name}`, line)).filter((line) => {
|
|
918
|
+
const timestamp = line.timestamp ? Date.parse(line.timestamp) : parseLogTimestamp(line.line);
|
|
919
|
+
if (!timestamp) return true;
|
|
920
|
+
return timestamp >= (options.sinceMs || 0) && timestamp <= (options.untilMs || Date.now());
|
|
921
|
+
});
|
|
922
|
+
logs = { lineCount: parsed.length, analysis: analyzeLogLines(parsed) };
|
|
923
|
+
} catch (error) {
|
|
924
|
+
logs = { lineCount: 0, analysis: analyzeLogLines([]), error: getErrorMessage(error) };
|
|
925
|
+
}
|
|
926
|
+
enriched.push({ ...container, stats, logs });
|
|
927
|
+
}
|
|
928
|
+
results.push({ stack: { id: stack.id, name: stack.name, adapter: stack.adapter }, containers: enriched });
|
|
929
|
+
} catch (error) {
|
|
930
|
+
results.push({
|
|
931
|
+
stack: { id: stack.id, name: stack.name, adapter: stack.adapter },
|
|
932
|
+
error: getErrorMessage(error)
|
|
933
|
+
});
|
|
934
|
+
}
|
|
935
|
+
}
|
|
936
|
+
return {
|
|
937
|
+
configured: true,
|
|
938
|
+
adapter: adapter.name,
|
|
939
|
+
stacks: results
|
|
940
|
+
};
|
|
941
|
+
}
|
|
942
|
+
function buildRecommendations(params) {
|
|
943
|
+
const recommendations = [];
|
|
944
|
+
if (!params.redisAvailable) {
|
|
945
|
+
recommendations.push({
|
|
946
|
+
level: "critical",
|
|
947
|
+
code: "redis_unavailable",
|
|
948
|
+
message: "Redis is unavailable, so cluster-wide diagnostic collection and locking are degraded."
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
if (!params.databaseOk) {
|
|
952
|
+
recommendations.push({
|
|
953
|
+
level: "critical",
|
|
954
|
+
code: "database_unhealthy",
|
|
955
|
+
message: "Database ping failed during the diagnostic session."
|
|
956
|
+
});
|
|
957
|
+
}
|
|
958
|
+
if (params.snapshotErrors > 0) {
|
|
959
|
+
recommendations.push({
|
|
960
|
+
level: "warning",
|
|
961
|
+
code: "snapshot_collection_failed",
|
|
962
|
+
message: `${params.snapshotErrors} node(s) did not return a diagnostic snapshot.`
|
|
963
|
+
});
|
|
964
|
+
}
|
|
965
|
+
if (params.versionDrift || params.runtimeDrift) {
|
|
966
|
+
recommendations.push({
|
|
967
|
+
level: "warning",
|
|
968
|
+
code: "cluster_runtime_drift",
|
|
969
|
+
message: "Cluster nodes are not running the same application/runtime version."
|
|
970
|
+
});
|
|
971
|
+
}
|
|
972
|
+
if (params.pluginVersionDrifts > 0 || params.pluginLoadDrifts > 0) {
|
|
973
|
+
recommendations.push({
|
|
974
|
+
level: "warning",
|
|
975
|
+
code: "plugin_drift",
|
|
976
|
+
message: "Installed or loaded plugin state is inconsistent across diagnostic snapshots."
|
|
977
|
+
});
|
|
978
|
+
}
|
|
979
|
+
if (params.packageDrifts > 0) {
|
|
980
|
+
recommendations.push({
|
|
981
|
+
level: "warning",
|
|
982
|
+
code: "package_drift",
|
|
983
|
+
message: "One or more worker nodes are missing configured packages or have failed package initialization."
|
|
984
|
+
});
|
|
985
|
+
}
|
|
986
|
+
if (params.topErrors > 0) {
|
|
987
|
+
recommendations.push({
|
|
988
|
+
level: "warning",
|
|
989
|
+
code: "log_errors_detected",
|
|
990
|
+
message: "Error or warning signatures were found in node/container logs during the diagnostic window."
|
|
991
|
+
});
|
|
992
|
+
}
|
|
993
|
+
return recommendations;
|
|
994
|
+
}
|
|
995
|
+
async function buildDoctorReport(app, run, finishReason) {
|
|
996
|
+
const runId = String(run.runId);
|
|
997
|
+
const startedAt = new Date(String(run.startedAt));
|
|
998
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
999
|
+
const sinceMs = startedAt.getTime();
|
|
1000
|
+
const untilMs = finishedAt.getTime();
|
|
1001
|
+
const nodes = await getClusterNodes(app);
|
|
1002
|
+
const snapshots = await collectNodeSnapshots(app, nodes, {
|
|
1003
|
+
runId,
|
|
1004
|
+
sinceMs,
|
|
1005
|
+
untilMs,
|
|
1006
|
+
maxLines: MAX_NODE_LOG_LINES
|
|
1007
|
+
});
|
|
1008
|
+
const pluginRows = await getApplicationPluginRows(app);
|
|
1009
|
+
const versionDiagnostics = buildVersionDiagnostics(nodes, snapshots);
|
|
1010
|
+
const pluginDiagnostics = buildPluginDiagnostics(pluginRows, snapshots);
|
|
1011
|
+
const packageDiagnostics = await getPackageDiagnostics(app, nodes);
|
|
1012
|
+
const databaseDiagnostics = await getDatabaseDiagnostics(app);
|
|
1013
|
+
const redisDiagnostics = await getRedisDiagnostics(app);
|
|
1014
|
+
const queueDiagnostics = await getQueueDiagnostics(app);
|
|
1015
|
+
const orchestratorDiagnostics = await getOrchestratorDiagnostics(app, { sinceMs, untilMs });
|
|
1016
|
+
const topSignatures = aggregateTopSignatures(snapshots, orchestratorDiagnostics);
|
|
1017
|
+
const snapshotErrors = snapshots.filter((snapshot) => snapshot.error).length;
|
|
1018
|
+
const recommendations = buildRecommendations({
|
|
1019
|
+
snapshotErrors,
|
|
1020
|
+
topErrors: topSignatures.length,
|
|
1021
|
+
versionDrift: Boolean(versionDiagnostics.versionDrift),
|
|
1022
|
+
runtimeDrift: Boolean(versionDiagnostics.runtimeDrift),
|
|
1023
|
+
pluginVersionDrifts: pluginDiagnostics.versionDrifts.length,
|
|
1024
|
+
pluginLoadDrifts: pluginDiagnostics.loadDrifts.length,
|
|
1025
|
+
packageDrifts: packageDiagnostics.packageDrifts.length,
|
|
1026
|
+
redisAvailable: Boolean(redisDiagnostics.available),
|
|
1027
|
+
databaseOk: Boolean(databaseDiagnostics.ping.ok)
|
|
1028
|
+
});
|
|
1029
|
+
const criticalFindings = recommendations.filter((item) => item.level === "critical").length;
|
|
1030
|
+
const warningFindings = recommendations.filter((item) => item.level === "warning").length;
|
|
1031
|
+
const healthStatus = criticalFindings > 0 ? "critical" : warningFindings > 0 ? "warning" : "healthy";
|
|
1032
|
+
const summary = {
|
|
1033
|
+
status: healthStatus,
|
|
1034
|
+
nodes: snapshots.length,
|
|
1035
|
+
snapshotErrors,
|
|
1036
|
+
errors: topSignatures.filter((item) => item.level === "error").reduce((sum, item) => sum + item.count, 0),
|
|
1037
|
+
warnings: topSignatures.filter((item) => item.level === "warn").reduce((sum, item) => sum + item.count, 0),
|
|
1038
|
+
versionDrift: versionDiagnostics.versionDrift,
|
|
1039
|
+
runtimeDrift: versionDiagnostics.runtimeDrift,
|
|
1040
|
+
pluginVersionDrifts: pluginDiagnostics.versionDrifts.length,
|
|
1041
|
+
pluginLoadDrifts: pluginDiagnostics.loadDrifts.length,
|
|
1042
|
+
packageDrifts: packageDiagnostics.packageDrifts.length,
|
|
1043
|
+
failedTasks: databaseDiagnostics.asyncTasks.failed,
|
|
1044
|
+
failedWorkflows: databaseDiagnostics.workflowExecutions.failed
|
|
1045
|
+
};
|
|
1046
|
+
return {
|
|
1047
|
+
runId,
|
|
1048
|
+
startedAt: startedAt.toISOString(),
|
|
1049
|
+
finishedAt: finishedAt.toISOString(),
|
|
1050
|
+
durationMs: untilMs - sinceMs,
|
|
1051
|
+
requestedDurationMs: Number(run.durationMs || 0),
|
|
1052
|
+
finishReason,
|
|
1053
|
+
summary,
|
|
1054
|
+
nodes: snapshots,
|
|
1055
|
+
versionDiagnostics,
|
|
1056
|
+
pluginDiagnostics,
|
|
1057
|
+
packageDiagnostics,
|
|
1058
|
+
databaseDiagnostics,
|
|
1059
|
+
redisDiagnostics,
|
|
1060
|
+
queueDiagnostics,
|
|
1061
|
+
orchestratorDiagnostics,
|
|
1062
|
+
logAnalysis: {
|
|
1063
|
+
topSignatures,
|
|
1064
|
+
byNode: snapshots.map((snapshot) => ({
|
|
1065
|
+
nodeId: snapshot.nodeId,
|
|
1066
|
+
hostname: snapshot.node.hostname,
|
|
1067
|
+
role: snapshot.node.role,
|
|
1068
|
+
files: snapshot.logs.files,
|
|
1069
|
+
levels: snapshot.logs.analysis.levels,
|
|
1070
|
+
signatures: snapshot.logs.analysis.signatures.slice(0, 10),
|
|
1071
|
+
error: snapshot.error
|
|
1072
|
+
}))
|
|
1073
|
+
},
|
|
1074
|
+
recommendations
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
async function finishDoctorRun(app, runId, reason) {
|
|
1078
|
+
const hasFinishLock = await acquireFinishLock(app, runId);
|
|
1079
|
+
if (!hasFinishLock) {
|
|
1080
|
+
return modelToJSON(await getRunById(app, runId));
|
|
1081
|
+
}
|
|
1082
|
+
clearRunTimer(runId);
|
|
1083
|
+
try {
|
|
1084
|
+
const run = await getRunById(app, runId);
|
|
1085
|
+
const runJson = modelToJSON(run);
|
|
1086
|
+
if (!run || !runJson) {
|
|
1087
|
+
throw new Error(`Diagnostic run ${runId} was not found.`);
|
|
1088
|
+
}
|
|
1089
|
+
if (runJson.status !== "running") {
|
|
1090
|
+
await releaseActiveRun(app, runId);
|
|
1091
|
+
return runJson;
|
|
1092
|
+
}
|
|
1093
|
+
await updateRun(app, runId, { progress: 40 });
|
|
1094
|
+
const report = await buildDoctorReport(app, runJson, reason);
|
|
1095
|
+
await updateRun(app, runId, {
|
|
1096
|
+
status: "finished",
|
|
1097
|
+
progress: 100,
|
|
1098
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1099
|
+
finishReason: reason,
|
|
1100
|
+
summary: report.summary,
|
|
1101
|
+
report,
|
|
1102
|
+
error: null
|
|
1103
|
+
});
|
|
1104
|
+
await releaseActiveRun(app, runId);
|
|
1105
|
+
return modelToJSON(await getRunById(app, runId));
|
|
1106
|
+
} catch (error) {
|
|
1107
|
+
await updateRun(app, runId, {
|
|
1108
|
+
status: "failed",
|
|
1109
|
+
progress: 100,
|
|
1110
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
1111
|
+
finishReason: reason,
|
|
1112
|
+
error: getErrorMessage(error)
|
|
1113
|
+
});
|
|
1114
|
+
await releaseActiveRun(app, runId);
|
|
1115
|
+
throw error;
|
|
1116
|
+
} finally {
|
|
1117
|
+
await releaseFinishLock(app, runId);
|
|
1118
|
+
}
|
|
1119
|
+
}
|
|
1120
|
+
async function finishExpiredActiveRun(app) {
|
|
1121
|
+
const activeRun = await getActiveRunState(app);
|
|
1122
|
+
if (activeRun && Date.parse(activeRun.deadlineAt) <= Date.now()) {
|
|
1123
|
+
await finishDoctorRun(app, activeRun.runId, "timeout");
|
|
1124
|
+
return;
|
|
1125
|
+
}
|
|
1126
|
+
const latestRun = modelToJSON(await getLatestRun(app));
|
|
1127
|
+
if ((latestRun == null ? void 0 : latestRun.status) === "running" && latestRun.deadlineAt && Date.parse(String(latestRun.deadlineAt)) <= Date.now()) {
|
|
1128
|
+
await finishDoctorRun(app, String(latestRun.runId), "timeout");
|
|
1129
|
+
}
|
|
1130
|
+
}
|
|
1131
|
+
function sanitizeRunForResponse(run, includeReport = false) {
|
|
1132
|
+
if (!run) return null;
|
|
1133
|
+
if (includeReport) return run;
|
|
1134
|
+
const { report, ...rest } = run;
|
|
1135
|
+
return {
|
|
1136
|
+
...rest,
|
|
1137
|
+
hasReport: Boolean(report)
|
|
1138
|
+
};
|
|
1139
|
+
}
|
|
1140
|
+
const doctorActions = {
|
|
1141
|
+
async start(ctx, next) {
|
|
1142
|
+
const payload = getPayload(ctx);
|
|
1143
|
+
const durationMs = clampDuration(payload.durationMs);
|
|
1144
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
1145
|
+
const deadlineAt = new Date(startedAt.getTime() + durationMs);
|
|
1146
|
+
const runId = import_crypto.default.randomBytes(8).toString("hex");
|
|
1147
|
+
const activeState = {
|
|
1148
|
+
runId,
|
|
1149
|
+
startedAt: startedAt.toISOString(),
|
|
1150
|
+
deadlineAt: deadlineAt.toISOString(),
|
|
1151
|
+
durationMs,
|
|
1152
|
+
startedBy: getUserLabel(ctx)
|
|
1153
|
+
};
|
|
1154
|
+
try {
|
|
1155
|
+
await finishExpiredActiveRun(ctx.app);
|
|
1156
|
+
const blockingRun = await getBlockingRunStateFromDb(ctx.app);
|
|
1157
|
+
if (blockingRun) {
|
|
1158
|
+
throw new ActiveDoctorRunError(blockingRun);
|
|
1159
|
+
}
|
|
1160
|
+
await acquireActiveRun(ctx.app, activeState);
|
|
1161
|
+
} catch (error) {
|
|
1162
|
+
if (error instanceof ActiveDoctorRunError) {
|
|
1163
|
+
ctx.throw(409, "A diagnostic session is already running.", {
|
|
1164
|
+
activeRun: error.activeRun
|
|
1165
|
+
});
|
|
1166
|
+
}
|
|
1167
|
+
throw error;
|
|
1168
|
+
}
|
|
1169
|
+
try {
|
|
1170
|
+
const repo = getRepository(ctx.app, "clusterManagerDoctorRuns");
|
|
1171
|
+
await repo.create({
|
|
1172
|
+
values: {
|
|
1173
|
+
runId,
|
|
1174
|
+
status: "running",
|
|
1175
|
+
durationMs,
|
|
1176
|
+
progress: 5,
|
|
1177
|
+
startedAt,
|
|
1178
|
+
deadlineAt,
|
|
1179
|
+
startedBy: activeState.startedBy
|
|
1180
|
+
}
|
|
1181
|
+
});
|
|
1182
|
+
scheduleAutoFinish(ctx.app, runId, deadlineAt.toISOString());
|
|
1183
|
+
getApp(ctx.app).logger.info(`[ClusterDoctor] Diagnostic run ${runId} started by ${activeState.startedBy}`);
|
|
1184
|
+
ctx.body = {
|
|
1185
|
+
success: true,
|
|
1186
|
+
runId,
|
|
1187
|
+
status: "running",
|
|
1188
|
+
startedAt: activeState.startedAt,
|
|
1189
|
+
deadlineAt: activeState.deadlineAt,
|
|
1190
|
+
durationMs
|
|
1191
|
+
};
|
|
1192
|
+
} catch (error) {
|
|
1193
|
+
await releaseActiveRun(ctx.app, runId);
|
|
1194
|
+
throw error;
|
|
1195
|
+
}
|
|
1196
|
+
await next();
|
|
1197
|
+
},
|
|
1198
|
+
async stop(ctx, next) {
|
|
1199
|
+
const payload = getPayload(ctx);
|
|
1200
|
+
await finishExpiredActiveRun(ctx.app);
|
|
1201
|
+
const activeRun = await getActiveRunState(ctx.app);
|
|
1202
|
+
const runId = String(payload.runId || (activeRun == null ? void 0 : activeRun.runId) || ctx.action.params.runId || "");
|
|
1203
|
+
if (!runId) {
|
|
1204
|
+
ctx.throw(404, "No active diagnostic session was found.");
|
|
1205
|
+
}
|
|
1206
|
+
const run = await finishDoctorRun(ctx.app, runId, "manual");
|
|
1207
|
+
ctx.body = sanitizeRunForResponse(run, true);
|
|
1208
|
+
await next();
|
|
1209
|
+
},
|
|
1210
|
+
async status(ctx, next) {
|
|
1211
|
+
await finishExpiredActiveRun(ctx.app);
|
|
1212
|
+
const activeRun = await getActiveRunState(ctx.app);
|
|
1213
|
+
const runId = String(ctx.action.params.runId || (activeRun == null ? void 0 : activeRun.runId) || "");
|
|
1214
|
+
const run = runId ? await getRunById(ctx.app, runId) : await getLatestRun(ctx.app);
|
|
1215
|
+
ctx.body = {
|
|
1216
|
+
activeRun,
|
|
1217
|
+
run: sanitizeRunForResponse(modelToJSON(run), false)
|
|
1218
|
+
};
|
|
1219
|
+
await next();
|
|
1220
|
+
},
|
|
1221
|
+
async report(ctx, next) {
|
|
1222
|
+
await finishExpiredActiveRun(ctx.app);
|
|
1223
|
+
const runId = String(ctx.action.params.runId || "");
|
|
1224
|
+
const run = runId ? await getRunById(ctx.app, runId) : await getLatestRun(ctx.app);
|
|
1225
|
+
const runJson = modelToJSON(run);
|
|
1226
|
+
if (!runJson) {
|
|
1227
|
+
ctx.throw(404, "Diagnostic report not found.");
|
|
1228
|
+
}
|
|
1229
|
+
ctx.body = sanitizeRunForResponse(runJson, true);
|
|
1230
|
+
await next();
|
|
1231
|
+
},
|
|
1232
|
+
async download(ctx, next) {
|
|
1233
|
+
await finishExpiredActiveRun(ctx.app);
|
|
1234
|
+
const runId = String(ctx.action.params.runId || "");
|
|
1235
|
+
const run = runId ? await getRunById(ctx.app, runId) : await getLatestRun(ctx.app);
|
|
1236
|
+
const runJson = modelToJSON(run);
|
|
1237
|
+
if (!(runJson == null ? void 0 : runJson.report)) {
|
|
1238
|
+
ctx.throw(404, "Diagnostic report not found.");
|
|
1239
|
+
}
|
|
1240
|
+
ctx.attachment(`doctor-report-${runJson.runId}.json`);
|
|
1241
|
+
ctx.set("Content-Type", "application/json; charset=utf-8");
|
|
1242
|
+
ctx.body = JSON.stringify(runJson.report, null, 2);
|
|
1243
|
+
await next();
|
|
1244
|
+
}
|
|
1245
|
+
};
|
|
1246
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
1247
|
+
0 && (module.exports = {
|
|
1248
|
+
collectLocalDoctorSnapshot,
|
|
1249
|
+
doctorActions
|
|
1250
|
+
});
|