plugin-cluster-manager 1.1.16 → 1.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/376.cd1d86e85a50088e.js +10 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +6 -6
- package/dist/locale/en-US.json +16 -6
- package/dist/locale/vi-VN.json +16 -6
- package/dist/locale/zh-CN.json +16 -6
- package/dist/server/actions/cluster-nodes.js +43 -10
- package/dist/server/actions/doctor.js +69 -4
- package/dist/server/actions/event-queue-monitor.js +33 -3
- package/dist/server/actions/orchestrator.js +48 -32
- package/dist/server/actions/queue-mappings.js +1 -0
- package/dist/server/actions/tasks.js +8 -8
- package/dist/server/adapters/redis-event-queue-adapter.js +188 -0
- package/dist/server/adapters/redis-node-registry.js +42 -3
- package/dist/server/collections/orchestrator-stacks.js +6 -0
- package/dist/server/collections/worker-queue-mappings.js +1 -1
- package/dist/server/orchestrator/PackageManager.js +47 -12
- package/dist/server/plugin.js +36 -5
- package/dist/server/queue-scanner.js +54 -34
- package/dist/server/utils/node.js +3 -7
- package/dist/server/utils/redis.js +37 -9
- package/dist/shared/worker-processes.js +233 -0
- package/package.json +1 -1
- package/src/client/ClusterNodes.tsx +76 -10
- package/src/client/ContainerOrchestrator.tsx +146 -8
- package/src/client/QueueAssignment.tsx +10 -2
- package/src/locale/en-US.json +16 -6
- package/src/locale/vi-VN.json +16 -6
- package/src/locale/zh-CN.json +16 -6
- package/src/server/__tests__/worker-processes.test.ts +42 -0
- package/src/server/actions/cluster-nodes.ts +43 -8
- package/src/server/actions/doctor.ts +77 -0
- package/src/server/actions/event-queue-monitor.ts +34 -3
- package/src/server/actions/orchestrator.ts +58 -38
- package/src/server/actions/queue-mappings.ts +1 -0
- package/src/server/actions/tasks.ts +142 -142
- package/src/server/adapters/redis-event-queue-adapter.ts +189 -0
- package/src/server/adapters/redis-node-registry.ts +44 -4
- package/src/server/collections/orchestrator-stacks.ts +6 -0
- package/src/server/collections/worker-queue-mappings.ts +3 -3
- package/src/server/orchestrator/PackageManager.ts +48 -11
- package/src/server/orchestrator/types.ts +5 -4
- package/src/server/plugin.ts +40 -6
- package/src/server/queue-scanner.ts +65 -51
- package/src/server/utils/node.ts +3 -10
- package/src/server/utils/redis.ts +39 -4
- package/src/shared/worker-processes.ts +216 -0
- package/dist/client-v2/914.c0bce51908fd81d7.js +0 -10
package/dist/locale/vi-VN.json
CHANGED
|
@@ -157,10 +157,16 @@
|
|
|
157
157
|
"Deprecated multi-app share collection is active. Avoid schema/table sharing for new cluster deployments.": "Deprecated multi-app share collection is active. Avoid schema/table sharing for new cluster deployments.",
|
|
158
158
|
"{count} legacy application record(s) were found in the applications collection.": "{count} legacy application record(s) were found in the applications collection.",
|
|
159
159
|
"App Supervisor is not enabled. Use it for new multi-application management instead of deprecated multi-app plugins.": "App Supervisor is not enabled. Use it for new multi-application management instead of deprecated multi-app plugins.",
|
|
160
|
-
"Queue Assignment": "Gán hàng đợi",
|
|
161
|
-
"Queue Name": "Tên hàng đợi",
|
|
162
|
-
"Map queues to worker stacks. Unassigned queues run on all workers.": "Gán hàng đợi vào worker stack. Hàng đợi chưa gán sẽ chạy trên tất cả worker.",
|
|
163
|
-
"
|
|
160
|
+
"Queue Assignment": "Gán hàng đợi",
|
|
161
|
+
"Queue Name": "Tên hàng đợi",
|
|
162
|
+
"Map queues to worker stacks. Unassigned queues run on all workers.": "Gán hàng đợi vào worker stack. Hàng đợi chưa gán sẽ chạy trên tất cả worker.",
|
|
163
|
+
"Fallback mapping for legacy stacks without explicit Processes / queues.": "Mapping dự phòng cho các stack cũ chưa cấu hình Processes / queues rõ ràng.",
|
|
164
|
+
"Processes / queues": "Processes / queues",
|
|
165
|
+
"Saved as WORKER_MODE. Use tags for custom process keys that are not discovered yet.": "Được lưu thành WORKER_MODE. Có thể nhập tag cho process key tùy chỉnh chưa được phát hiện.",
|
|
166
|
+
"Select at least one process or queue": "Chọn ít nhất một process hoặc queue",
|
|
167
|
+
"WORKER_MODE=* makes this stack consume every queue. Prefer explicit queues in HA.": "WORKER_MODE=* khiến stack này tiêu thụ mọi queue. Nên chọn queue rõ ràng trong HA.",
|
|
168
|
+
"WORKER_MODE is managed by Processes / queues above.": "WORKER_MODE được quản lý bởi Processes / queues ở trên.",
|
|
169
|
+
"Scan Queues": "Quét hàng đợi",
|
|
164
170
|
"Auto-map ({count})": "Tự động gán ({count})",
|
|
165
171
|
"Register": "Đăng ký",
|
|
166
172
|
"Unregister": "Hủy đăng ký",
|
|
@@ -202,5 +208,9 @@
|
|
|
202
208
|
"Log Files": "Log Files",
|
|
203
209
|
"Package": "Package",
|
|
204
210
|
"DB Version": "DB Version",
|
|
205
|
-
"Runtime Versions": "Runtime Versions"
|
|
206
|
-
|
|
211
|
+
"Runtime Versions": "Runtime Versions",
|
|
212
|
+
"Cluster registry has no worker heartbeats": "Registry cụm chưa có heartbeat từ worker",
|
|
213
|
+
"Cluster registry Redis is not configured": "Chưa cấu hình Redis cho registry cụm",
|
|
214
|
+
"Cluster Nodes reads Redis heartbeats, not the container runtime. Check worker boot, plugin-cluster-manager, and shared Redis configuration.": "Cluster Nodes đọc heartbeat trong Redis, không đọc trực tiếp container runtime. Hãy kiểm tra worker đã boot, plugin-cluster-manager đã load, và cấu hình Redis dùng chung.",
|
|
215
|
+
"Set REDIS_URL or CLUSTER_MANAGER_REDIS_URL on every app and worker to enable cluster node discovery.": "Thiết lập REDIS_URL hoặc CLUSTER_MANAGER_REDIS_URL trên mọi app và worker để bật khám phá node trong cụm."
|
|
216
|
+
}
|
package/dist/locale/zh-CN.json
CHANGED
|
@@ -190,10 +190,16 @@
|
|
|
190
190
|
"Package": "Package",
|
|
191
191
|
"DB Version": "DB Version",
|
|
192
192
|
"Runtime Versions": "Runtime Versions",
|
|
193
|
-
"Queue Assignment": "队列分配",
|
|
194
|
-
"Queue Name": "队列名称",
|
|
195
|
-
"Map queues to worker stacks. Unassigned queues run on all workers.": "将队列映射到工作节点栈。未分配的队列将在所有工作节点上运行。",
|
|
196
|
-
"
|
|
193
|
+
"Queue Assignment": "队列分配",
|
|
194
|
+
"Queue Name": "队列名称",
|
|
195
|
+
"Map queues to worker stacks. Unassigned queues run on all workers.": "将队列映射到工作节点栈。未分配的队列将在所有工作节点上运行。",
|
|
196
|
+
"Fallback mapping for legacy stacks without explicit Processes / queues.": "用于没有显式进程/队列配置的旧工作栈的兜底映射。",
|
|
197
|
+
"Processes / queues": "进程 / 队列",
|
|
198
|
+
"Saved as WORKER_MODE. Use tags for custom process keys that are not discovered yet.": "保存为 WORKER_MODE。可用标签输入尚未发现的自定义进程键。",
|
|
199
|
+
"Select at least one process or queue": "请至少选择一个进程或队列",
|
|
200
|
+
"WORKER_MODE=* makes this stack consume every queue. Prefer explicit queues in HA.": "WORKER_MODE=* 会让此工作栈消费所有队列。HA 环境建议显式选择队列。",
|
|
201
|
+
"WORKER_MODE is managed by Processes / queues above.": "WORKER_MODE 由上方的进程 / 队列配置管理。",
|
|
202
|
+
"Scan Queues": "扫描队列",
|
|
197
203
|
"Auto-map ({count})": "自动映射 ({count})",
|
|
198
204
|
"Register": "注册",
|
|
199
205
|
"Unregister": "注销",
|
|
@@ -203,5 +209,9 @@
|
|
|
203
209
|
"No queues discovered. Click \"Scan Queues\" to detect registered queues.": "未发现队列。点击\"扫描队列\"以检测已注册的队列。",
|
|
204
210
|
"Auto-mapped {count} queue(s)": "已自动映射 {count} 个队列",
|
|
205
211
|
"Queue Assignment updated": "队列分配已更新",
|
|
206
|
-
"Unassigned (worker runs all queues)": "未分配(工作节点运行所有队列)"
|
|
207
|
-
|
|
212
|
+
"Unassigned (worker runs all queues)": "未分配(工作节点运行所有队列)",
|
|
213
|
+
"Cluster registry has no worker heartbeats": "集群注册表没有 worker 心跳",
|
|
214
|
+
"Cluster registry Redis is not configured": "未配置集群注册表 Redis",
|
|
215
|
+
"Cluster Nodes reads Redis heartbeats, not the container runtime. Check worker boot, plugin-cluster-manager, and shared Redis configuration.": "Cluster Nodes 读取 Redis 心跳,而不是直接读取容器运行时。请检查 worker 启动、plugin-cluster-manager 加载以及共享 Redis 配置。",
|
|
216
|
+
"Set REDIS_URL or CLUSTER_MANAGER_REDIS_URL on every app and worker to enable cluster node discovery.": "请在每个 app 和 worker 上设置 REDIS_URL 或 CLUSTER_MANAGER_REDIS_URL,以启用集群节点发现。"
|
|
217
|
+
}
|
|
@@ -145,11 +145,13 @@ function getReferenceVersion(nodes) {
|
|
|
145
145
|
}
|
|
146
146
|
return ((_a = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]) == null ? void 0 : _a[0]) || null;
|
|
147
147
|
}
|
|
148
|
-
|
|
148
|
+
function getClusterRegistry(ctx) {
|
|
149
149
|
var _a, _b;
|
|
150
150
|
const plugin = (_b = (_a = ctx.app.pm) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, "plugin-cluster-manager");
|
|
151
|
-
|
|
152
|
-
|
|
151
|
+
return (plugin == null ? void 0 : plugin.nodeRegistry) ?? new import_redis_node_registry.RedisNodeRegistry(ctx.app);
|
|
152
|
+
}
|
|
153
|
+
async function getClusterNodes(ctx) {
|
|
154
|
+
return getClusterRegistry(ctx).getNodes();
|
|
153
155
|
}
|
|
154
156
|
async function getExpectedPackages(ctx) {
|
|
155
157
|
var _a;
|
|
@@ -212,7 +214,9 @@ async function readLocalLogs(app, maxLines) {
|
|
|
212
214
|
const nodeInfo = {
|
|
213
215
|
hostname: import_os.default.hostname(),
|
|
214
216
|
pid: process.pid,
|
|
215
|
-
workerMode: process.env.WORKER_MODE || "main"
|
|
217
|
+
workerMode: process.env.WORKER_MODE || "main",
|
|
218
|
+
appRole: process.env.APP_ROLE || "",
|
|
219
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true"
|
|
216
220
|
};
|
|
217
221
|
if (logFiles.length === 0) {
|
|
218
222
|
return { node: nodeInfo, lines: [], file: null };
|
|
@@ -242,7 +246,11 @@ const clusterActions = {
|
|
|
242
246
|
async current(ctx, next) {
|
|
243
247
|
var _a, _b;
|
|
244
248
|
const currentMode = process.env.WORKER_MODE || "main";
|
|
245
|
-
const isApp =
|
|
249
|
+
const isApp = (0, import_node.getNodeRoleFrom)({
|
|
250
|
+
workerMode: process.env.WORKER_MODE,
|
|
251
|
+
appRole: process.env.APP_ROLE,
|
|
252
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true"
|
|
253
|
+
}) === "app";
|
|
246
254
|
if (isApp) {
|
|
247
255
|
const mem = process.memoryUsage();
|
|
248
256
|
ctx.body = {
|
|
@@ -254,6 +262,8 @@ const clusterActions = {
|
|
|
254
262
|
arch: process.arch,
|
|
255
263
|
uptime: process.uptime(),
|
|
256
264
|
workerMode: currentMode,
|
|
265
|
+
appRole: process.env.APP_ROLE || "",
|
|
266
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true",
|
|
257
267
|
appPort: process.env.APP_PORT || "",
|
|
258
268
|
clusterMode: process.env.CLUSTER_MODE || ""
|
|
259
269
|
},
|
|
@@ -275,7 +285,9 @@ const clusterActions = {
|
|
|
275
285
|
const plugin = (_b = (_a = ctx.app.pm) == null ? void 0 : _a.get) == null ? void 0 : _b.call(_a, "plugin-cluster-manager");
|
|
276
286
|
const registry = (plugin == null ? void 0 : plugin.nodeRegistry) ?? new import_redis_node_registry.RedisNodeRegistry(ctx.app);
|
|
277
287
|
const nodes = await registry.getNodes();
|
|
278
|
-
const appNode = nodes.find(
|
|
288
|
+
const appNode = nodes.find(
|
|
289
|
+
(n) => (0, import_node.getNodeRoleFrom)({ workerMode: n.workerMode, appRole: n.appRole, isSandbox: n.isSandbox }) === "app"
|
|
290
|
+
);
|
|
279
291
|
if (appNode == null ? void 0 : appNode.nodeDetails) {
|
|
280
292
|
ctx.body = appNode.nodeDetails;
|
|
281
293
|
} else {
|
|
@@ -289,6 +301,8 @@ const clusterActions = {
|
|
|
289
301
|
arch: process.arch,
|
|
290
302
|
uptime: process.uptime(),
|
|
291
303
|
workerMode: currentMode,
|
|
304
|
+
appRole: process.env.APP_ROLE || "",
|
|
305
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true",
|
|
292
306
|
appPort: process.env.APP_PORT || "",
|
|
293
307
|
clusterMode: process.env.CLUSTER_MODE || ""
|
|
294
308
|
},
|
|
@@ -318,7 +332,10 @@ const clusterActions = {
|
|
|
318
332
|
*/
|
|
319
333
|
async list(ctx, next) {
|
|
320
334
|
const environments = [];
|
|
321
|
-
const
|
|
335
|
+
const registry = getClusterRegistry(ctx);
|
|
336
|
+
const nodes = await registry.getNodes();
|
|
337
|
+
const registryStatus = registry.getStatus();
|
|
338
|
+
let fallback = false;
|
|
322
339
|
if (nodes && nodes.length > 0) {
|
|
323
340
|
for (const env of nodes) {
|
|
324
341
|
environments.push({
|
|
@@ -331,23 +348,39 @@ const clusterActions = {
|
|
|
331
348
|
lastHeartbeatAt: env.lastHeartbeatAt ? new Date(env.lastHeartbeatAt).toISOString() : null,
|
|
332
349
|
status: env.status || "online",
|
|
333
350
|
workerMode: env.workerMode,
|
|
351
|
+
appRole: env.appRole,
|
|
334
352
|
isSandbox: env.isSandbox,
|
|
335
353
|
pid: env.pid
|
|
336
354
|
});
|
|
337
355
|
}
|
|
338
356
|
}
|
|
339
357
|
if (environments.length === 0) {
|
|
358
|
+
fallback = true;
|
|
359
|
+
const currentMode = process.env.WORKER_MODE || "main";
|
|
360
|
+
const appName = process.env.APP_NAME || ctx.app.name || "main";
|
|
340
361
|
environments.push({
|
|
341
|
-
|
|
362
|
+
id: (0, import_node.getLocalNodeId)(ctx.app),
|
|
363
|
+
name: `${appName} (${import_os.default.hostname()})`,
|
|
342
364
|
hostname: import_os.default.hostname(),
|
|
343
365
|
url: null,
|
|
344
366
|
available: true,
|
|
345
367
|
appVersion: null,
|
|
346
368
|
lastHeartbeatAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
347
|
-
status: "online"
|
|
369
|
+
status: "online",
|
|
370
|
+
workerMode: currentMode,
|
|
371
|
+
appRole: process.env.APP_ROLE || "",
|
|
372
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true",
|
|
373
|
+
pid: process.pid
|
|
348
374
|
});
|
|
349
375
|
}
|
|
350
|
-
ctx.body = {
|
|
376
|
+
ctx.body = {
|
|
377
|
+
data: environments,
|
|
378
|
+
meta: {
|
|
379
|
+
count: environments.length,
|
|
380
|
+
fallback,
|
|
381
|
+
registry: registryStatus
|
|
382
|
+
}
|
|
383
|
+
};
|
|
351
384
|
await next();
|
|
352
385
|
},
|
|
353
386
|
/**
|
|
@@ -48,6 +48,7 @@ var import_redis_node_registry = require("../adapters/redis-node-registry");
|
|
|
48
48
|
var import_node = require("../utils/node");
|
|
49
49
|
var import_redis = require("../utils/redis");
|
|
50
50
|
var import_packages = require("../../shared/packages");
|
|
51
|
+
var import_worker_processes = require("../../shared/worker-processes");
|
|
51
52
|
const ACTIVE_RUN_KEY = "cluster-manager:doctor:active";
|
|
52
53
|
const RESPONSE_KEY_PREFIX = "cluster-manager:doctor-response:";
|
|
53
54
|
const FINISH_LOCK_PREFIX = "cluster-manager:doctor:finish-lock:";
|
|
@@ -853,34 +854,88 @@ async function getRedisDiagnostics(app) {
|
|
|
853
854
|
}
|
|
854
855
|
}
|
|
855
856
|
async function getQueueDiagnostics(app) {
|
|
856
|
-
var _a, _b, _c;
|
|
857
|
+
var _a, _b, _c, _d, _e;
|
|
857
858
|
const eventQueue = getApp(app).eventQueue;
|
|
858
859
|
if (!eventQueue) {
|
|
859
860
|
return { available: false };
|
|
860
861
|
}
|
|
861
862
|
const adapter = eventQueue.adapter;
|
|
862
863
|
const channels = [];
|
|
864
|
+
const requiredProcesses = /* @__PURE__ */ new Set();
|
|
863
865
|
for (const [channel, options] of eventQueue.events || /* @__PURE__ */ new Map()) {
|
|
864
866
|
let pending = null;
|
|
865
867
|
if ((adapter == null ? void 0 : adapter.queues) && eventQueue.getFullChannel) {
|
|
866
868
|
const fullChannel = eventQueue.getFullChannel(channel, options.shared);
|
|
867
869
|
pending = ((_a = adapter.queues.get(fullChannel)) == null ? void 0 : _a.length) || 0;
|
|
868
870
|
}
|
|
871
|
+
const processName = (0, import_worker_processes.resolveWorkerProcessName)(channel);
|
|
872
|
+
const definition = (0, import_worker_processes.getWorkerProcessDefinition)(processName);
|
|
873
|
+
if (definition && !definition.sandbox) {
|
|
874
|
+
requiredProcesses.add(definition.name);
|
|
875
|
+
}
|
|
869
876
|
channels.push({
|
|
870
877
|
channel,
|
|
878
|
+
workerProcessName: definition == null ? void 0 : definition.name,
|
|
871
879
|
concurrency: options.concurrency || 1,
|
|
872
880
|
interval: options.interval || 250,
|
|
873
881
|
pending
|
|
874
882
|
});
|
|
875
883
|
}
|
|
884
|
+
for (const definition of import_worker_processes.WORKER_PROCESS_DEFINITIONS) {
|
|
885
|
+
if (definition.common && definition.pluginName) {
|
|
886
|
+
try {
|
|
887
|
+
if ((_c = (_b = app.pm) == null ? void 0 : _b.get) == null ? void 0 : _c.call(_b, definition.pluginName)) {
|
|
888
|
+
requiredProcesses.add(definition.name);
|
|
889
|
+
}
|
|
890
|
+
} catch {
|
|
891
|
+
}
|
|
892
|
+
}
|
|
893
|
+
}
|
|
894
|
+
const coverage = await getWorkerProcessCoverage(app, Array.from(requiredProcesses));
|
|
876
895
|
return {
|
|
877
896
|
available: true,
|
|
878
|
-
connected: ((
|
|
879
|
-
adapter: ((
|
|
897
|
+
connected: ((_d = eventQueue.isConnected) == null ? void 0 : _d.call(eventQueue)) || false,
|
|
898
|
+
adapter: ((_e = adapter == null ? void 0 : adapter.constructor) == null ? void 0 : _e.name) || "unknown",
|
|
880
899
|
channels,
|
|
900
|
+
coverage,
|
|
881
901
|
totalPending: channels.reduce((sum, item) => sum + (item.pending || 0), 0)
|
|
882
902
|
};
|
|
883
903
|
}
|
|
904
|
+
async function getWorkerProcessCoverage(app, requiredProcesses) {
|
|
905
|
+
var _a, _b, _c, _d;
|
|
906
|
+
const result = {
|
|
907
|
+
required: requiredProcesses,
|
|
908
|
+
covered: [],
|
|
909
|
+
missing: [],
|
|
910
|
+
wildcard: false,
|
|
911
|
+
stacks: []
|
|
912
|
+
};
|
|
913
|
+
try {
|
|
914
|
+
const repo = app.db.getRepository("orchestratorStacks");
|
|
915
|
+
const stacks = await repo.find({ filter: { enabled: true } });
|
|
916
|
+
const covered = /* @__PURE__ */ new Set();
|
|
917
|
+
for (const stack of stacks) {
|
|
918
|
+
const envVars = (_a = stack.get) == null ? void 0 : _a.call(stack, "envVars");
|
|
919
|
+
const workerMode = (0, import_worker_processes.normalizeWorkerMode)(((_b = stack.get) == null ? void 0 : _b.call(stack, "workerMode")) || (envVars == null ? void 0 : envVars.WORKER_MODE)) || "*";
|
|
920
|
+
result.stacks.push({
|
|
921
|
+
id: (_c = stack.get) == null ? void 0 : _c.call(stack, "id"),
|
|
922
|
+
name: (_d = stack.get) == null ? void 0 : _d.call(stack, "name"),
|
|
923
|
+
workerMode
|
|
924
|
+
});
|
|
925
|
+
if (workerMode === "*") {
|
|
926
|
+
result.wildcard = true;
|
|
927
|
+
}
|
|
928
|
+
for (const token of workerMode.split(",").filter(Boolean)) {
|
|
929
|
+
covered.add(token);
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
result.covered = Array.from(covered);
|
|
933
|
+
result.missing = result.wildcard ? [] : requiredProcesses.filter((processName) => !covered.has(processName));
|
|
934
|
+
} catch {
|
|
935
|
+
result.missing = requiredProcesses;
|
|
936
|
+
}
|
|
937
|
+
return result;
|
|
938
|
+
}
|
|
884
939
|
async function getOrchestratorDiagnostics(app, options) {
|
|
885
940
|
var _a;
|
|
886
941
|
const plugin = (_a = getApp(app).pm) == null ? void 0 : _a.get("plugin-cluster-manager");
|
|
@@ -937,6 +992,7 @@ async function getOrchestratorDiagnostics(app, options) {
|
|
|
937
992
|
};
|
|
938
993
|
}
|
|
939
994
|
function buildRecommendations(params) {
|
|
995
|
+
var _a;
|
|
940
996
|
const recommendations = [];
|
|
941
997
|
if (!params.redisAvailable) {
|
|
942
998
|
recommendations.push({
|
|
@@ -980,6 +1036,13 @@ function buildRecommendations(params) {
|
|
|
980
1036
|
message: "One or more worker nodes are missing configured packages or have failed package initialization."
|
|
981
1037
|
});
|
|
982
1038
|
}
|
|
1039
|
+
if ((_a = params.queueCoverageMissing) == null ? void 0 : _a.length) {
|
|
1040
|
+
recommendations.push({
|
|
1041
|
+
level: "warning",
|
|
1042
|
+
code: "worker_process_coverage_missing",
|
|
1043
|
+
message: `No explicit worker stack covers: ${params.queueCoverageMissing.join(", ")}.`
|
|
1044
|
+
});
|
|
1045
|
+
}
|
|
983
1046
|
if (params.topErrors > 0) {
|
|
984
1047
|
recommendations.push({
|
|
985
1048
|
level: "warning",
|
|
@@ -990,6 +1053,7 @@ function buildRecommendations(params) {
|
|
|
990
1053
|
return recommendations;
|
|
991
1054
|
}
|
|
992
1055
|
async function buildDoctorReport(app, run, finishReason) {
|
|
1056
|
+
var _a;
|
|
993
1057
|
const runId = String(run.runId);
|
|
994
1058
|
const startedAt = new Date(String(run.startedAt));
|
|
995
1059
|
const finishedAt = /* @__PURE__ */ new Date();
|
|
@@ -1021,7 +1085,8 @@ async function buildDoctorReport(app, run, finishReason) {
|
|
|
1021
1085
|
pluginLoadDrifts: pluginDiagnostics.loadDrifts.length,
|
|
1022
1086
|
packageDrifts: packageDiagnostics.packageDrifts.length,
|
|
1023
1087
|
redisAvailable: Boolean(redisDiagnostics.available),
|
|
1024
|
-
databaseOk: Boolean(databaseDiagnostics.ping.ok)
|
|
1088
|
+
databaseOk: Boolean(databaseDiagnostics.ping.ok),
|
|
1089
|
+
queueCoverageMissing: ((_a = queueDiagnostics.coverage) == null ? void 0 : _a.missing) || []
|
|
1025
1090
|
});
|
|
1026
1091
|
const criticalFindings = recommendations.filter((item) => item.level === "critical").length;
|
|
1027
1092
|
const warningFindings = recommendations.filter((item) => item.level === "warning").length;
|
|
@@ -31,7 +31,13 @@ __export(event_queue_monitor_exports, {
|
|
|
31
31
|
module.exports = __toCommonJS(event_queue_monitor_exports);
|
|
32
32
|
var import_redis = require("../utils/redis");
|
|
33
33
|
const REDIS_QUEUE_CONNECTION = "cluster-manager:queue-monitor";
|
|
34
|
-
const REDIS_QUEUE_PATTERNS = [
|
|
34
|
+
const REDIS_QUEUE_PATTERNS = [
|
|
35
|
+
"*:plugin-git-manager:review:queue",
|
|
36
|
+
"*:plugin-build-guide-block:build:queue",
|
|
37
|
+
"*:plugin-build-visualization-block:build:queue",
|
|
38
|
+
"file-preview-auth.ocr.queue",
|
|
39
|
+
"nocobase:event-queue:*"
|
|
40
|
+
];
|
|
35
41
|
function getQueueRedisUrl() {
|
|
36
42
|
return process.env.QUEUE_ADAPTER_REDIS_URL || process.env.REDIS_URL;
|
|
37
43
|
}
|
|
@@ -45,15 +51,38 @@ async function getQueueRedis(ctx) {
|
|
|
45
51
|
}
|
|
46
52
|
function knownRedisQueueKeys(ctx) {
|
|
47
53
|
const appName = ctx.app.name || process.env.APP_NAME || "main";
|
|
48
|
-
return [
|
|
54
|
+
return [
|
|
55
|
+
`${appName}:plugin-git-manager:review:queue`,
|
|
56
|
+
`${appName}:plugin-build-guide-block:build:queue`,
|
|
57
|
+
`${appName}:plugin-build-visualization-block:build:queue`,
|
|
58
|
+
"file-preview-auth.ocr.queue"
|
|
59
|
+
];
|
|
49
60
|
}
|
|
50
61
|
function isKnownRedisQueueKey(key) {
|
|
62
|
+
if (key.startsWith("nocobase:event-queue:")) return true;
|
|
51
63
|
return REDIS_QUEUE_PATTERNS.some((pattern) => {
|
|
52
64
|
const suffix = pattern.replace("*:", "");
|
|
53
65
|
return key === suffix || key.endsWith(`:${suffix}`);
|
|
54
66
|
});
|
|
55
67
|
}
|
|
56
68
|
function describeRedisQueueKey(key) {
|
|
69
|
+
if (key === "file-preview-auth.ocr.queue") {
|
|
70
|
+
return {
|
|
71
|
+
appName: "main",
|
|
72
|
+
plugin: "plugin-file-preview-auth",
|
|
73
|
+
queue: "ocr",
|
|
74
|
+
channel: key
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
if (key.startsWith("nocobase:event-queue:")) {
|
|
78
|
+
const channel = key.slice("nocobase:event-queue:".length);
|
|
79
|
+
return {
|
|
80
|
+
appName: channel.split(".")[0] || "main",
|
|
81
|
+
plugin: "event-queue",
|
|
82
|
+
queue: channel,
|
|
83
|
+
channel
|
|
84
|
+
};
|
|
85
|
+
}
|
|
57
86
|
const parts = String(key).split(":");
|
|
58
87
|
const queue = parts[parts.length - 2] || key;
|
|
59
88
|
const plugin = parts[parts.length - 3] || "unknown";
|
|
@@ -108,12 +137,13 @@ async function getRedisQueues(ctx) {
|
|
|
108
137
|
};
|
|
109
138
|
}
|
|
110
139
|
function parseRedisQueueMessage(raw, key, index) {
|
|
140
|
+
var _a;
|
|
111
141
|
let content = raw;
|
|
112
142
|
try {
|
|
113
143
|
content = JSON.parse(raw);
|
|
114
144
|
} catch {
|
|
115
145
|
}
|
|
116
|
-
const queuedAt = (content == null ? void 0 : content.queuedAt) ? Date.parse(content.queuedAt) : null;
|
|
146
|
+
const queuedAt = (content == null ? void 0 : content.queuedAt) ? Date.parse(content.queuedAt) : ((_a = content == null ? void 0 : content.options) == null ? void 0 : _a.timestamp) || null;
|
|
117
147
|
return {
|
|
118
148
|
id: `${key}:${index}`,
|
|
119
149
|
index,
|
|
@@ -30,6 +30,7 @@ __export(orchestrator_exports, {
|
|
|
30
30
|
});
|
|
31
31
|
module.exports = __toCommonJS(orchestrator_exports);
|
|
32
32
|
var import_redis = require("../utils/redis");
|
|
33
|
+
var import_worker_processes = require("../../shared/worker-processes");
|
|
33
34
|
function getAdapter(ctx) {
|
|
34
35
|
const plugin = ctx.app.pm.get("plugin-cluster-manager");
|
|
35
36
|
if (!(plugin == null ? void 0 : plugin.orchestrator)) {
|
|
@@ -63,6 +64,28 @@ async function assertManagedContainer(ctx, adapter, stack, containerId) {
|
|
|
63
64
|
ctx.throw(403, err.message || `Container ${containerId} is not managed by stack "${stack.name}"`);
|
|
64
65
|
}
|
|
65
66
|
}
|
|
67
|
+
function applyWorkerMode(stack, workerMode) {
|
|
68
|
+
var _a, _b;
|
|
69
|
+
stack.workerMode = workerMode;
|
|
70
|
+
stack.envVars = {
|
|
71
|
+
...stack.envVars || {},
|
|
72
|
+
APP_ROLE: ((_a = stack.envVars) == null ? void 0 : _a.APP_ROLE) || "worker",
|
|
73
|
+
WORKER_MODE: workerMode,
|
|
74
|
+
SKILL_HUB_SANDBOX: ((_b = stack.envVars) == null ? void 0 : _b.SKILL_HUB_SANDBOX) || "false"
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
async function resolveMappedWorkerMode(ctx, stack) {
|
|
78
|
+
const mappingsRepo = ctx.db.getRepository("workerQueueMappings");
|
|
79
|
+
const assigned = await mappingsRepo.find({
|
|
80
|
+
filter: {
|
|
81
|
+
stackId: stack.id,
|
|
82
|
+
enabled: true
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
return (0, import_worker_processes.normalizeWorkerMode)(
|
|
86
|
+
assigned.map((mapping) => String(mapping.get("queueName") || "")).filter(Boolean).join(",")
|
|
87
|
+
);
|
|
88
|
+
}
|
|
66
89
|
const orchestratorActions = {
|
|
67
90
|
/**
|
|
68
91
|
* GET /workerOrchestrator:ping
|
|
@@ -123,49 +146,42 @@ const orchestratorActions = {
|
|
|
123
146
|
* Body: { stackId: 1, replicas: 3 }
|
|
124
147
|
* Leader-only
|
|
125
148
|
*
|
|
126
|
-
* Before scaling, resolves
|
|
127
|
-
* WORKER_MODE into
|
|
128
|
-
*
|
|
149
|
+
* Before scaling, resolves the stack-level worker mode and injects
|
|
150
|
+
* WORKER_MODE into envVars so new containers process only selected queues.
|
|
151
|
+
* Queue mappings remain as a fallback for legacy stacks.
|
|
129
152
|
*/
|
|
130
153
|
async scale(ctx, next) {
|
|
154
|
+
var _a;
|
|
131
155
|
assertLeader(ctx);
|
|
132
156
|
const adapter = getAdapter(ctx);
|
|
133
157
|
const { stackId, replicas } = ctx.action.params.values || ctx.action.params;
|
|
134
158
|
if (replicas === void 0 || replicas === null) ctx.throw(400, "replicas is required");
|
|
135
159
|
if (replicas < 0 || replicas > 20) ctx.throw(400, "replicas must be between 0 and 20");
|
|
136
160
|
const stack = await getStack(ctx, stackId);
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
);
|
|
152
|
-
stack.envVars = {
|
|
153
|
-
...stack.envVars || {},
|
|
154
|
-
WORKER_MODE: workerMode
|
|
155
|
-
};
|
|
161
|
+
const stackWorkerMode = (0, import_worker_processes.normalizeWorkerMode)(stack.workerMode);
|
|
162
|
+
const envWorkerMode = (0, import_worker_processes.normalizeWorkerMode)((_a = stack.envVars) == null ? void 0 : _a.WORKER_MODE);
|
|
163
|
+
if (stackWorkerMode) {
|
|
164
|
+
applyWorkerMode(stack, stackWorkerMode);
|
|
165
|
+
ctx.app.logger.info(`[Orchestrator] Using stack WORKER_MODE=${stackWorkerMode} for "${stack.name}"`);
|
|
166
|
+
} else if (envWorkerMode && envWorkerMode !== "*") {
|
|
167
|
+
applyWorkerMode(stack, envWorkerMode);
|
|
168
|
+
ctx.app.logger.info(`[Orchestrator] Using env WORKER_MODE=${envWorkerMode} for "${stack.name}"`);
|
|
169
|
+
} else {
|
|
170
|
+
try {
|
|
171
|
+
const mappedWorkerMode = await resolveMappedWorkerMode(ctx, stack);
|
|
172
|
+
if (mappedWorkerMode) {
|
|
173
|
+
applyWorkerMode(stack, mappedWorkerMode);
|
|
174
|
+
ctx.app.logger.info(`[Orchestrator] Using mapped WORKER_MODE=${mappedWorkerMode} for "${stack.name}"`);
|
|
156
175
|
} else {
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
};
|
|
176
|
+
const fallbackWorkerMode = envWorkerMode || "*";
|
|
177
|
+
applyWorkerMode(stack, fallbackWorkerMode);
|
|
178
|
+
ctx.app.logger.info(`[Orchestrator] Using fallback WORKER_MODE=${fallbackWorkerMode} for "${stack.name}"`);
|
|
161
179
|
}
|
|
180
|
+
} catch (err) {
|
|
181
|
+
const fallbackWorkerMode = envWorkerMode || "*";
|
|
182
|
+
ctx.app.logger.debug(`[Orchestrator] Queue mappings not available: ${err.message}`);
|
|
183
|
+
applyWorkerMode(stack, fallbackWorkerMode);
|
|
162
184
|
}
|
|
163
|
-
} catch (err) {
|
|
164
|
-
ctx.app.logger.debug(`[Orchestrator] Queue mappings not available: ${err.message}`);
|
|
165
|
-
stack.envVars = {
|
|
166
|
-
...stack.envVars || {},
|
|
167
|
-
WORKER_MODE: "*"
|
|
168
|
-
};
|
|
169
185
|
}
|
|
170
186
|
const result = await adapter.scale(stack, Number(replicas));
|
|
171
187
|
const repo = ctx.db.getRepository("orchestratorStacks");
|
|
@@ -69,9 +69,12 @@ const tasksActions = {
|
|
|
69
69
|
ctx.app.logger.info(`[cluster-manager] Canceling task ${filterByTk} by user ${user}`);
|
|
70
70
|
const pluginName = "@nocobase/plugin-async-task-manager";
|
|
71
71
|
try {
|
|
72
|
-
await ctx.app.pubSubManager.publish(
|
|
73
|
-
|
|
74
|
-
|
|
72
|
+
await ctx.app.pubSubManager.publish(
|
|
73
|
+
`${pluginName}.task.cancel`,
|
|
74
|
+
JSON.stringify({
|
|
75
|
+
taskId: filterByTk
|
|
76
|
+
})
|
|
77
|
+
);
|
|
75
78
|
} catch {
|
|
76
79
|
}
|
|
77
80
|
await repo.update({
|
|
@@ -116,7 +119,7 @@ const tasksActions = {
|
|
|
116
119
|
const pluginName = "@nocobase/plugin-async-task-manager";
|
|
117
120
|
try {
|
|
118
121
|
await ctx.app.eventQueue.publish(`${pluginName}.task`, {
|
|
119
|
-
|
|
122
|
+
id: filterByTk
|
|
120
123
|
});
|
|
121
124
|
} catch {
|
|
122
125
|
}
|
|
@@ -130,10 +133,7 @@ const tasksActions = {
|
|
|
130
133
|
ctx.app.logger.info(`[cluster-manager] Purging tasks (days=${days}) by user ${user}`);
|
|
131
134
|
const repo = ctx.db.getRepository("asyncTasks");
|
|
132
135
|
const filter = {
|
|
133
|
-
$and: [
|
|
134
|
-
{ status: { $ne: 0 } },
|
|
135
|
-
{ status: { $ne: null } }
|
|
136
|
-
]
|
|
136
|
+
$and: [{ status: { $ne: 0 } }, { status: { $ne: null } }]
|
|
137
137
|
};
|
|
138
138
|
if (days && Number(days) > 0) {
|
|
139
139
|
const date = /* @__PURE__ */ new Date();
|