plugin-cluster-manager 1.1.10 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client-v2.d.ts +2 -0
- package/client-v2.js +1 -0
- package/client.js +1 -0
- package/dist/client/index.js +1 -1
- package/dist/client-v2/914.5dc1105cf3ada6a6.js +10 -0
- package/dist/client-v2/index.js +10 -0
- package/dist/externalVersion.js +6 -5
- package/dist/locale/en-US.json +138 -28
- package/dist/locale/vi-VN.json +139 -28
- package/dist/locale/zh-CN.json +140 -28
- package/dist/server/actions/cache-monitor.js +301 -0
- package/dist/server/actions/cluster-nodes.js +391 -11
- package/dist/server/actions/doctor.js +1246 -0
- package/dist/server/actions/orchestrator.js +37 -0
- package/dist/server/actions/queue-mappings.js +107 -0
- package/dist/server/collections/cluster-manager-doctor-runs.js +52 -0
- package/dist/server/collections/cluster-manager-doctor.js +44 -0
- package/dist/server/collections/worker-queue-mappings.js +106 -0
- package/dist/server/hooks/cacheInvalidationHooks.js +81 -0
- package/dist/server/middlewares/listMetaCacheMiddleware.js +79 -0
- package/dist/server/orchestrator/PackageManager.js +21 -24
- package/dist/server/orchestrator/docker-adapter.js +49 -27
- package/dist/server/plugin.js +71 -16
- package/dist/server/queue-scanner.js +141 -0
- package/dist/server/utils/node.js +30 -2
- package/dist/server/utils/versionManager.js +91 -0
- package/package.json +9 -5
- package/server.js +1 -0
- package/src/client/AclCacheManager.tsx +292 -287
- package/src/client/CacheMonitor.tsx +166 -179
- package/src/client/ClusterManagerLayout.tsx +54 -42
- package/src/client/ClusterNodes.tsx +698 -418
- package/src/client/ContainerOrchestrator.tsx +184 -102
- package/src/client/Doctor.tsx +559 -0
- package/src/client/NginxCacheManager.tsx +415 -0
- package/src/client/PluginOperations.tsx +234 -234
- package/src/client/QueueAssignment.tsx +355 -0
- package/src/client/TaskManager.tsx +194 -187
- package/src/client/WorkflowExecutions.tsx +243 -238
- package/src/client/index.tsx +22 -14
- package/src/client/utils/clientSafeCache.ts +41 -0
- package/src/client/utils/requestDedupInterceptor.ts +213 -0
- package/src/client-v2/plugin.tsx +24 -0
- package/src/locale/en-US.json +138 -28
- package/src/locale/vi-VN.json +139 -28
- package/src/locale/zh-CN.json +140 -28
- package/src/server/__tests__/doctor.test.ts +53 -0
- package/src/server/actions/acl-cache.ts +272 -272
- package/src/server/actions/cache-monitor.ts +453 -116
- package/src/server/actions/cluster-nodes.ts +878 -378
- package/src/server/actions/doctor.ts +1536 -0
- package/src/server/actions/orchestrator.ts +54 -2
- package/src/server/actions/queue-mappings.ts +94 -0
- package/src/server/collections/cluster-manager-doctor-runs.ts +23 -0
- package/src/server/collections/cluster-manager-doctor.ts +19 -0
- package/src/server/collections/worker-queue-mappings.ts +85 -0
- package/src/server/hooks/cacheInvalidationHooks.ts +58 -0
- package/src/server/middlewares/listMetaCacheMiddleware.ts +55 -0
- package/src/server/orchestrator/PackageManager.ts +20 -24
- package/src/server/orchestrator/docker-adapter.ts +74 -37
- package/src/server/plugin.ts +347 -270
- package/src/server/queue-scanner.ts +154 -0
- package/src/server/utils/node.ts +48 -0
- package/src/server/utils/versionManager.ts +69 -0
- package/dist/client/AclCacheManager.d.ts +0 -2
- package/dist/client/CacheMonitor.d.ts +0 -2
- package/dist/client/ClusterManagerLayout.d.ts +0 -2
- package/dist/client/ClusterNodes.d.ts +0 -2
- package/dist/client/ContainerOrchestrator.d.ts +0 -2
- package/dist/client/EventQueueMonitor.d.ts +0 -2
- package/dist/client/LockMonitor.d.ts +0 -2
- package/dist/client/PackageInstaller.d.ts +0 -2
- package/dist/client/PluginOperations.d.ts +0 -2
- package/dist/client/RedisMonitor.d.ts +0 -2
- package/dist/client/TaskManager.d.ts +0 -2
- package/dist/client/WorkflowExecutions.d.ts +0 -2
- package/dist/client/index.d.ts +0 -5
- package/dist/client/utils.d.ts +0 -12
- package/dist/index.d.ts +0 -2
- package/dist/server/actions/acl-cache.d.ts +0 -53
- package/dist/server/actions/cache-monitor.d.ts +0 -23
- package/dist/server/actions/cluster-nodes.d.ts +0 -49
- package/dist/server/actions/event-queue-monitor.d.ts +0 -13
- package/dist/server/actions/lock-monitor.d.ts +0 -19
- package/dist/server/actions/orchestrator.d.ts +0 -58
- package/dist/server/actions/package-manager.d.ts +0 -6
- package/dist/server/actions/plugin-operations.d.ts +0 -6
- package/dist/server/actions/redis-monitor.d.ts +0 -12
- package/dist/server/actions/tasks.d.ts +0 -7
- package/dist/server/actions/workflow-executions.d.ts +0 -7
- package/dist/server/adapters/redis-lock-adapter.d.ts +0 -15
- package/dist/server/adapters/redis-node-registry.d.ts +0 -12
- package/dist/server/adapters/redis-pubsub-adapter.d.ts +0 -16
- package/dist/server/collections/app.d.ts +0 -8
- package/dist/server/collections/cluster-manager-acl-cache.d.ts +0 -22
- package/dist/server/collections/cluster-manager-cache-mgr.d.ts +0 -22
- package/dist/server/collections/cluster-manager-cluster.d.ts +0 -22
- package/dist/server/collections/cluster-manager-lock.d.ts +0 -22
- package/dist/server/collections/cluster-manager-plugins.d.ts +0 -18
- package/dist/server/collections/cluster-manager-queue.d.ts +0 -22
- package/dist/server/collections/cluster-manager-redis.d.ts +0 -22
- package/dist/server/collections/cluster-manager-workflow.d.ts +0 -22
- package/dist/server/collections/cluster-manager.d.ts +0 -22
- package/dist/server/collections/orchestrator-settings.d.ts +0 -59
- package/dist/server/collections/orchestrator-stacks.d.ts +0 -102
- package/dist/server/collections/worker-orchestrator.d.ts +0 -22
- package/dist/server/collections/worker-packages-configs.d.ts +0 -3
- package/dist/server/collections/worker-packages.d.ts +0 -22
- package/dist/server/orchestrator/PackageManager.d.ts +0 -39
- package/dist/server/orchestrator/docker-adapter.d.ts +0 -41
- package/dist/server/orchestrator/index.d.ts +0 -4
- package/dist/server/orchestrator/k8s-adapter.d.ts +0 -50
- package/dist/server/orchestrator/leader-election.d.ts +0 -48
- package/dist/server/orchestrator/types.d.ts +0 -84
- package/dist/server/plugin.d.ts +0 -26
- package/dist/server/utils/node.d.ts +0 -6
- package/dist/server/utils/redis.d.ts +0 -29
- package/dist/shared/packages.d.ts +0 -23
- /package/{dist/server/index.d.ts → src/client-v2/index.tsx} +0 -0
package/src/server/plugin.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { redisActions } from './actions/redis-monitor';
|
|
|
7
7
|
import { aclCacheActions, createAclCacheMiddleware } from './actions/acl-cache';
|
|
8
8
|
import { clusterActions, readLocalLogs } from './actions/cluster-nodes';
|
|
9
9
|
import { getRedisClient } from './utils/redis';
|
|
10
|
-
import { getLocalNodeId } from './utils/node';
|
|
10
|
+
import { getLocalNodeId, isWorkerMode } from './utils/node';
|
|
11
11
|
import { eventQueueActions } from './actions/event-queue-monitor';
|
|
12
12
|
import { lockActions } from './actions/lock-monitor';
|
|
13
13
|
import { cacheMonitorActions } from './actions/cache-monitor';
|
|
@@ -16,12 +16,16 @@ import { RedisNodeRegistry } from './adapters/redis-node-registry';
|
|
|
16
16
|
import { RedisLockAdapter } from './adapters/redis-lock-adapter';
|
|
17
17
|
import { orchestratorActions } from './actions/orchestrator';
|
|
18
18
|
import { pluginOperationsActions } from './actions/plugin-operations';
|
|
19
|
+
import { queueMappingsActions } from './actions/queue-mappings';
|
|
19
20
|
import type { IOrchestratorAdapter } from './orchestrator/types';
|
|
20
21
|
import { DockerAdapter } from './orchestrator/docker-adapter';
|
|
21
22
|
import { K8sAdapter } from './orchestrator/k8s-adapter';
|
|
22
23
|
import { LeaderElection } from './orchestrator/leader-election';
|
|
23
24
|
import { packageManagerActions } from './actions/package-manager';
|
|
24
25
|
import { PackageManager } from './orchestrator/PackageManager';
|
|
26
|
+
import { createListMetaCacheMiddleware } from './middlewares/listMetaCacheMiddleware';
|
|
27
|
+
import { registerCacheHooks } from './hooks/cacheInvalidationHooks';
|
|
28
|
+
import { collectLocalDoctorSnapshot, doctorActions } from './actions/doctor';
|
|
25
29
|
|
|
26
30
|
export class PluginClusterManagerServer extends Plugin {
|
|
27
31
|
public nodeRegistry: RedisNodeRegistry;
|
|
@@ -54,8 +58,10 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
54
58
|
this.nodeRegistry?.start();
|
|
55
59
|
|
|
56
60
|
// Automatically install packages on boot for worker nodes
|
|
57
|
-
const
|
|
58
|
-
|
|
61
|
+
const isWorker =
|
|
62
|
+
isWorkerMode(process.env.WORKER_MODE) ||
|
|
63
|
+
process.env.APP_ROLE === 'worker' ||
|
|
64
|
+
process.env.APP_ROLE === 'sandbox';
|
|
59
65
|
if (isWorker) {
|
|
60
66
|
setTimeout(async () => {
|
|
61
67
|
try {
|
|
@@ -68,29 +74,31 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
68
74
|
}
|
|
69
75
|
if (config) {
|
|
70
76
|
this.app.logger.info('[ClusterManager] Auto-installing configured packages on worker boot...');
|
|
71
|
-
|
|
77
|
+
|
|
72
78
|
const { packagesFromConfig } = require('../shared/packages');
|
|
73
|
-
|
|
79
|
+
|
|
74
80
|
const configured = packagesFromConfig({
|
|
75
81
|
aptPackages: config.get('aptPackages'),
|
|
76
82
|
pythonPackages: config.get('pythonPackages'),
|
|
77
83
|
npmPackages: config.get('npmPackages'),
|
|
78
84
|
});
|
|
79
|
-
|
|
85
|
+
|
|
80
86
|
let custom = { python: [], node: [], npm: [] };
|
|
81
87
|
try {
|
|
82
88
|
const customRaw = config.get('customPackages');
|
|
83
89
|
if (customRaw) custom = typeof customRaw === 'string' ? JSON.parse(customRaw) : customRaw;
|
|
84
|
-
} catch {
|
|
85
|
-
|
|
90
|
+
} catch (err) {
|
|
91
|
+
// ignore
|
|
92
|
+
}
|
|
93
|
+
|
|
86
94
|
const unique = (arr: any[]) => Array.from(new Set(arr.filter(Boolean)));
|
|
87
|
-
|
|
95
|
+
|
|
88
96
|
const packages = {
|
|
89
97
|
apt: unique([...(configured.apt || [])]),
|
|
90
98
|
npm: unique([...(configured.npm || []), ...(custom.node || []), ...(custom.npm || [])]),
|
|
91
99
|
python: unique([...(configured.python || []), ...(custom.python || [])]),
|
|
92
100
|
};
|
|
93
|
-
|
|
101
|
+
|
|
94
102
|
const pm = new PackageManager(this.app);
|
|
95
103
|
await pm.executeInstall({
|
|
96
104
|
targetRole: 'all', // executeInstall will filter internally based on current role
|
|
@@ -99,7 +107,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
99
107
|
aptMirrorUrl: config.get('aptMirrorUrl'),
|
|
100
108
|
npmRegistryUrl: config.get('npmRegistryUrl'),
|
|
101
109
|
pypiIndexUrl: config.get('pypiIndexUrl'),
|
|
102
|
-
}
|
|
110
|
+
},
|
|
103
111
|
});
|
|
104
112
|
}
|
|
105
113
|
} catch (err: any) {
|
|
@@ -115,9 +123,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
115
123
|
|
|
116
124
|
// Workflow hook to trace executing node
|
|
117
125
|
this.app.db.on('executions.afterSave', async (model: any) => {
|
|
118
|
-
|
|
119
|
-
const isWorker = mode === 'worker' || mode === 'task' || mode === '*';
|
|
120
|
-
if (isWorker) {
|
|
126
|
+
if (isWorkerMode(process.env.WORKER_MODE)) {
|
|
121
127
|
const id = model.get('id');
|
|
122
128
|
const redis = getRedisClient(this.app);
|
|
123
129
|
if (id && redis) {
|
|
@@ -136,7 +142,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
136
142
|
if (lockMgr && lockMgr.registry && !lockMgr.registry.get('redis') && !lockMgr.adapters.get('redis')) {
|
|
137
143
|
lockMgr.registerAdapter('redis', {
|
|
138
144
|
Adapter: RedisLockAdapter,
|
|
139
|
-
options: { app: this.app }
|
|
145
|
+
options: { app: this.app },
|
|
140
146
|
});
|
|
141
147
|
this.app.logger.info('[ClusterManager] Polyfilled RedisLockAdapter as an active distributed lock provider');
|
|
142
148
|
}
|
|
@@ -145,7 +151,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
145
151
|
const pubSub = (this.app as any).pubSubManager;
|
|
146
152
|
if (pubSub) {
|
|
147
153
|
const myNodeId = getLocalNodeId(this.app);
|
|
148
|
-
|
|
154
|
+
|
|
149
155
|
// ── Log request handler: ONLY the targeted node receives this via dynamic channel ──
|
|
150
156
|
pubSub.subscribe(`cluster-manager:log-request:${myNodeId}`, async (msg: string) => {
|
|
151
157
|
try {
|
|
@@ -156,14 +162,56 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
156
162
|
|
|
157
163
|
const logData = await readLocalLogs(this.app, lines || 200);
|
|
158
164
|
const responseKey = `cluster-manager:log-response:${requestId}`;
|
|
159
|
-
await redis.sendCommand([
|
|
160
|
-
'SET', responseKey, JSON.stringify(logData), 'EX', '30',
|
|
161
|
-
]);
|
|
165
|
+
await redis.sendCommand(['SET', responseKey, JSON.stringify(logData), 'EX', '30']);
|
|
162
166
|
this.app.logger.debug(`[ClusterManager] Served log request ${requestId} for ${targetNodeId}`);
|
|
163
167
|
} catch (err: any) {
|
|
164
168
|
this.app.logger.error(`[ClusterManager] Error handling log request: ${err.message}`);
|
|
165
169
|
}
|
|
166
|
-
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
pubSub.subscribe(`cluster-manager:doctor-collect:${myNodeId}`, async (msg: string) => {
|
|
173
|
+
const redis = getRedisClient(this.app);
|
|
174
|
+
let requestId = '';
|
|
175
|
+
try {
|
|
176
|
+
const parsed = typeof msg === 'string' ? JSON.parse(msg) : msg;
|
|
177
|
+
requestId = parsed.requestId;
|
|
178
|
+
if (!redis || !requestId) return;
|
|
179
|
+
|
|
180
|
+
const snapshot = await collectLocalDoctorSnapshot(this.app, {
|
|
181
|
+
runId: parsed.runId,
|
|
182
|
+
sinceMs: parsed.sinceMs,
|
|
183
|
+
untilMs: parsed.untilMs,
|
|
184
|
+
maxLines: parsed.maxLines,
|
|
185
|
+
});
|
|
186
|
+
await redis.sendCommand([
|
|
187
|
+
'SET',
|
|
188
|
+
`cluster-manager:doctor-response:${requestId}`,
|
|
189
|
+
JSON.stringify(snapshot),
|
|
190
|
+
'EX',
|
|
191
|
+
'90',
|
|
192
|
+
]);
|
|
193
|
+
this.app.logger.debug(`[ClusterManager] Served doctor snapshot request ${requestId}`);
|
|
194
|
+
} catch (err: unknown) {
|
|
195
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
196
|
+
this.app.logger.error(`[ClusterManager] Error handling doctor snapshot request: ${message}`);
|
|
197
|
+
if (redis && requestId) {
|
|
198
|
+
const fallback = {
|
|
199
|
+
nodeId: getLocalNodeId(this.app),
|
|
200
|
+
collectedAt: new Date().toISOString(),
|
|
201
|
+
error: message,
|
|
202
|
+
};
|
|
203
|
+
await redis
|
|
204
|
+
.sendCommand([
|
|
205
|
+
'SET',
|
|
206
|
+
`cluster-manager:doctor-response:${requestId}`,
|
|
207
|
+
JSON.stringify(fallback),
|
|
208
|
+
'EX',
|
|
209
|
+
'90',
|
|
210
|
+
])
|
|
211
|
+
.catch(() => {});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
});
|
|
167
215
|
// Package installation handler. PubSub delivers this to every node, and PackageManager
|
|
168
216
|
// filters by the requested target role before executing anything locally.
|
|
169
217
|
pubSub.subscribe('cluster-manager.install-packages', async (payload: any) => {
|
|
@@ -177,85 +225,97 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
177
225
|
});
|
|
178
226
|
|
|
179
227
|
pubSub.subscribe('cluster-manager:restart', (msg: string) => {
|
|
180
|
-
try {
|
|
181
|
-
let target = msg;
|
|
182
|
-
let mode = 'hard';
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
//
|
|
218
|
-
this.app.resourcer.define({
|
|
219
|
-
name: '
|
|
220
|
-
actions:
|
|
221
|
-
});
|
|
222
|
-
|
|
223
|
-
//
|
|
224
|
-
this.app.resourcer.define({
|
|
225
|
-
name: '
|
|
226
|
-
actions:
|
|
227
|
-
});
|
|
228
|
-
|
|
229
|
-
//
|
|
230
|
-
this.app.resourcer.define({
|
|
231
|
-
name: '
|
|
232
|
-
actions:
|
|
233
|
-
});
|
|
234
|
-
|
|
235
|
-
//
|
|
236
|
-
this.app.resourcer.define({
|
|
237
|
-
name: '
|
|
238
|
-
actions:
|
|
239
|
-
});
|
|
240
|
-
|
|
241
|
-
//
|
|
242
|
-
this.app.resourcer.define({
|
|
243
|
-
name: '
|
|
244
|
-
actions:
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
//
|
|
248
|
-
this.app.resourcer.define({
|
|
249
|
-
name: '
|
|
250
|
-
actions:
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
//
|
|
254
|
-
this.app.resourcer.define({
|
|
255
|
-
name: '
|
|
256
|
-
actions:
|
|
257
|
-
});
|
|
258
|
-
|
|
228
|
+
try {
|
|
229
|
+
let target = msg;
|
|
230
|
+
let mode = 'hard';
|
|
231
|
+
let targetNodeId = '';
|
|
232
|
+
|
|
233
|
+
if (msg.startsWith('{')) {
|
|
234
|
+
const parsed = JSON.parse(msg);
|
|
235
|
+
target = parsed.hostname || parsed.target || '';
|
|
236
|
+
targetNodeId = parsed.targetNodeId || '';
|
|
237
|
+
mode = parsed.mode || 'hard';
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const myNodeId = getLocalNodeId(this.app);
|
|
241
|
+
const shouldRestart = targetNodeId ? targetNodeId === myNodeId : target === os.hostname() || target === '*';
|
|
242
|
+
if (shouldRestart) {
|
|
243
|
+
this.app.logger.warn(`[ClusterManager] Received ${mode} restart command for node ${os.hostname()}...`);
|
|
244
|
+
setTimeout(async () => {
|
|
245
|
+
try {
|
|
246
|
+
if (mode === 'soft') {
|
|
247
|
+
this.app.logger.warn(`[ClusterManager] Triggering NocoBase Soft Restart...`);
|
|
248
|
+
await this.app.restart();
|
|
249
|
+
} else {
|
|
250
|
+
this.app.logger.warn(`[ClusterManager] Shutting down Node.js process for Hard Restart...`);
|
|
251
|
+
await this.app.stop();
|
|
252
|
+
process.exit(1);
|
|
253
|
+
}
|
|
254
|
+
} catch (e: any) {
|
|
255
|
+
// ignore
|
|
256
|
+
}
|
|
257
|
+
}, 1000); // 1-second delay so HTTP API can gracefully respond first
|
|
258
|
+
}
|
|
259
|
+
} catch (err) {
|
|
260
|
+
this.app.logger.error(`[ClusterManager] Parse error for restart message: ${msg}`);
|
|
261
|
+
}
|
|
262
|
+
});
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Task management (reads asyncTasks table)
|
|
266
|
+
this.app.resourcer.define({
|
|
267
|
+
name: 'clusterManager',
|
|
268
|
+
actions: tasksActions,
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
// Workflow execution management (reads executions + jobs tables)
|
|
272
|
+
this.app.resourcer.define({
|
|
273
|
+
name: 'clusterManagerWorkflow',
|
|
274
|
+
actions: workflowActions,
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
// Redis live metrics
|
|
278
|
+
this.app.resourcer.define({
|
|
279
|
+
name: 'clusterManagerRedis',
|
|
280
|
+
actions: redisActions,
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
// ACL cache management
|
|
284
|
+
this.app.resourcer.define({
|
|
285
|
+
name: 'clusterManagerAclCache',
|
|
286
|
+
actions: aclCacheActions,
|
|
287
|
+
});
|
|
288
|
+
|
|
289
|
+
// Cluster nodes & health
|
|
290
|
+
this.app.resourcer.define({
|
|
291
|
+
name: 'clusterManagerCluster',
|
|
292
|
+
actions: clusterActions,
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
// Time-boxed diagnostic sessions and report download
|
|
296
|
+
this.app.resourcer.define({
|
|
297
|
+
name: 'clusterManagerDoctor',
|
|
298
|
+
actions: doctorActions,
|
|
299
|
+
});
|
|
300
|
+
|
|
301
|
+
// Event queue monitoring
|
|
302
|
+
this.app.resourcer.define({
|
|
303
|
+
name: 'clusterManagerQueue',
|
|
304
|
+
actions: eventQueueActions,
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
// Distributed lock monitoring
|
|
308
|
+
this.app.resourcer.define({
|
|
309
|
+
name: 'clusterManagerLock',
|
|
310
|
+
actions: lockActions,
|
|
311
|
+
});
|
|
312
|
+
|
|
313
|
+
// Cache manager monitoring
|
|
314
|
+
this.app.resourcer.define({
|
|
315
|
+
name: 'clusterManagerCacheMgr',
|
|
316
|
+
actions: cacheMonitorActions,
|
|
317
|
+
});
|
|
318
|
+
|
|
259
319
|
// Package manager (installs apt/npm/python packages across nodes)
|
|
260
320
|
this.app.resourcer.define({
|
|
261
321
|
name: 'workerPackages',
|
|
@@ -267,7 +327,13 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
267
327
|
name: 'clusterManagerPlugins',
|
|
268
328
|
actions: pluginOperationsActions,
|
|
269
329
|
});
|
|
270
|
-
|
|
330
|
+
|
|
331
|
+
// Queue Mappings (queue-to-worker-stack assignments)
|
|
332
|
+
this.app.resourcer.define({
|
|
333
|
+
name: 'workerQueueMappings',
|
|
334
|
+
actions: queueMappingsActions,
|
|
335
|
+
});
|
|
336
|
+
|
|
271
337
|
// Install ACL cache middleware inside the ACL chain so cached permissions are not overwritten.
|
|
272
338
|
const aclCacheMiddleware = createAclCacheMiddleware(this.app);
|
|
273
339
|
(this.app as any).acl.use(aclCacheMiddleware, {
|
|
@@ -275,119 +341,131 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
275
341
|
before: 'core',
|
|
276
342
|
after: 'allow-manager',
|
|
277
343
|
});
|
|
278
|
-
|
|
279
|
-
//
|
|
280
|
-
this.app
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
344
|
+
|
|
345
|
+
// Install collections:listMeta resource cache middleware after setCurrentRole
|
|
346
|
+
const listMetaCacheMiddleware = createListMetaCacheMiddleware(this.app);
|
|
347
|
+
this.app.resourcer.use(listMetaCacheMiddleware, {
|
|
348
|
+
tag: 'listMetaCache',
|
|
349
|
+
after: 'setCurrentRole',
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
// Register DB hooks for invalidating cache versions
|
|
353
|
+
registerCacheHooks(this.app);
|
|
354
|
+
|
|
355
|
+
// Lightweight healthcheck endpoint avoiding workflow pre-action and resourcer spam
|
|
356
|
+
this.app.use(async (ctx: any, next: any) => {
|
|
357
|
+
if (ctx.path === '/api/clusterManager:health' && (ctx.method === 'GET' || ctx.method === 'HEAD')) {
|
|
358
|
+
ctx.body = {
|
|
359
|
+
status: 'ok',
|
|
360
|
+
version: process.env.NOCOBASE_VERSION || process.version,
|
|
361
|
+
mode: process.env.WORKER_MODE || 'main',
|
|
362
|
+
};
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
await next();
|
|
366
|
+
});
|
|
367
|
+
|
|
368
|
+
// Admin-only access
|
|
369
|
+
this.app.acl.registerSnippet({
|
|
370
|
+
name: `pm.${this.name}`,
|
|
371
|
+
actions: [
|
|
372
|
+
'clusterManager:*',
|
|
373
|
+
'clusterManagerWorkflow:*',
|
|
374
|
+
'clusterManagerRedis:*',
|
|
375
|
+
'clusterManagerAclCache:*',
|
|
376
|
+
'clusterManagerCluster:*',
|
|
377
|
+
'clusterManagerDoctor:*',
|
|
378
|
+
'clusterManagerQueue:*',
|
|
379
|
+
'clusterManagerLock:*',
|
|
380
|
+
'clusterManagerCacheMgr:*',
|
|
304
381
|
'workerOrchestrator:*',
|
|
305
382
|
'orchestratorStacks:*',
|
|
306
383
|
'workerPackages:*',
|
|
307
384
|
'clusterManagerPlugins:*',
|
|
385
|
+
'workerQueueMappings:*',
|
|
308
386
|
],
|
|
309
387
|
});
|
|
310
|
-
|
|
311
|
-
// ── Container Orchestrator ──
|
|
312
|
-
await this.initOrchestrator();
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
private registerPubSubAdapter() {
|
|
316
|
-
const url = process.env.PUBSUB_ADAPTER_REDIS_URL;
|
|
317
|
-
if (!url) {
|
|
318
|
-
this.app.logger.info('[cluster-manager] PUBSUB_ADAPTER_REDIS_URL not set, skipping Redis PubSub adapter');
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
// Don't override if another plugin already set an adapter
|
|
323
|
-
const existingAdapter = (this.app.pubSubManager as any).adapter;
|
|
324
|
-
if (existingAdapter) {
|
|
325
|
-
this.app.logger.info('[cluster-manager] PubSub adapter already registered, skipping');
|
|
326
|
-
return;
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
const adapter = new RedisPubSubAdapter(url, this.app.logger);
|
|
330
|
-
this.app.pubSubManager.setAdapter(adapter);
|
|
331
|
-
this.app.logger.info('[cluster-manager] Redis PubSub adapter registered');
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
/**
|
|
335
|
-
* Initialize the Container Orchestrator subsystem.
|
|
336
|
-
* Config is loaded from DB (orchestratorSettings collection) first,
|
|
337
|
-
* then falls back to ORCHESTRATOR_ADAPTER env var.
|
|
338
|
-
* This allows manual configuration via the NocoBase admin UI.
|
|
339
|
-
*/
|
|
340
|
-
private async initOrchestrator() {
|
|
341
|
-
// Always register actions + collections so the UI can configure settings
|
|
342
|
-
// even before an adapter is connected
|
|
343
|
-
this.app.resourcer.define({
|
|
344
|
-
name: 'workerOrchestrator',
|
|
345
|
-
actions: {
|
|
346
|
-
...orchestratorActions,
|
|
347
|
-
// Settings CRUD: read/write orchestrator config from DB
|
|
348
|
-
async getSettings(ctx: any, next: () => Promise<void>) {
|
|
349
|
-
try {
|
|
350
|
-
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
351
|
-
let settings = await repo.findOne();
|
|
352
|
-
if (!settings) {
|
|
353
|
-
settings = await repo.create({
|
|
354
|
-
values: { adapterType: process.env.ORCHESTRATOR_ADAPTER || 'none' },
|
|
355
|
-
});
|
|
356
|
-
}
|
|
357
|
-
ctx.body = settings.toJSON();
|
|
358
|
-
} catch {
|
|
359
|
-
// Table may not exist during migration window
|
|
360
|
-
ctx.body = { adapterType: 'none', _note: 'Settings table not yet ready.' };
|
|
361
|
-
}
|
|
362
|
-
await next();
|
|
363
|
-
},
|
|
364
|
-
async saveSettings(ctx: any, next: () => Promise<void>) {
|
|
365
|
-
const values = ctx.action.params.values || {};
|
|
366
|
-
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
367
|
-
let settings = await repo.findOne();
|
|
368
|
-
if (settings) {
|
|
369
|
-
await repo.update({ filterByTk: settings.get('id'), values });
|
|
370
|
-
} else {
|
|
371
|
-
settings = await repo.create({ values });
|
|
372
|
-
}
|
|
373
|
-
// Reinitialize adapter with new settings
|
|
374
|
-
const plugin = ctx.app.pm.get('plugin-cluster-manager') as PluginClusterManagerServer;
|
|
375
|
-
await plugin.connectAdapter(values);
|
|
376
|
-
ctx.body = { success: true, message: 'Settings saved. Adapter reinitialized.' };
|
|
377
|
-
await next();
|
|
378
|
-
},
|
|
379
|
-
},
|
|
380
|
-
});
|
|
381
|
-
|
|
382
|
-
// Load settings from DB and try to connect
|
|
383
|
-
try {
|
|
384
|
-
const repo = this.app.db.getRepository('orchestratorSettings');
|
|
385
|
-
const settings = await repo.findOne();
|
|
386
|
-
if (settings) {
|
|
387
|
-
await this.connectAdapter(settings.toJSON());
|
|
388
|
-
} else {
|
|
389
|
-
// Fall back to env var for initial setup
|
|
390
|
-
const envAdapter = process.env.ORCHESTRATOR_ADAPTER;
|
|
388
|
+
|
|
389
|
+
// ── Container Orchestrator ──
|
|
390
|
+
await this.initOrchestrator();
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
private registerPubSubAdapter() {
|
|
394
|
+
const url = process.env.PUBSUB_ADAPTER_REDIS_URL;
|
|
395
|
+
if (!url) {
|
|
396
|
+
this.app.logger.info('[cluster-manager] PUBSUB_ADAPTER_REDIS_URL not set, skipping Redis PubSub adapter');
|
|
397
|
+
return;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
// Don't override if another plugin already set an adapter
|
|
401
|
+
const existingAdapter = (this.app.pubSubManager as any).adapter;
|
|
402
|
+
if (existingAdapter) {
|
|
403
|
+
this.app.logger.info('[cluster-manager] PubSub adapter already registered, skipping');
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
const adapter = new RedisPubSubAdapter(url, this.app.logger);
|
|
408
|
+
this.app.pubSubManager.setAdapter(adapter);
|
|
409
|
+
this.app.logger.info('[cluster-manager] Redis PubSub adapter registered');
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/**
|
|
413
|
+
* Initialize the Container Orchestrator subsystem.
|
|
414
|
+
* Config is loaded from DB (orchestratorSettings collection) first,
|
|
415
|
+
* then falls back to ORCHESTRATOR_ADAPTER env var.
|
|
416
|
+
* This allows manual configuration via the NocoBase admin UI.
|
|
417
|
+
*/
|
|
418
|
+
private async initOrchestrator() {
|
|
419
|
+
// Always register actions + collections so the UI can configure settings
|
|
420
|
+
// even before an adapter is connected
|
|
421
|
+
this.app.resourcer.define({
|
|
422
|
+
name: 'workerOrchestrator',
|
|
423
|
+
actions: {
|
|
424
|
+
...orchestratorActions,
|
|
425
|
+
// Settings CRUD: read/write orchestrator config from DB
|
|
426
|
+
async getSettings(ctx: any, next: () => Promise<void>) {
|
|
427
|
+
try {
|
|
428
|
+
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
429
|
+
let settings = await repo.findOne();
|
|
430
|
+
if (!settings) {
|
|
431
|
+
settings = await repo.create({
|
|
432
|
+
values: { adapterType: process.env.ORCHESTRATOR_ADAPTER || 'none' },
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
ctx.body = settings.toJSON();
|
|
436
|
+
} catch {
|
|
437
|
+
// Table may not exist during migration window
|
|
438
|
+
ctx.body = { adapterType: 'none', _note: 'Settings table not yet ready.' };
|
|
439
|
+
}
|
|
440
|
+
await next();
|
|
441
|
+
},
|
|
442
|
+
async saveSettings(ctx: any, next: () => Promise<void>) {
|
|
443
|
+
const values = ctx.action.params.values || {};
|
|
444
|
+
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
445
|
+
let settings = await repo.findOne();
|
|
446
|
+
if (settings) {
|
|
447
|
+
await repo.update({ filterByTk: settings.get('id'), values });
|
|
448
|
+
} else {
|
|
449
|
+
settings = await repo.create({ values });
|
|
450
|
+
}
|
|
451
|
+
// Reinitialize adapter with new settings
|
|
452
|
+
const plugin = ctx.app.pm.get('plugin-cluster-manager') as PluginClusterManagerServer;
|
|
453
|
+
await plugin.connectAdapter(values);
|
|
454
|
+
ctx.body = { success: true, message: 'Settings saved. Adapter reinitialized.' };
|
|
455
|
+
await next();
|
|
456
|
+
},
|
|
457
|
+
},
|
|
458
|
+
});
|
|
459
|
+
|
|
460
|
+
// Load settings from DB and try to connect
|
|
461
|
+
try {
|
|
462
|
+
const repo = this.app.db.getRepository('orchestratorSettings');
|
|
463
|
+
const settings = await repo.findOne();
|
|
464
|
+
if (settings) {
|
|
465
|
+
await this.connectAdapter(settings.toJSON());
|
|
466
|
+
} else {
|
|
467
|
+
// Fall back to env var for initial setup
|
|
468
|
+
const envAdapter = process.env.ORCHESTRATOR_ADAPTER;
|
|
391
469
|
if (envAdapter && envAdapter !== 'none') {
|
|
392
470
|
await this.connectAdapter({
|
|
393
471
|
adapterType: envAdapter,
|
|
@@ -398,10 +476,10 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
398
476
|
this.app.logger.info('[Orchestrator] No adapter configured — configurable via Cluster Manager UI');
|
|
399
477
|
}
|
|
400
478
|
}
|
|
401
|
-
} catch (err: any) {
|
|
402
|
-
this.app.logger.warn(`[Orchestrator] Could not load settings: ${err.message}`);
|
|
403
|
-
}
|
|
404
|
-
|
|
479
|
+
} catch (err: any) {
|
|
480
|
+
this.app.logger.warn(`[Orchestrator] Could not load settings: ${err.message}`);
|
|
481
|
+
}
|
|
482
|
+
|
|
405
483
|
// Leader election runs on app nodes only. Worker-only pods still load the
|
|
406
484
|
// plugin for monitoring/package installation, but they must not become the
|
|
407
485
|
// Kubernetes orchestrator leader.
|
|
@@ -411,41 +489,41 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
411
489
|
disabledReason: workerOnlyNode ? 'Worker-only nodes do not run orchestrator write operations.' : '',
|
|
412
490
|
});
|
|
413
491
|
await this.leaderElection.init();
|
|
414
|
-
|
|
415
|
-
(this.app as any).on('afterStart', async () => {
|
|
416
|
-
if (this.leaderElection) {
|
|
417
|
-
await this.leaderElection.tryBecomeLeader();
|
|
418
|
-
}
|
|
419
|
-
});
|
|
420
|
-
|
|
421
|
-
(this.app as any).on('beforeStop', async () => {
|
|
422
|
-
if (this.leaderElection) {
|
|
423
|
-
await this.leaderElection.release();
|
|
424
|
-
}
|
|
425
|
-
});
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
/**
|
|
429
|
-
* Connect (or reconnect) the orchestrator adapter based on settings.
|
|
430
|
-
* Can be called at startup or when user saves new settings via UI.
|
|
431
|
-
*/
|
|
492
|
+
|
|
493
|
+
(this.app as any).on('afterStart', async () => {
|
|
494
|
+
if (this.leaderElection) {
|
|
495
|
+
await this.leaderElection.tryBecomeLeader();
|
|
496
|
+
}
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
(this.app as any).on('beforeStop', async () => {
|
|
500
|
+
if (this.leaderElection) {
|
|
501
|
+
await this.leaderElection.release();
|
|
502
|
+
}
|
|
503
|
+
});
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
/**
|
|
507
|
+
* Connect (or reconnect) the orchestrator adapter based on settings.
|
|
508
|
+
* Can be called at startup or when user saves new settings via UI.
|
|
509
|
+
*/
|
|
432
510
|
public async connectAdapter(settings: any): Promise<boolean> {
|
|
433
|
-
const adapterType = settings?.adapterType || 'none';
|
|
434
|
-
|
|
435
|
-
if (adapterType === 'none') {
|
|
436
|
-
this.orchestrator = null;
|
|
437
|
-
this.app.logger.info('[Orchestrator] Adapter disabled');
|
|
438
|
-
return false;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
try {
|
|
442
|
-
if (adapterType === 'docker') {
|
|
443
|
-
const opts: any = {};
|
|
511
|
+
const adapterType = settings?.adapterType || 'none';
|
|
512
|
+
|
|
513
|
+
if (adapterType === 'none') {
|
|
514
|
+
this.orchestrator = null;
|
|
515
|
+
this.app.logger.info('[Orchestrator] Adapter disabled');
|
|
516
|
+
return false;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
try {
|
|
520
|
+
if (adapterType === 'docker') {
|
|
521
|
+
const opts: any = {};
|
|
444
522
|
if (settings.dockerHost) {
|
|
445
|
-
// TCP connection (remote Docker or Docker Desktop on Windows)
|
|
446
|
-
const url = new URL(settings.dockerHost);
|
|
447
|
-
opts.host = url.hostname;
|
|
448
|
-
opts.port = parseInt(url.port, 10) || 2376;
|
|
523
|
+
// TCP connection (remote Docker or Docker Desktop on Windows)
|
|
524
|
+
const url = new URL(settings.dockerHost);
|
|
525
|
+
opts.host = url.hostname;
|
|
526
|
+
opts.port = parseInt(url.port, 10) || 2376;
|
|
449
527
|
} else {
|
|
450
528
|
opts.socketPath = settings.dockerSocketPath || '/var/run/docker.sock';
|
|
451
529
|
}
|
|
@@ -458,34 +536,33 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
458
536
|
namespace: settings.k8sNamespace || 'nocobase',
|
|
459
537
|
workerLabelSelector: settings.workerLabelSelector || 'role=worker',
|
|
460
538
|
});
|
|
461
|
-
this.app.logger.info('[Orchestrator] Kubernetes adapter initialized');
|
|
462
|
-
} else {
|
|
463
|
-
this.app.logger.warn(`[Orchestrator] Unknown adapter type: ${adapterType}`);
|
|
464
|
-
this.orchestrator = null;
|
|
465
|
-
return false;
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// Test connectivity
|
|
469
|
-
const connected = await this.orchestrator.ping();
|
|
470
|
-
if (!connected) {
|
|
471
|
-
this.app.logger.error(`[Orchestrator] Failed to connect to ${adapterType} runtime`);
|
|
472
|
-
this.orchestrator = null;
|
|
473
|
-
return false;
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
this.app.logger.info(`[Orchestrator] ✅ Connected to ${adapterType} runtime`);
|
|
477
|
-
return true;
|
|
478
|
-
} catch (err: any) {
|
|
479
|
-
this.app.logger.error(`[Orchestrator] Adapter init failed: ${err.message}`);
|
|
480
|
-
this.orchestrator = null;
|
|
481
|
-
return false;
|
|
482
|
-
}
|
|
539
|
+
this.app.logger.info('[Orchestrator] Kubernetes adapter initialized');
|
|
540
|
+
} else {
|
|
541
|
+
this.app.logger.warn(`[Orchestrator] Unknown adapter type: ${adapterType}`);
|
|
542
|
+
this.orchestrator = null;
|
|
543
|
+
return false;
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
// Test connectivity
|
|
547
|
+
const connected = await this.orchestrator.ping();
|
|
548
|
+
if (!connected) {
|
|
549
|
+
this.app.logger.error(`[Orchestrator] Failed to connect to ${adapterType} runtime`);
|
|
550
|
+
this.orchestrator = null;
|
|
551
|
+
return false;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
this.app.logger.info(`[Orchestrator] ✅ Connected to ${adapterType} runtime`);
|
|
555
|
+
return true;
|
|
556
|
+
} catch (err: any) {
|
|
557
|
+
this.app.logger.error(`[Orchestrator] Adapter init failed: ${err.message}`);
|
|
558
|
+
this.orchestrator = null;
|
|
559
|
+
return false;
|
|
560
|
+
}
|
|
483
561
|
}
|
|
484
562
|
|
|
485
563
|
private isWorkerOnlyNode(): boolean {
|
|
486
|
-
|
|
487
|
-
return workerMode === 'worker' || workerMode === 'task' || workerMode === '*';
|
|
564
|
+
return isWorkerMode(process.env.WORKER_MODE);
|
|
488
565
|
}
|
|
489
566
|
}
|
|
490
|
-
|
|
491
|
-
export default PluginClusterManagerServer;
|
|
567
|
+
|
|
568
|
+
export default PluginClusterManagerServer;
|