plugin-cluster-manager 1.1.10 → 1.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client.js +1 -0
- package/dist/client/Doctor.d.ts +2 -0
- package/dist/client/NginxCacheManager.d.ts +2 -0
- package/dist/client/index.js +1 -1
- package/dist/client/utils/clientSafeCache.d.ts +3 -0
- package/dist/client/utils/requestDedupInterceptor.d.ts +2 -0
- package/dist/externalVersion.js +5 -5
- package/dist/locale/en-US.json +97 -1
- package/dist/locale/vi-VN.json +98 -1
- package/dist/locale/zh-CN.json +98 -1
- package/dist/server/actions/cache-monitor.d.ts +10 -0
- package/dist/server/actions/cache-monitor.js +301 -0
- package/dist/server/actions/cluster-nodes.d.ts +15 -0
- package/dist/server/actions/cluster-nodes.js +394 -10
- package/dist/server/actions/doctor.d.ts +82 -0
- package/dist/server/actions/doctor.js +1250 -0
- package/dist/server/collections/cluster-manager-doctor-runs.d.ts +3 -0
- package/dist/server/collections/cluster-manager-doctor-runs.js +52 -0
- package/dist/server/collections/cluster-manager-doctor.d.ts +18 -0
- package/dist/server/collections/cluster-manager-doctor.js +44 -0
- package/dist/server/hooks/cacheInvalidationHooks.d.ts +1 -0
- package/dist/server/hooks/cacheInvalidationHooks.js +81 -0
- package/dist/server/middlewares/listMetaCacheMiddleware.d.ts +2 -0
- package/dist/server/middlewares/listMetaCacheMiddleware.js +79 -0
- package/dist/server/orchestrator/PackageManager.js +20 -16
- package/dist/server/plugin.js +61 -8
- package/dist/server/utils/versionManager.d.ts +10 -0
- package/dist/server/utils/versionManager.js +91 -0
- package/package.json +41 -41
- package/server.js +1 -0
- package/src/client/CacheMonitor.tsx +166 -179
- package/src/client/ClusterManagerLayout.tsx +48 -42
- package/src/client/ClusterNodes.tsx +691 -418
- package/src/client/Doctor.tsx +559 -0
- package/src/client/NginxCacheManager.tsx +415 -0
- package/src/client/PluginOperations.tsx +234 -234
- package/src/client/index.tsx +22 -14
- package/src/client/utils/clientSafeCache.ts +41 -0
- package/src/client/utils/requestDedupInterceptor.ts +213 -0
- package/src/locale/en-US.json +97 -1
- package/src/locale/vi-VN.json +98 -1
- package/src/locale/zh-CN.json +98 -1
- package/src/server/__tests__/doctor.test.ts +53 -0
- package/src/server/actions/acl-cache.ts +272 -272
- package/src/server/actions/cache-monitor.ts +453 -116
- package/src/server/actions/cluster-nodes.ts +882 -378
- package/src/server/actions/doctor.ts +1540 -0
- package/src/server/collections/cluster-manager-doctor-runs.ts +23 -0
- package/src/server/collections/cluster-manager-doctor.ts +19 -0
- package/src/server/hooks/cacheInvalidationHooks.ts +58 -0
- package/src/server/middlewares/listMetaCacheMiddleware.ts +55 -0
- package/src/server/orchestrator/PackageManager.ts +19 -15
- package/src/server/plugin.ts +338 -263
- package/src/server/utils/versionManager.ts +69 -0
package/src/server/plugin.ts
CHANGED
|
@@ -22,6 +22,9 @@ import { K8sAdapter } from './orchestrator/k8s-adapter';
|
|
|
22
22
|
import { LeaderElection } from './orchestrator/leader-election';
|
|
23
23
|
import { packageManagerActions } from './actions/package-manager';
|
|
24
24
|
import { PackageManager } from './orchestrator/PackageManager';
|
|
25
|
+
import { createListMetaCacheMiddleware } from './middlewares/listMetaCacheMiddleware';
|
|
26
|
+
import { registerCacheHooks } from './hooks/cacheInvalidationHooks';
|
|
27
|
+
import { collectLocalDoctorSnapshot, doctorActions } from './actions/doctor';
|
|
25
28
|
|
|
26
29
|
export class PluginClusterManagerServer extends Plugin {
|
|
27
30
|
public nodeRegistry: RedisNodeRegistry;
|
|
@@ -55,7 +58,12 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
55
58
|
|
|
56
59
|
// Automatically install packages on boot for worker nodes
|
|
57
60
|
const mode = process.env.WORKER_MODE || 'main';
|
|
58
|
-
const isWorker =
|
|
61
|
+
const isWorker =
|
|
62
|
+
mode === 'worker' ||
|
|
63
|
+
mode === 'task' ||
|
|
64
|
+
mode === '*' ||
|
|
65
|
+
process.env.APP_ROLE === 'worker' ||
|
|
66
|
+
process.env.APP_ROLE === 'sandbox';
|
|
59
67
|
if (isWorker) {
|
|
60
68
|
setTimeout(async () => {
|
|
61
69
|
try {
|
|
@@ -68,29 +76,31 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
68
76
|
}
|
|
69
77
|
if (config) {
|
|
70
78
|
this.app.logger.info('[ClusterManager] Auto-installing configured packages on worker boot...');
|
|
71
|
-
|
|
79
|
+
|
|
72
80
|
const { packagesFromConfig } = require('../shared/packages');
|
|
73
|
-
|
|
81
|
+
|
|
74
82
|
const configured = packagesFromConfig({
|
|
75
83
|
aptPackages: config.get('aptPackages'),
|
|
76
84
|
pythonPackages: config.get('pythonPackages'),
|
|
77
85
|
npmPackages: config.get('npmPackages'),
|
|
78
86
|
});
|
|
79
|
-
|
|
87
|
+
|
|
80
88
|
let custom = { python: [], node: [], npm: [] };
|
|
81
89
|
try {
|
|
82
90
|
const customRaw = config.get('customPackages');
|
|
83
91
|
if (customRaw) custom = typeof customRaw === 'string' ? JSON.parse(customRaw) : customRaw;
|
|
84
|
-
} catch {
|
|
85
|
-
|
|
92
|
+
} catch (err) {
|
|
93
|
+
// ignore
|
|
94
|
+
}
|
|
95
|
+
|
|
86
96
|
const unique = (arr: any[]) => Array.from(new Set(arr.filter(Boolean)));
|
|
87
|
-
|
|
97
|
+
|
|
88
98
|
const packages = {
|
|
89
99
|
apt: unique([...(configured.apt || [])]),
|
|
90
100
|
npm: unique([...(configured.npm || []), ...(custom.node || []), ...(custom.npm || [])]),
|
|
91
101
|
python: unique([...(configured.python || []), ...(custom.python || [])]),
|
|
92
102
|
};
|
|
93
|
-
|
|
103
|
+
|
|
94
104
|
const pm = new PackageManager(this.app);
|
|
95
105
|
await pm.executeInstall({
|
|
96
106
|
targetRole: 'all', // executeInstall will filter internally based on current role
|
|
@@ -99,7 +109,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
99
109
|
aptMirrorUrl: config.get('aptMirrorUrl'),
|
|
100
110
|
npmRegistryUrl: config.get('npmRegistryUrl'),
|
|
101
111
|
pypiIndexUrl: config.get('pypiIndexUrl'),
|
|
102
|
-
}
|
|
112
|
+
},
|
|
103
113
|
});
|
|
104
114
|
}
|
|
105
115
|
} catch (err: any) {
|
|
@@ -136,7 +146,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
136
146
|
if (lockMgr && lockMgr.registry && !lockMgr.registry.get('redis') && !lockMgr.adapters.get('redis')) {
|
|
137
147
|
lockMgr.registerAdapter('redis', {
|
|
138
148
|
Adapter: RedisLockAdapter,
|
|
139
|
-
options: { app: this.app }
|
|
149
|
+
options: { app: this.app },
|
|
140
150
|
});
|
|
141
151
|
this.app.logger.info('[ClusterManager] Polyfilled RedisLockAdapter as an active distributed lock provider');
|
|
142
152
|
}
|
|
@@ -145,7 +155,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
145
155
|
const pubSub = (this.app as any).pubSubManager;
|
|
146
156
|
if (pubSub) {
|
|
147
157
|
const myNodeId = getLocalNodeId(this.app);
|
|
148
|
-
|
|
158
|
+
|
|
149
159
|
// ── Log request handler: ONLY the targeted node receives this via dynamic channel ──
|
|
150
160
|
pubSub.subscribe(`cluster-manager:log-request:${myNodeId}`, async (msg: string) => {
|
|
151
161
|
try {
|
|
@@ -156,14 +166,56 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
156
166
|
|
|
157
167
|
const logData = await readLocalLogs(this.app, lines || 200);
|
|
158
168
|
const responseKey = `cluster-manager:log-response:${requestId}`;
|
|
159
|
-
await redis.sendCommand([
|
|
160
|
-
'SET', responseKey, JSON.stringify(logData), 'EX', '30',
|
|
161
|
-
]);
|
|
169
|
+
await redis.sendCommand(['SET', responseKey, JSON.stringify(logData), 'EX', '30']);
|
|
162
170
|
this.app.logger.debug(`[ClusterManager] Served log request ${requestId} for ${targetNodeId}`);
|
|
163
171
|
} catch (err: any) {
|
|
164
172
|
this.app.logger.error(`[ClusterManager] Error handling log request: ${err.message}`);
|
|
165
173
|
}
|
|
166
|
-
});
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
pubSub.subscribe(`cluster-manager:doctor-collect:${myNodeId}`, async (msg: string) => {
|
|
177
|
+
const redis = getRedisClient(this.app);
|
|
178
|
+
let requestId = '';
|
|
179
|
+
try {
|
|
180
|
+
const parsed = typeof msg === 'string' ? JSON.parse(msg) : msg;
|
|
181
|
+
requestId = parsed.requestId;
|
|
182
|
+
if (!redis || !requestId) return;
|
|
183
|
+
|
|
184
|
+
const snapshot = await collectLocalDoctorSnapshot(this.app, {
|
|
185
|
+
runId: parsed.runId,
|
|
186
|
+
sinceMs: parsed.sinceMs,
|
|
187
|
+
untilMs: parsed.untilMs,
|
|
188
|
+
maxLines: parsed.maxLines,
|
|
189
|
+
});
|
|
190
|
+
await redis.sendCommand([
|
|
191
|
+
'SET',
|
|
192
|
+
`cluster-manager:doctor-response:${requestId}`,
|
|
193
|
+
JSON.stringify(snapshot),
|
|
194
|
+
'EX',
|
|
195
|
+
'90',
|
|
196
|
+
]);
|
|
197
|
+
this.app.logger.debug(`[ClusterManager] Served doctor snapshot request ${requestId}`);
|
|
198
|
+
} catch (err: unknown) {
|
|
199
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
200
|
+
this.app.logger.error(`[ClusterManager] Error handling doctor snapshot request: ${message}`);
|
|
201
|
+
if (redis && requestId) {
|
|
202
|
+
const fallback = {
|
|
203
|
+
nodeId: getLocalNodeId(this.app),
|
|
204
|
+
collectedAt: new Date().toISOString(),
|
|
205
|
+
error: message,
|
|
206
|
+
};
|
|
207
|
+
await redis
|
|
208
|
+
.sendCommand([
|
|
209
|
+
'SET',
|
|
210
|
+
`cluster-manager:doctor-response:${requestId}`,
|
|
211
|
+
JSON.stringify(fallback),
|
|
212
|
+
'EX',
|
|
213
|
+
'90',
|
|
214
|
+
])
|
|
215
|
+
.catch(() => {});
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
});
|
|
167
219
|
// Package installation handler. PubSub delivers this to every node, and PackageManager
|
|
168
220
|
// filters by the requested target role before executing anything locally.
|
|
169
221
|
pubSub.subscribe('cluster-manager.install-packages', async (payload: any) => {
|
|
@@ -177,85 +229,97 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
177
229
|
});
|
|
178
230
|
|
|
179
231
|
pubSub.subscribe('cluster-manager:restart', (msg: string) => {
|
|
180
|
-
try {
|
|
181
|
-
let target = msg;
|
|
182
|
-
let mode = 'hard';
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
//
|
|
218
|
-
this.app.resourcer.define({
|
|
219
|
-
name: '
|
|
220
|
-
actions:
|
|
221
|
-
});
|
|
222
|
-
|
|
223
|
-
//
|
|
224
|
-
this.app.resourcer.define({
|
|
225
|
-
name: '
|
|
226
|
-
actions:
|
|
227
|
-
});
|
|
228
|
-
|
|
229
|
-
//
|
|
230
|
-
this.app.resourcer.define({
|
|
231
|
-
name: '
|
|
232
|
-
actions:
|
|
233
|
-
});
|
|
234
|
-
|
|
235
|
-
//
|
|
236
|
-
this.app.resourcer.define({
|
|
237
|
-
name: '
|
|
238
|
-
actions:
|
|
239
|
-
});
|
|
240
|
-
|
|
241
|
-
//
|
|
242
|
-
this.app.resourcer.define({
|
|
243
|
-
name: '
|
|
244
|
-
actions:
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
//
|
|
248
|
-
this.app.resourcer.define({
|
|
249
|
-
name: '
|
|
250
|
-
actions:
|
|
251
|
-
});
|
|
252
|
-
|
|
253
|
-
//
|
|
254
|
-
this.app.resourcer.define({
|
|
255
|
-
name: '
|
|
256
|
-
actions:
|
|
257
|
-
});
|
|
258
|
-
|
|
232
|
+
try {
|
|
233
|
+
let target = msg;
|
|
234
|
+
let mode = 'hard';
|
|
235
|
+
let targetNodeId = '';
|
|
236
|
+
|
|
237
|
+
if (msg.startsWith('{')) {
|
|
238
|
+
const parsed = JSON.parse(msg);
|
|
239
|
+
target = parsed.hostname || parsed.target || '';
|
|
240
|
+
targetNodeId = parsed.targetNodeId || '';
|
|
241
|
+
mode = parsed.mode || 'hard';
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const myNodeId = getLocalNodeId(this.app);
|
|
245
|
+
const shouldRestart = targetNodeId ? targetNodeId === myNodeId : target === os.hostname() || target === '*';
|
|
246
|
+
if (shouldRestart) {
|
|
247
|
+
this.app.logger.warn(`[ClusterManager] Received ${mode} restart command for node ${os.hostname()}...`);
|
|
248
|
+
setTimeout(async () => {
|
|
249
|
+
try {
|
|
250
|
+
if (mode === 'soft') {
|
|
251
|
+
this.app.logger.warn(`[ClusterManager] Triggering NocoBase Soft Restart...`);
|
|
252
|
+
await this.app.restart();
|
|
253
|
+
} else {
|
|
254
|
+
this.app.logger.warn(`[ClusterManager] Shutting down Node.js process for Hard Restart...`);
|
|
255
|
+
await this.app.stop();
|
|
256
|
+
process.exit(1);
|
|
257
|
+
}
|
|
258
|
+
} catch (e: any) {
|
|
259
|
+
// ignore
|
|
260
|
+
}
|
|
261
|
+
}, 1000); // 1-second delay so HTTP API can gracefully respond first
|
|
262
|
+
}
|
|
263
|
+
} catch (err) {
|
|
264
|
+
this.app.logger.error(`[ClusterManager] Parse error for restart message: ${msg}`);
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
// Task management (reads asyncTasks table)
|
|
270
|
+
this.app.resourcer.define({
|
|
271
|
+
name: 'clusterManager',
|
|
272
|
+
actions: tasksActions,
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
// Workflow execution management (reads executions + jobs tables)
|
|
276
|
+
this.app.resourcer.define({
|
|
277
|
+
name: 'clusterManagerWorkflow',
|
|
278
|
+
actions: workflowActions,
|
|
279
|
+
});
|
|
280
|
+
|
|
281
|
+
// Redis live metrics
|
|
282
|
+
this.app.resourcer.define({
|
|
283
|
+
name: 'clusterManagerRedis',
|
|
284
|
+
actions: redisActions,
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
// ACL cache management
|
|
288
|
+
this.app.resourcer.define({
|
|
289
|
+
name: 'clusterManagerAclCache',
|
|
290
|
+
actions: aclCacheActions,
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
// Cluster nodes & health
|
|
294
|
+
this.app.resourcer.define({
|
|
295
|
+
name: 'clusterManagerCluster',
|
|
296
|
+
actions: clusterActions,
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
// Time-boxed diagnostic sessions and report download
|
|
300
|
+
this.app.resourcer.define({
|
|
301
|
+
name: 'clusterManagerDoctor',
|
|
302
|
+
actions: doctorActions,
|
|
303
|
+
});
|
|
304
|
+
|
|
305
|
+
// Event queue monitoring
|
|
306
|
+
this.app.resourcer.define({
|
|
307
|
+
name: 'clusterManagerQueue',
|
|
308
|
+
actions: eventQueueActions,
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
// Distributed lock monitoring
|
|
312
|
+
this.app.resourcer.define({
|
|
313
|
+
name: 'clusterManagerLock',
|
|
314
|
+
actions: lockActions,
|
|
315
|
+
});
|
|
316
|
+
|
|
317
|
+
// Cache manager monitoring
|
|
318
|
+
this.app.resourcer.define({
|
|
319
|
+
name: 'clusterManagerCacheMgr',
|
|
320
|
+
actions: cacheMonitorActions,
|
|
321
|
+
});
|
|
322
|
+
|
|
259
323
|
// Package manager (installs apt/npm/python packages across nodes)
|
|
260
324
|
this.app.resourcer.define({
|
|
261
325
|
name: 'workerPackages',
|
|
@@ -267,7 +331,7 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
267
331
|
name: 'clusterManagerPlugins',
|
|
268
332
|
actions: pluginOperationsActions,
|
|
269
333
|
});
|
|
270
|
-
|
|
334
|
+
|
|
271
335
|
// Install ACL cache middleware inside the ACL chain so cached permissions are not overwritten.
|
|
272
336
|
const aclCacheMiddleware = createAclCacheMiddleware(this.app);
|
|
273
337
|
(this.app as any).acl.use(aclCacheMiddleware, {
|
|
@@ -275,119 +339,130 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
275
339
|
before: 'core',
|
|
276
340
|
after: 'allow-manager',
|
|
277
341
|
});
|
|
278
|
-
|
|
279
|
-
//
|
|
280
|
-
this.app
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
342
|
+
|
|
343
|
+
// Install collections:listMeta resource cache middleware after setCurrentRole
|
|
344
|
+
const listMetaCacheMiddleware = createListMetaCacheMiddleware(this.app);
|
|
345
|
+
this.app.resourcer.use(listMetaCacheMiddleware, {
|
|
346
|
+
tag: 'listMetaCache',
|
|
347
|
+
after: 'setCurrentRole',
|
|
348
|
+
});
|
|
349
|
+
|
|
350
|
+
// Register DB hooks for invalidating cache versions
|
|
351
|
+
registerCacheHooks(this.app);
|
|
352
|
+
|
|
353
|
+
// Lightweight healthcheck endpoint avoiding workflow pre-action and resourcer spam
|
|
354
|
+
this.app.use(async (ctx: any, next: any) => {
|
|
355
|
+
if (ctx.path === '/api/clusterManager:health' && (ctx.method === 'GET' || ctx.method === 'HEAD')) {
|
|
356
|
+
ctx.body = {
|
|
357
|
+
status: 'ok',
|
|
358
|
+
version: process.env.NOCOBASE_VERSION || process.version,
|
|
359
|
+
mode: process.env.WORKER_MODE || 'main',
|
|
360
|
+
};
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
await next();
|
|
364
|
+
});
|
|
365
|
+
|
|
366
|
+
// Admin-only access
|
|
367
|
+
this.app.acl.registerSnippet({
|
|
368
|
+
name: `pm.${this.name}`,
|
|
369
|
+
actions: [
|
|
370
|
+
'clusterManager:*',
|
|
371
|
+
'clusterManagerWorkflow:*',
|
|
372
|
+
'clusterManagerRedis:*',
|
|
373
|
+
'clusterManagerAclCache:*',
|
|
374
|
+
'clusterManagerCluster:*',
|
|
375
|
+
'clusterManagerDoctor:*',
|
|
376
|
+
'clusterManagerQueue:*',
|
|
377
|
+
'clusterManagerLock:*',
|
|
378
|
+
'clusterManagerCacheMgr:*',
|
|
304
379
|
'workerOrchestrator:*',
|
|
305
380
|
'orchestratorStacks:*',
|
|
306
381
|
'workerPackages:*',
|
|
307
382
|
'clusterManagerPlugins:*',
|
|
308
383
|
],
|
|
309
384
|
});
|
|
310
|
-
|
|
311
|
-
// ── Container Orchestrator ──
|
|
312
|
-
await this.initOrchestrator();
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
private registerPubSubAdapter() {
|
|
316
|
-
const url = process.env.PUBSUB_ADAPTER_REDIS_URL;
|
|
317
|
-
if (!url) {
|
|
318
|
-
this.app.logger.info('[cluster-manager] PUBSUB_ADAPTER_REDIS_URL not set, skipping Redis PubSub adapter');
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
// Don't override if another plugin already set an adapter
|
|
323
|
-
const existingAdapter = (this.app.pubSubManager as any).adapter;
|
|
324
|
-
if (existingAdapter) {
|
|
325
|
-
this.app.logger.info('[cluster-manager] PubSub adapter already registered, skipping');
|
|
326
|
-
return;
|
|
327
|
-
}
|
|
328
|
-
|
|
329
|
-
const adapter = new RedisPubSubAdapter(url, this.app.logger);
|
|
330
|
-
this.app.pubSubManager.setAdapter(adapter);
|
|
331
|
-
this.app.logger.info('[cluster-manager] Redis PubSub adapter registered');
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
/**
|
|
335
|
-
* Initialize the Container Orchestrator subsystem.
|
|
336
|
-
* Config is loaded from DB (orchestratorSettings collection) first,
|
|
337
|
-
* then falls back to ORCHESTRATOR_ADAPTER env var.
|
|
338
|
-
* This allows manual configuration via the NocoBase admin UI.
|
|
339
|
-
*/
|
|
340
|
-
private async initOrchestrator() {
|
|
341
|
-
// Always register actions + collections so the UI can configure settings
|
|
342
|
-
// even before an adapter is connected
|
|
343
|
-
this.app.resourcer.define({
|
|
344
|
-
name: 'workerOrchestrator',
|
|
345
|
-
actions: {
|
|
346
|
-
...orchestratorActions,
|
|
347
|
-
// Settings CRUD: read/write orchestrator config from DB
|
|
348
|
-
async getSettings(ctx: any, next: () => Promise<void>) {
|
|
349
|
-
try {
|
|
350
|
-
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
351
|
-
let settings = await repo.findOne();
|
|
352
|
-
if (!settings) {
|
|
353
|
-
settings = await repo.create({
|
|
354
|
-
values: { adapterType: process.env.ORCHESTRATOR_ADAPTER || 'none' },
|
|
355
|
-
});
|
|
356
|
-
}
|
|
357
|
-
ctx.body = settings.toJSON();
|
|
358
|
-
} catch {
|
|
359
|
-
// Table may not exist during migration window
|
|
360
|
-
ctx.body = { adapterType: 'none', _note: 'Settings table not yet ready.' };
|
|
361
|
-
}
|
|
362
|
-
await next();
|
|
363
|
-
},
|
|
364
|
-
async saveSettings(ctx: any, next: () => Promise<void>) {
|
|
365
|
-
const values = ctx.action.params.values || {};
|
|
366
|
-
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
367
|
-
let settings = await repo.findOne();
|
|
368
|
-
if (settings) {
|
|
369
|
-
await repo.update({ filterByTk: settings.get('id'), values });
|
|
370
|
-
} else {
|
|
371
|
-
settings = await repo.create({ values });
|
|
372
|
-
}
|
|
373
|
-
// Reinitialize adapter with new settings
|
|
374
|
-
const plugin = ctx.app.pm.get('plugin-cluster-manager') as PluginClusterManagerServer;
|
|
375
|
-
await plugin.connectAdapter(values);
|
|
376
|
-
ctx.body = { success: true, message: 'Settings saved. Adapter reinitialized.' };
|
|
377
|
-
await next();
|
|
378
|
-
},
|
|
379
|
-
},
|
|
380
|
-
});
|
|
381
|
-
|
|
382
|
-
// Load settings from DB and try to connect
|
|
383
|
-
try {
|
|
384
|
-
const repo = this.app.db.getRepository('orchestratorSettings');
|
|
385
|
-
const settings = await repo.findOne();
|
|
386
|
-
if (settings) {
|
|
387
|
-
await this.connectAdapter(settings.toJSON());
|
|
388
|
-
} else {
|
|
389
|
-
// Fall back to env var for initial setup
|
|
390
|
-
const envAdapter = process.env.ORCHESTRATOR_ADAPTER;
|
|
385
|
+
|
|
386
|
+
// ── Container Orchestrator ──
|
|
387
|
+
await this.initOrchestrator();
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
private registerPubSubAdapter() {
|
|
391
|
+
const url = process.env.PUBSUB_ADAPTER_REDIS_URL;
|
|
392
|
+
if (!url) {
|
|
393
|
+
this.app.logger.info('[cluster-manager] PUBSUB_ADAPTER_REDIS_URL not set, skipping Redis PubSub adapter');
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
|
|
397
|
+
// Don't override if another plugin already set an adapter
|
|
398
|
+
const existingAdapter = (this.app.pubSubManager as any).adapter;
|
|
399
|
+
if (existingAdapter) {
|
|
400
|
+
this.app.logger.info('[cluster-manager] PubSub adapter already registered, skipping');
|
|
401
|
+
return;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
const adapter = new RedisPubSubAdapter(url, this.app.logger);
|
|
405
|
+
this.app.pubSubManager.setAdapter(adapter);
|
|
406
|
+
this.app.logger.info('[cluster-manager] Redis PubSub adapter registered');
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/**
|
|
410
|
+
* Initialize the Container Orchestrator subsystem.
|
|
411
|
+
* Config is loaded from DB (orchestratorSettings collection) first,
|
|
412
|
+
* then falls back to ORCHESTRATOR_ADAPTER env var.
|
|
413
|
+
* This allows manual configuration via the NocoBase admin UI.
|
|
414
|
+
*/
|
|
415
|
+
private async initOrchestrator() {
|
|
416
|
+
// Always register actions + collections so the UI can configure settings
|
|
417
|
+
// even before an adapter is connected
|
|
418
|
+
this.app.resourcer.define({
|
|
419
|
+
name: 'workerOrchestrator',
|
|
420
|
+
actions: {
|
|
421
|
+
...orchestratorActions,
|
|
422
|
+
// Settings CRUD: read/write orchestrator config from DB
|
|
423
|
+
async getSettings(ctx: any, next: () => Promise<void>) {
|
|
424
|
+
try {
|
|
425
|
+
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
426
|
+
let settings = await repo.findOne();
|
|
427
|
+
if (!settings) {
|
|
428
|
+
settings = await repo.create({
|
|
429
|
+
values: { adapterType: process.env.ORCHESTRATOR_ADAPTER || 'none' },
|
|
430
|
+
});
|
|
431
|
+
}
|
|
432
|
+
ctx.body = settings.toJSON();
|
|
433
|
+
} catch {
|
|
434
|
+
// Table may not exist during migration window
|
|
435
|
+
ctx.body = { adapterType: 'none', _note: 'Settings table not yet ready.' };
|
|
436
|
+
}
|
|
437
|
+
await next();
|
|
438
|
+
},
|
|
439
|
+
async saveSettings(ctx: any, next: () => Promise<void>) {
|
|
440
|
+
const values = ctx.action.params.values || {};
|
|
441
|
+
const repo = ctx.db.getRepository('orchestratorSettings');
|
|
442
|
+
let settings = await repo.findOne();
|
|
443
|
+
if (settings) {
|
|
444
|
+
await repo.update({ filterByTk: settings.get('id'), values });
|
|
445
|
+
} else {
|
|
446
|
+
settings = await repo.create({ values });
|
|
447
|
+
}
|
|
448
|
+
// Reinitialize adapter with new settings
|
|
449
|
+
const plugin = ctx.app.pm.get('plugin-cluster-manager') as PluginClusterManagerServer;
|
|
450
|
+
await plugin.connectAdapter(values);
|
|
451
|
+
ctx.body = { success: true, message: 'Settings saved. Adapter reinitialized.' };
|
|
452
|
+
await next();
|
|
453
|
+
},
|
|
454
|
+
},
|
|
455
|
+
});
|
|
456
|
+
|
|
457
|
+
// Load settings from DB and try to connect
|
|
458
|
+
try {
|
|
459
|
+
const repo = this.app.db.getRepository('orchestratorSettings');
|
|
460
|
+
const settings = await repo.findOne();
|
|
461
|
+
if (settings) {
|
|
462
|
+
await this.connectAdapter(settings.toJSON());
|
|
463
|
+
} else {
|
|
464
|
+
// Fall back to env var for initial setup
|
|
465
|
+
const envAdapter = process.env.ORCHESTRATOR_ADAPTER;
|
|
391
466
|
if (envAdapter && envAdapter !== 'none') {
|
|
392
467
|
await this.connectAdapter({
|
|
393
468
|
adapterType: envAdapter,
|
|
@@ -398,10 +473,10 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
398
473
|
this.app.logger.info('[Orchestrator] No adapter configured — configurable via Cluster Manager UI');
|
|
399
474
|
}
|
|
400
475
|
}
|
|
401
|
-
} catch (err: any) {
|
|
402
|
-
this.app.logger.warn(`[Orchestrator] Could not load settings: ${err.message}`);
|
|
403
|
-
}
|
|
404
|
-
|
|
476
|
+
} catch (err: any) {
|
|
477
|
+
this.app.logger.warn(`[Orchestrator] Could not load settings: ${err.message}`);
|
|
478
|
+
}
|
|
479
|
+
|
|
405
480
|
// Leader election runs on app nodes only. Worker-only pods still load the
|
|
406
481
|
// plugin for monitoring/package installation, but they must not become the
|
|
407
482
|
// Kubernetes orchestrator leader.
|
|
@@ -411,41 +486,41 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
411
486
|
disabledReason: workerOnlyNode ? 'Worker-only nodes do not run orchestrator write operations.' : '',
|
|
412
487
|
});
|
|
413
488
|
await this.leaderElection.init();
|
|
414
|
-
|
|
415
|
-
(this.app as any).on('afterStart', async () => {
|
|
416
|
-
if (this.leaderElection) {
|
|
417
|
-
await this.leaderElection.tryBecomeLeader();
|
|
418
|
-
}
|
|
419
|
-
});
|
|
420
|
-
|
|
421
|
-
(this.app as any).on('beforeStop', async () => {
|
|
422
|
-
if (this.leaderElection) {
|
|
423
|
-
await this.leaderElection.release();
|
|
424
|
-
}
|
|
425
|
-
});
|
|
426
|
-
}
|
|
427
|
-
|
|
428
|
-
/**
|
|
429
|
-
* Connect (or reconnect) the orchestrator adapter based on settings.
|
|
430
|
-
* Can be called at startup or when user saves new settings via UI.
|
|
431
|
-
*/
|
|
489
|
+
|
|
490
|
+
(this.app as any).on('afterStart', async () => {
|
|
491
|
+
if (this.leaderElection) {
|
|
492
|
+
await this.leaderElection.tryBecomeLeader();
|
|
493
|
+
}
|
|
494
|
+
});
|
|
495
|
+
|
|
496
|
+
(this.app as any).on('beforeStop', async () => {
|
|
497
|
+
if (this.leaderElection) {
|
|
498
|
+
await this.leaderElection.release();
|
|
499
|
+
}
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* Connect (or reconnect) the orchestrator adapter based on settings.
|
|
505
|
+
* Can be called at startup or when user saves new settings via UI.
|
|
506
|
+
*/
|
|
432
507
|
public async connectAdapter(settings: any): Promise<boolean> {
|
|
433
|
-
const adapterType = settings?.adapterType || 'none';
|
|
434
|
-
|
|
435
|
-
if (adapterType === 'none') {
|
|
436
|
-
this.orchestrator = null;
|
|
437
|
-
this.app.logger.info('[Orchestrator] Adapter disabled');
|
|
438
|
-
return false;
|
|
439
|
-
}
|
|
440
|
-
|
|
441
|
-
try {
|
|
442
|
-
if (adapterType === 'docker') {
|
|
443
|
-
const opts: any = {};
|
|
508
|
+
const adapterType = settings?.adapterType || 'none';
|
|
509
|
+
|
|
510
|
+
if (adapterType === 'none') {
|
|
511
|
+
this.orchestrator = null;
|
|
512
|
+
this.app.logger.info('[Orchestrator] Adapter disabled');
|
|
513
|
+
return false;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
try {
|
|
517
|
+
if (adapterType === 'docker') {
|
|
518
|
+
const opts: any = {};
|
|
444
519
|
if (settings.dockerHost) {
|
|
445
|
-
// TCP connection (remote Docker or Docker Desktop on Windows)
|
|
446
|
-
const url = new URL(settings.dockerHost);
|
|
447
|
-
opts.host = url.hostname;
|
|
448
|
-
opts.port = parseInt(url.port, 10) || 2376;
|
|
520
|
+
// TCP connection (remote Docker or Docker Desktop on Windows)
|
|
521
|
+
const url = new URL(settings.dockerHost);
|
|
522
|
+
opts.host = url.hostname;
|
|
523
|
+
opts.port = parseInt(url.port, 10) || 2376;
|
|
449
524
|
} else {
|
|
450
525
|
opts.socketPath = settings.dockerSocketPath || '/var/run/docker.sock';
|
|
451
526
|
}
|
|
@@ -458,28 +533,28 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
458
533
|
namespace: settings.k8sNamespace || 'nocobase',
|
|
459
534
|
workerLabelSelector: settings.workerLabelSelector || 'role=worker',
|
|
460
535
|
});
|
|
461
|
-
this.app.logger.info('[Orchestrator] Kubernetes adapter initialized');
|
|
462
|
-
} else {
|
|
463
|
-
this.app.logger.warn(`[Orchestrator] Unknown adapter type: ${adapterType}`);
|
|
464
|
-
this.orchestrator = null;
|
|
465
|
-
return false;
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// Test connectivity
|
|
469
|
-
const connected = await this.orchestrator.ping();
|
|
470
|
-
if (!connected) {
|
|
471
|
-
this.app.logger.error(`[Orchestrator] Failed to connect to ${adapterType} runtime`);
|
|
472
|
-
this.orchestrator = null;
|
|
473
|
-
return false;
|
|
474
|
-
}
|
|
475
|
-
|
|
476
|
-
this.app.logger.info(`[Orchestrator] ✅ Connected to ${adapterType} runtime`);
|
|
477
|
-
return true;
|
|
478
|
-
} catch (err: any) {
|
|
479
|
-
this.app.logger.error(`[Orchestrator] Adapter init failed: ${err.message}`);
|
|
480
|
-
this.orchestrator = null;
|
|
481
|
-
return false;
|
|
482
|
-
}
|
|
536
|
+
this.app.logger.info('[Orchestrator] Kubernetes adapter initialized');
|
|
537
|
+
} else {
|
|
538
|
+
this.app.logger.warn(`[Orchestrator] Unknown adapter type: ${adapterType}`);
|
|
539
|
+
this.orchestrator = null;
|
|
540
|
+
return false;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
// Test connectivity
|
|
544
|
+
const connected = await this.orchestrator.ping();
|
|
545
|
+
if (!connected) {
|
|
546
|
+
this.app.logger.error(`[Orchestrator] Failed to connect to ${adapterType} runtime`);
|
|
547
|
+
this.orchestrator = null;
|
|
548
|
+
return false;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
this.app.logger.info(`[Orchestrator] ✅ Connected to ${adapterType} runtime`);
|
|
552
|
+
return true;
|
|
553
|
+
} catch (err: any) {
|
|
554
|
+
this.app.logger.error(`[Orchestrator] Adapter init failed: ${err.message}`);
|
|
555
|
+
this.orchestrator = null;
|
|
556
|
+
return false;
|
|
557
|
+
}
|
|
483
558
|
}
|
|
484
559
|
|
|
485
560
|
private isWorkerOnlyNode(): boolean {
|
|
@@ -487,5 +562,5 @@ export class PluginClusterManagerServer extends Plugin {
|
|
|
487
562
|
return workerMode === 'worker' || workerMode === 'task' || workerMode === '*';
|
|
488
563
|
}
|
|
489
564
|
}
|
|
490
|
-
|
|
491
|
-
export default PluginClusterManagerServer;
|
|
565
|
+
|
|
566
|
+
export default PluginClusterManagerServer;
|