plugin-cluster-manager 1.1.15 → 1.1.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client/index.js +1 -1
- package/dist/client-v2/376.cd1d86e85a50088e.js +10 -0
- package/dist/client-v2/index.js +1 -1
- package/dist/externalVersion.js +6 -6
- package/dist/locale/en-US.json +16 -6
- package/dist/locale/vi-VN.json +16 -6
- package/dist/locale/zh-CN.json +16 -6
- package/dist/server/actions/cluster-nodes.js +44 -11
- package/dist/server/actions/doctor.js +73 -7
- package/dist/server/actions/event-queue-monitor.js +33 -3
- package/dist/server/actions/orchestrator.js +48 -32
- package/dist/server/actions/queue-mappings.js +1 -0
- package/dist/server/actions/tasks.js +8 -8
- package/dist/server/adapters/redis-event-queue-adapter.js +188 -0
- package/dist/server/adapters/redis-node-registry.js +44 -10
- package/dist/server/collections/orchestrator-stacks.js +6 -0
- package/dist/server/collections/worker-queue-mappings.js +1 -1
- package/dist/server/orchestrator/PackageManager.js +47 -12
- package/dist/server/plugin.js +37 -6
- package/dist/server/queue-scanner.js +54 -34
- package/dist/server/utils/node.js +3 -7
- package/dist/server/utils/redis.js +37 -9
- package/dist/shared/worker-processes.js +233 -0
- package/package.json +1 -1
- package/src/client/ClusterNodes.tsx +76 -10
- package/src/client/ContainerOrchestrator.tsx +146 -8
- package/src/client/QueueAssignment.tsx +10 -2
- package/src/locale/en-US.json +16 -6
- package/src/locale/vi-VN.json +16 -6
- package/src/locale/zh-CN.json +16 -6
- package/src/server/__tests__/worker-processes.test.ts +42 -0
- package/src/server/actions/cluster-nodes.ts +43 -8
- package/src/server/actions/doctor.ts +77 -0
- package/src/server/actions/event-queue-monitor.ts +34 -3
- package/src/server/actions/orchestrator.ts +58 -38
- package/src/server/actions/queue-mappings.ts +1 -0
- package/src/server/actions/tasks.ts +142 -142
- package/src/server/adapters/redis-event-queue-adapter.ts +189 -0
- package/src/server/adapters/redis-node-registry.ts +44 -4
- package/src/server/collections/orchestrator-stacks.ts +6 -0
- package/src/server/collections/worker-queue-mappings.ts +3 -3
- package/src/server/orchestrator/PackageManager.ts +48 -11
- package/src/server/orchestrator/types.ts +5 -4
- package/src/server/plugin.ts +40 -6
- package/src/server/queue-scanner.ts +65 -51
- package/src/server/utils/node.ts +3 -10
- package/src/server/utils/redis.ts +39 -4
- package/src/shared/worker-processes.ts +216 -0
- package/dist/client-v2/914.c0bce51908fd81d7.js +0 -10
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This file is part of the NocoBase (R) project.
|
|
3
|
+
* Copyright (c) 2020-2024 NocoBase Co., Ltd.
|
|
4
|
+
* Authors: NocoBase Team.
|
|
5
|
+
*
|
|
6
|
+
* This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
|
|
7
|
+
* For more information, please refer to: https://www.nocobase.com/agreement.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
var __defProp = Object.defineProperty;
|
|
11
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
12
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
13
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
14
|
+
var __export = (target, all) => {
|
|
15
|
+
for (var name in all)
|
|
16
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
17
|
+
};
|
|
18
|
+
var __copyProps = (to, from, except, desc) => {
|
|
19
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
20
|
+
for (let key of __getOwnPropNames(from))
|
|
21
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
22
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
23
|
+
}
|
|
24
|
+
return to;
|
|
25
|
+
};
|
|
26
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
27
|
+
var redis_event_queue_adapter_exports = {};
|
|
28
|
+
__export(redis_event_queue_adapter_exports, {
|
|
29
|
+
RedisEventQueueAdapter: () => RedisEventQueueAdapter
|
|
30
|
+
});
|
|
31
|
+
module.exports = __toCommonJS(redis_event_queue_adapter_exports);
|
|
32
|
+
var import_redis = require("redis");
|
|
33
|
+
var import_crypto = require("crypto");
|
|
34
|
+
var import_worker_processes = require("../../shared/worker-processes");
|
|
35
|
+
const DEFAULT_INTERVAL_MS = 250;
|
|
36
|
+
const DEFAULT_CONCURRENCY = 1;
|
|
37
|
+
const DEFAULT_ACK_TIMEOUT_MS = 15e3;
|
|
38
|
+
const REDIS_QUEUE_PREFIX = "nocobase:event-queue";
|
|
39
|
+
function sleep(ms) {
|
|
40
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
41
|
+
}
|
|
42
|
+
function createTimeoutSignal(timeout) {
|
|
43
|
+
if (typeof AbortSignal !== "undefined" && typeof AbortSignal.timeout === "function") {
|
|
44
|
+
return AbortSignal.timeout(timeout);
|
|
45
|
+
}
|
|
46
|
+
const controller = new AbortController();
|
|
47
|
+
setTimeout(() => controller.abort(), timeout);
|
|
48
|
+
return controller.signal;
|
|
49
|
+
}
|
|
50
|
+
class RedisEventQueueAdapter {
|
|
51
|
+
constructor(options) {
|
|
52
|
+
this.options = options;
|
|
53
|
+
this.client = (0, import_redis.createClient)({ url: options.url });
|
|
54
|
+
this.client.on("error", (error) => {
|
|
55
|
+
this.options.app.logger.error(`[RedisEventQueueAdapter] Redis error: ${error.message}`);
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
client;
|
|
59
|
+
connected = false;
|
|
60
|
+
events = /* @__PURE__ */ new Map();
|
|
61
|
+
reading = /* @__PURE__ */ new Map();
|
|
62
|
+
consuming = /* @__PURE__ */ new Set();
|
|
63
|
+
isConnected() {
|
|
64
|
+
return this.connected;
|
|
65
|
+
}
|
|
66
|
+
async connect() {
|
|
67
|
+
if (this.connected) return;
|
|
68
|
+
if (!this.client.isOpen) {
|
|
69
|
+
await this.client.connect();
|
|
70
|
+
}
|
|
71
|
+
this.connected = true;
|
|
72
|
+
for (const channel of this.events.keys()) {
|
|
73
|
+
this.startConsumer(channel);
|
|
74
|
+
}
|
|
75
|
+
this.options.app.logger.info("[RedisEventQueueAdapter] Connected");
|
|
76
|
+
}
|
|
77
|
+
async close() {
|
|
78
|
+
this.connected = false;
|
|
79
|
+
const batches = Array.from(this.reading.values()).flatMap((items) => Array.from(items));
|
|
80
|
+
if (batches.length) {
|
|
81
|
+
await Promise.allSettled(batches);
|
|
82
|
+
}
|
|
83
|
+
if (this.client.isOpen) {
|
|
84
|
+
await this.client.quit().catch(() => this.client.disconnect());
|
|
85
|
+
}
|
|
86
|
+
this.options.app.logger.info("[RedisEventQueueAdapter] Closed");
|
|
87
|
+
}
|
|
88
|
+
subscribe(channel, event) {
|
|
89
|
+
this.events.set(channel, event);
|
|
90
|
+
if (this.connected) {
|
|
91
|
+
this.startConsumer(channel);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
unsubscribe(channel) {
|
|
95
|
+
this.events.delete(channel);
|
|
96
|
+
}
|
|
97
|
+
async publish(channel, content, options = {}) {
|
|
98
|
+
if (!this.connected) {
|
|
99
|
+
throw new Error("redis event queue is not connected");
|
|
100
|
+
}
|
|
101
|
+
const message = {
|
|
102
|
+
id: (0, import_crypto.randomUUID)(),
|
|
103
|
+
content,
|
|
104
|
+
options: {
|
|
105
|
+
...options,
|
|
106
|
+
timestamp: Date.now()
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
await this.client.rPush(this.getQueueKey(channel), JSON.stringify(message));
|
|
110
|
+
}
|
|
111
|
+
getQueueKey(channel) {
|
|
112
|
+
return `${REDIS_QUEUE_PREFIX}:${channel}`;
|
|
113
|
+
}
|
|
114
|
+
startConsumer(channel) {
|
|
115
|
+
if (this.consuming.has(channel)) return;
|
|
116
|
+
this.consuming.add(channel);
|
|
117
|
+
return this.consume(channel).catch((error) => {
|
|
118
|
+
this.options.app.logger.error(`[RedisEventQueueAdapter] Consumer failed for ${channel}: ${error.message}`);
|
|
119
|
+
}).finally(() => {
|
|
120
|
+
this.consuming.delete(channel);
|
|
121
|
+
if (this.connected && this.events.has(channel)) {
|
|
122
|
+
this.startConsumer(channel);
|
|
123
|
+
}
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
async consume(channel) {
|
|
127
|
+
while (this.connected && this.events.has(channel)) {
|
|
128
|
+
const event = this.events.get(channel);
|
|
129
|
+
if (event && this.canProcess(channel, event)) {
|
|
130
|
+
this.read(channel, event);
|
|
131
|
+
}
|
|
132
|
+
await sleep((event == null ? void 0 : event.interval) || DEFAULT_INTERVAL_MS);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
canProcess(channel, event) {
|
|
136
|
+
if (!(0, import_worker_processes.workerModeServesProcess)(process.env.WORKER_MODE, channel)) {
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
return event.idle();
|
|
140
|
+
}
|
|
141
|
+
read(channel, event) {
|
|
142
|
+
const active = this.reading.get(channel) || /* @__PURE__ */ new Set();
|
|
143
|
+
this.reading.set(channel, active);
|
|
144
|
+
const available = (event.concurrency || DEFAULT_CONCURRENCY) - active.size;
|
|
145
|
+
for (let index = 0; index < available; index += 1) {
|
|
146
|
+
const promise = this.readOne(channel, event).finally(() => active.delete(promise));
|
|
147
|
+
active.add(promise);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
async readOne(channel, event) {
|
|
151
|
+
const raw = await this.client.lPop(this.getQueueKey(channel));
|
|
152
|
+
if (!raw) return;
|
|
153
|
+
let message;
|
|
154
|
+
try {
|
|
155
|
+
message = JSON.parse(raw);
|
|
156
|
+
} catch (error) {
|
|
157
|
+
this.options.app.logger.warn(`[RedisEventQueueAdapter] Dropped invalid message from ${channel}`, error);
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
await this.process(channel, event, message);
|
|
161
|
+
}
|
|
162
|
+
async process(channel, event, message) {
|
|
163
|
+
const { timeout = DEFAULT_ACK_TIMEOUT_MS, maxRetries = 0, retried = 0 } = message.options || {};
|
|
164
|
+
try {
|
|
165
|
+
await event.process(message.content, {
|
|
166
|
+
id: message.id,
|
|
167
|
+
retried,
|
|
168
|
+
signal: createTimeoutSignal(timeout),
|
|
169
|
+
queueOptions: message.options
|
|
170
|
+
});
|
|
171
|
+
} catch (error) {
|
|
172
|
+
if (maxRetries > 0 && retried < maxRetries) {
|
|
173
|
+
await this.publish(channel, message.content, {
|
|
174
|
+
...message.options,
|
|
175
|
+
timeout,
|
|
176
|
+
maxRetries,
|
|
177
|
+
retried: retried + 1
|
|
178
|
+
});
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
this.options.app.logger.error(`[RedisEventQueueAdapter] Message failed on ${channel}`, error);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
186
|
+
0 && (module.exports = {
|
|
187
|
+
RedisEventQueueAdapter
|
|
188
|
+
});
|
|
@@ -52,6 +52,10 @@ class RedisNodeRegistry {
|
|
|
52
52
|
intervalMs = 1e4;
|
|
53
53
|
// Heartbeat every 10 seconds
|
|
54
54
|
keyPrefix = "cluster-manager:nodes:";
|
|
55
|
+
warnedMissingRedis = false;
|
|
56
|
+
lastHeartbeatAt = null;
|
|
57
|
+
lastHeartbeatError = null;
|
|
58
|
+
lastReadError = null;
|
|
55
59
|
start() {
|
|
56
60
|
if (this.timer) {
|
|
57
61
|
clearInterval(this.timer);
|
|
@@ -69,7 +73,16 @@ class RedisNodeRegistry {
|
|
|
69
73
|
}
|
|
70
74
|
async heartbeat() {
|
|
71
75
|
const redis = (0, import_redis.getRedisClient)(this.app);
|
|
72
|
-
if (!redis)
|
|
76
|
+
if (!redis) {
|
|
77
|
+
this.lastHeartbeatError = "Redis is not configured for cluster node discovery";
|
|
78
|
+
if (!this.warnedMissingRedis) {
|
|
79
|
+
this.warnedMissingRedis = true;
|
|
80
|
+
this.app.logger.warn(
|
|
81
|
+
"[RedisNodeRegistry] Redis is not configured; Cluster Nodes can only show the local fallback node."
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
73
86
|
const port = process.env.APP_PORT || "unknown";
|
|
74
87
|
const mode = process.env.WORKER_MODE || "main";
|
|
75
88
|
const appName = process.env.APP_NAME || this.app.name || "main";
|
|
@@ -82,6 +95,7 @@ class RedisNodeRegistry {
|
|
|
82
95
|
hostname: import_os.default.hostname(),
|
|
83
96
|
appVersion: process.env.NOCOBASE_VERSION || process.version,
|
|
84
97
|
workerMode: mode,
|
|
98
|
+
appRole: process.env.APP_ROLE,
|
|
85
99
|
isSandbox: process.env.SKILL_HUB_SANDBOX === "true",
|
|
86
100
|
pid: process.pid,
|
|
87
101
|
url: process.env.APP_PUBLIC_URL || null,
|
|
@@ -100,6 +114,8 @@ class RedisNodeRegistry {
|
|
|
100
114
|
arch: process.arch,
|
|
101
115
|
uptime: process.uptime(),
|
|
102
116
|
workerMode: mode,
|
|
117
|
+
appRole: process.env.APP_ROLE || "",
|
|
118
|
+
isSandbox: process.env.SKILL_HUB_SANDBOX === "true",
|
|
103
119
|
appPort: port,
|
|
104
120
|
clusterMode: process.env.CLUSTER_MODE || ""
|
|
105
121
|
},
|
|
@@ -119,23 +135,26 @@ class RedisNodeRegistry {
|
|
|
119
135
|
}
|
|
120
136
|
};
|
|
121
137
|
try {
|
|
122
|
-
await redis.sendCommand([
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
JSON.stringify(metadata),
|
|
126
|
-
"EX",
|
|
127
|
-
this.ttlSecs.toString()
|
|
128
|
-
]);
|
|
138
|
+
await redis.sendCommand(["SET", key, JSON.stringify(metadata), "EX", this.ttlSecs.toString()]);
|
|
139
|
+
this.lastHeartbeatAt = Date.now();
|
|
140
|
+
this.lastHeartbeatError = null;
|
|
129
141
|
} catch (err) {
|
|
142
|
+
this.lastHeartbeatError = err.message;
|
|
130
143
|
this.app.logger.error(`[RedisNodeRegistry] Heartbeat failed: ${err.message}`);
|
|
131
144
|
}
|
|
132
145
|
}
|
|
133
146
|
async getNodes() {
|
|
134
147
|
const redis = (0, import_redis.getRedisClient)(this.app);
|
|
135
|
-
if (!redis)
|
|
148
|
+
if (!redis) {
|
|
149
|
+
this.lastReadError = "Redis is not configured for cluster node discovery";
|
|
150
|
+
return [];
|
|
151
|
+
}
|
|
136
152
|
try {
|
|
137
153
|
const rawKeys = await (0, import_redis.scanKeys)(redis, `${this.keyPrefix}*`);
|
|
138
|
-
if (rawKeys.length === 0)
|
|
154
|
+
if (rawKeys.length === 0) {
|
|
155
|
+
this.lastReadError = null;
|
|
156
|
+
return [];
|
|
157
|
+
}
|
|
139
158
|
const values = await redis.sendCommand(["MGET", ...rawKeys]);
|
|
140
159
|
const nodes = [];
|
|
141
160
|
if (Array.isArray(values)) {
|
|
@@ -148,12 +167,27 @@ class RedisNodeRegistry {
|
|
|
148
167
|
}
|
|
149
168
|
}
|
|
150
169
|
}
|
|
170
|
+
this.lastReadError = null;
|
|
151
171
|
return nodes;
|
|
152
172
|
} catch (err) {
|
|
173
|
+
this.lastReadError = err.message;
|
|
153
174
|
this.app.logger.error(`[RedisNodeRegistry] Error fetching nodes: ${err.message}`);
|
|
154
175
|
return [];
|
|
155
176
|
}
|
|
156
177
|
}
|
|
178
|
+
getStatus() {
|
|
179
|
+
const redis = (0, import_redis.getRedisClient)(this.app);
|
|
180
|
+
return {
|
|
181
|
+
configured: (0, import_redis.isClusterRedisConfigured)(this.app),
|
|
182
|
+
connected: Boolean(redis),
|
|
183
|
+
keyPrefix: this.keyPrefix,
|
|
184
|
+
ttlSecs: this.ttlSecs,
|
|
185
|
+
intervalMs: this.intervalMs,
|
|
186
|
+
lastHeartbeatAt: this.lastHeartbeatAt,
|
|
187
|
+
lastHeartbeatError: this.lastHeartbeatError,
|
|
188
|
+
lastReadError: this.lastReadError
|
|
189
|
+
};
|
|
190
|
+
}
|
|
157
191
|
}
|
|
158
192
|
// Annotate the CommonJS export names for ESM import in node:
|
|
159
193
|
0 && (module.exports = {
|
|
@@ -75,6 +75,12 @@ var orchestrator_stacks_default = {
|
|
|
75
75
|
interface: "json",
|
|
76
76
|
uiSchema: { title: "Environment Variables" }
|
|
77
77
|
},
|
|
78
|
+
{
|
|
79
|
+
name: "workerMode",
|
|
80
|
+
type: "string",
|
|
81
|
+
interface: "input",
|
|
82
|
+
uiSchema: { title: "Worker Mode", "x-component": "Input" }
|
|
83
|
+
},
|
|
78
84
|
{
|
|
79
85
|
name: "volumes",
|
|
80
86
|
type: "json",
|
|
@@ -46,6 +46,17 @@ var import_fs = require("fs");
|
|
|
46
46
|
var import_path = __toESM(require("path"));
|
|
47
47
|
const SAFE_PKG_RE = /^(?:[a-zA-Z0-9_.@/-]|\[|\])+$/;
|
|
48
48
|
const INSTALL_CHANNEL = "cluster-manager.install-packages";
|
|
49
|
+
const APT_ENV = {
|
|
50
|
+
DEBIAN_FRONTEND: "noninteractive",
|
|
51
|
+
DEBCONF_NONINTERACTIVE_SEEN: "true",
|
|
52
|
+
APT_LISTCHANGES_FRONTEND: "none",
|
|
53
|
+
TERM: "dumb",
|
|
54
|
+
NEEDRESTART_MODE: "a",
|
|
55
|
+
UCF_FORCE_CONFOLD: "1",
|
|
56
|
+
UCF_FORCE_CONFFOLD: "1"
|
|
57
|
+
};
|
|
58
|
+
const APT_DPKG_OPTIONS = ["-o", "Dpkg::Options::=--force-confdef", "-o", "Dpkg::Options::=--force-confold"];
|
|
59
|
+
const DPKG_CONFIGURE_OPTIONS = ["--force-confdef", "--force-confold", "--configure", "-a"];
|
|
49
60
|
function sanitizePkg(name) {
|
|
50
61
|
if (typeof name !== "string") {
|
|
51
62
|
throw new Error("Package name must be a string");
|
|
@@ -162,21 +173,26 @@ class PackageManager {
|
|
|
162
173
|
npm: await this.getMissingPackages("npm", safePackages.npm, logs),
|
|
163
174
|
python: await this.getMissingPackages("python", safePackages.python, logs)
|
|
164
175
|
};
|
|
165
|
-
if (
|
|
176
|
+
if (safePackages.apt.length > 0) {
|
|
166
177
|
if (registryConfig.aptMirrorUrl) {
|
|
167
178
|
const aptMirrorUrl = sanitizeHttpUrl(registryConfig.aptMirrorUrl, "APT mirror URL");
|
|
168
179
|
logs.push(`Applying APT mirror: ${redactUrl(aptMirrorUrl)}`);
|
|
169
180
|
await this.configureAptMirror(aptMirrorUrl, logs);
|
|
170
181
|
}
|
|
171
|
-
await this.updateInstallStatus("running", 20, "
|
|
172
|
-
await this.
|
|
173
|
-
|
|
174
|
-
"
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
182
|
+
await this.updateInstallStatus("running", 20, "Preparing APT/dpkg...", logs);
|
|
183
|
+
await this.repairAptState(logs);
|
|
184
|
+
if (missingPackages.apt.length > 0) {
|
|
185
|
+
await this.updateInstallStatus("running", 25, "Installing APT packages...", logs);
|
|
186
|
+
await this.runCommand("apt-get", ["update", "-qq"], "Updating APT package index...", logs, 12e5, APT_ENV);
|
|
187
|
+
await this.runCommand(
|
|
188
|
+
"apt-get",
|
|
189
|
+
[...APT_DPKG_OPTIONS, "install", "-y", "--no-install-recommends", ...missingPackages.apt],
|
|
190
|
+
"Installing APT packages...",
|
|
191
|
+
logs,
|
|
192
|
+
12e5,
|
|
193
|
+
APT_ENV
|
|
194
|
+
);
|
|
195
|
+
}
|
|
180
196
|
}
|
|
181
197
|
if (missingPackages.npm.length > 0) {
|
|
182
198
|
if (registryConfig.npmRegistryUrl) {
|
|
@@ -259,12 +275,12 @@ ${logs.join("\n")}`);
|
|
|
259
275
|
/**
|
|
260
276
|
* Run a command without a shell so registry URLs and package names are not re-parsed as shell syntax.
|
|
261
277
|
*/
|
|
262
|
-
async runCommand(command, args, label, logs, timeoutMs = 12e5) {
|
|
278
|
+
async runCommand(command, args, label, logs, timeoutMs = 12e5, env) {
|
|
263
279
|
logs.push(`RUNNING: ${formatCommand(command, args)}`);
|
|
264
280
|
logs.push(`${label}`);
|
|
265
281
|
await new Promise((resolve, reject) => {
|
|
266
282
|
var _a, _b;
|
|
267
|
-
const child = (0, import_child_process.spawn)(command, args, { stdio: ["ignore", "pipe", "pipe"] });
|
|
283
|
+
const child = (0, import_child_process.spawn)(command, args, { stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, ...env } });
|
|
268
284
|
let stdout = "";
|
|
269
285
|
let stderr = "";
|
|
270
286
|
let settled = false;
|
|
@@ -302,6 +318,25 @@ ${logs.join("\n")}`);
|
|
|
302
318
|
});
|
|
303
319
|
});
|
|
304
320
|
}
|
|
321
|
+
async repairAptState(logs) {
|
|
322
|
+
logs.push("Preparing APT/dpkg in non-interactive mode...");
|
|
323
|
+
await this.runCommand(
|
|
324
|
+
"dpkg",
|
|
325
|
+
DPKG_CONFIGURE_OPTIONS,
|
|
326
|
+
"Repairing interrupted dpkg configuration...",
|
|
327
|
+
logs,
|
|
328
|
+
12e5,
|
|
329
|
+
APT_ENV
|
|
330
|
+
);
|
|
331
|
+
await this.runCommand(
|
|
332
|
+
"apt-get",
|
|
333
|
+
[...APT_DPKG_OPTIONS, "-f", "install", "-y", "--no-install-recommends"],
|
|
334
|
+
"Repairing incomplete APT dependencies...",
|
|
335
|
+
logs,
|
|
336
|
+
12e5,
|
|
337
|
+
APT_ENV
|
|
338
|
+
);
|
|
339
|
+
}
|
|
305
340
|
async getMissingPackages(kind, packages, logs) {
|
|
306
341
|
const missing = [];
|
|
307
342
|
for (const pkg of packages) {
|
package/dist/server/plugin.js
CHANGED
|
@@ -54,6 +54,7 @@ var import_event_queue_monitor = require("./actions/event-queue-monitor");
|
|
|
54
54
|
var import_lock_monitor = require("./actions/lock-monitor");
|
|
55
55
|
var import_cache_monitor = require("./actions/cache-monitor");
|
|
56
56
|
var import_redis_pubsub_adapter = require("./adapters/redis-pubsub-adapter");
|
|
57
|
+
var import_redis_event_queue_adapter = require("./adapters/redis-event-queue-adapter");
|
|
57
58
|
var import_redis_node_registry = require("./adapters/redis-node-registry");
|
|
58
59
|
var import_redis_lock_adapter = require("./adapters/redis-lock-adapter");
|
|
59
60
|
var import_orchestrator = require("./actions/orchestrator");
|
|
@@ -88,7 +89,7 @@ class PluginClusterManagerServer extends import_server.Plugin {
|
|
|
88
89
|
this.app.on("afterStart", () => {
|
|
89
90
|
var _a;
|
|
90
91
|
(_a = this.nodeRegistry) == null ? void 0 : _a.start();
|
|
91
|
-
const isWorker = (0, import_node.isWorkerMode)(process.env.WORKER_MODE) || process.env.APP_ROLE === "worker" || process.env.APP_ROLE === "sandbox";
|
|
92
|
+
const isWorker = (0, import_node.isWorkerMode)(process.env.WORKER_MODE) || process.env.APP_ROLE === "worker" || process.env.APP_ROLE === "sandbox" || process.env.SKILL_HUB_SANDBOX === "true";
|
|
92
93
|
if (isWorker) {
|
|
93
94
|
setTimeout(async () => {
|
|
94
95
|
try {
|
|
@@ -153,6 +154,7 @@ class PluginClusterManagerServer extends import_server.Plugin {
|
|
|
153
154
|
}
|
|
154
155
|
});
|
|
155
156
|
this.registerPubSubAdapter();
|
|
157
|
+
await this.registerEventQueueAdapter();
|
|
156
158
|
const lockMgr = this.app.lockManager;
|
|
157
159
|
if (lockMgr && lockMgr.registry && !lockMgr.registry.get("redis") && !lockMgr.adapters.get("redis")) {
|
|
158
160
|
lockMgr.registerAdapter("redis", {
|
|
@@ -368,6 +370,35 @@ class PluginClusterManagerServer extends import_server.Plugin {
|
|
|
368
370
|
this.app.pubSubManager.setAdapter(adapter);
|
|
369
371
|
this.app.logger.info("[cluster-manager] Redis PubSub adapter registered");
|
|
370
372
|
}
|
|
373
|
+
async registerEventQueueAdapter() {
|
|
374
|
+
var _a, _b;
|
|
375
|
+
const enabled = process.env.QUEUE_ADAPTER === "redis" || Boolean(process.env.QUEUE_ADAPTER_REDIS_URL);
|
|
376
|
+
if (!enabled) {
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
const url = process.env.QUEUE_ADAPTER_REDIS_URL || process.env.REDIS_URL;
|
|
380
|
+
if (!url) {
|
|
381
|
+
this.app.logger.warn("[cluster-manager] QUEUE_ADAPTER=redis but QUEUE_ADAPTER_REDIS_URL/REDIS_URL is not set");
|
|
382
|
+
return;
|
|
383
|
+
}
|
|
384
|
+
const eventQueue = this.app.eventQueue;
|
|
385
|
+
const existingAdapter = eventQueue == null ? void 0 : eventQueue.adapter;
|
|
386
|
+
const existingName = (_a = existingAdapter == null ? void 0 : existingAdapter.constructor) == null ? void 0 : _a.name;
|
|
387
|
+
if (existingAdapter && existingName !== "MemoryEventQueueAdapter") {
|
|
388
|
+
this.app.logger.info(`[cluster-manager] EventQueue adapter already registered (${existingName}), skipping`);
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
const adapter = new import_redis_event_queue_adapter.RedisEventQueueAdapter({ app: this.app, url });
|
|
392
|
+
const wasConnected = Boolean((_b = eventQueue == null ? void 0 : eventQueue.isConnected) == null ? void 0 : _b.call(eventQueue));
|
|
393
|
+
if (wasConnected) {
|
|
394
|
+
await eventQueue.close();
|
|
395
|
+
}
|
|
396
|
+
eventQueue.setAdapter(adapter);
|
|
397
|
+
if (wasConnected) {
|
|
398
|
+
await eventQueue.connect();
|
|
399
|
+
}
|
|
400
|
+
this.app.logger.info("[cluster-manager] Redis EventQueue adapter registered");
|
|
401
|
+
}
|
|
371
402
|
/**
|
|
372
403
|
* Initialize the Container Orchestrator subsystem.
|
|
373
404
|
* Config is loaded from DB (orchestratorSettings collection) first,
|
|
@@ -431,10 +462,10 @@ class PluginClusterManagerServer extends import_server.Plugin {
|
|
|
431
462
|
} catch (err) {
|
|
432
463
|
this.app.logger.warn(`[Orchestrator] Could not load settings: ${err.message}`);
|
|
433
464
|
}
|
|
434
|
-
const
|
|
465
|
+
const nonAppNode = this.isNonAppNode();
|
|
435
466
|
this.leaderElection = new import_leader_election.LeaderElection(this.app, {
|
|
436
|
-
enabled: !
|
|
437
|
-
disabledReason:
|
|
467
|
+
enabled: !nonAppNode,
|
|
468
|
+
disabledReason: nonAppNode ? "Non-app nodes do not run orchestrator write operations." : ""
|
|
438
469
|
});
|
|
439
470
|
await this.leaderElection.init();
|
|
440
471
|
this.app.on("afterStart", async () => {
|
|
@@ -498,8 +529,8 @@ class PluginClusterManagerServer extends import_server.Plugin {
|
|
|
498
529
|
return false;
|
|
499
530
|
}
|
|
500
531
|
}
|
|
501
|
-
|
|
502
|
-
return (0, import_node.
|
|
532
|
+
isNonAppNode() {
|
|
533
|
+
return (0, import_node.getLocalRole)() !== "app";
|
|
503
534
|
}
|
|
504
535
|
}
|
|
505
536
|
var plugin_default = PluginClusterManagerServer;
|
|
@@ -29,35 +29,24 @@ __export(queue_scanner_exports, {
|
|
|
29
29
|
scanQueues: () => scanQueues
|
|
30
30
|
});
|
|
31
31
|
module.exports = __toCommonJS(queue_scanner_exports);
|
|
32
|
+
var import_worker_processes = require("../shared/worker-processes");
|
|
32
33
|
var import_redis = require("./utils/redis");
|
|
33
|
-
const
|
|
34
|
-
"
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
label: "Async Tasks",
|
|
40
|
-
description: "Execute async tasks (plugin-async-task-manager)"
|
|
41
|
-
},
|
|
42
|
-
"knowledge-base:document-vectorize": {
|
|
43
|
-
label: "Document Vectorization",
|
|
44
|
-
description: "Vectorize knowledge base documents (plugin-knowledge-base)"
|
|
45
|
-
},
|
|
46
|
-
"git-review:process": {
|
|
47
|
-
label: "Git Review",
|
|
48
|
-
description: "AI code review jobs (plugin-git-manager)"
|
|
49
|
-
},
|
|
50
|
-
"build-guide:process": {
|
|
51
|
-
label: "Build Guide",
|
|
52
|
-
description: "Build user guide pages (plugin-build-guide-block)"
|
|
53
|
-
},
|
|
54
|
-
"build-ui-template:process": {
|
|
55
|
-
label: "Build UI Template",
|
|
56
|
-
description: "Build UI template pages (plugin-build-ui-template)"
|
|
57
|
-
}
|
|
58
|
-
};
|
|
59
|
-
const REDIS_QUEUE_PATTERNS = ["*:plugin-git-manager:review:queue", "*:plugin-build-guide-block:build:queue"];
|
|
34
|
+
const REDIS_QUEUE_PATTERNS = [
|
|
35
|
+
"*:plugin-git-manager:review:queue",
|
|
36
|
+
"*:plugin-build-guide-block:build:queue",
|
|
37
|
+
"*:plugin-build-visualization-block:build:queue",
|
|
38
|
+
"file-preview-auth.ocr.queue"
|
|
39
|
+
];
|
|
60
40
|
function describeRedisQueueKey(key) {
|
|
41
|
+
const workerProcessName = (0, import_worker_processes.resolveWorkerProcessName)(key);
|
|
42
|
+
const definition = (0, import_worker_processes.getWorkerProcessDefinition)(workerProcessName);
|
|
43
|
+
if (definition) {
|
|
44
|
+
return {
|
|
45
|
+
label: definition.label,
|
|
46
|
+
description: definition.description,
|
|
47
|
+
workerProcessName: definition.name
|
|
48
|
+
};
|
|
49
|
+
}
|
|
61
50
|
const parts = String(key).split(":");
|
|
62
51
|
const plugin = parts[parts.length - 3] || "unknown";
|
|
63
52
|
const queue = parts[parts.length - 2] || key;
|
|
@@ -68,23 +57,46 @@ function describeRedisQueueKey(key) {
|
|
|
68
57
|
}
|
|
69
58
|
function scanEventQueue(app) {
|
|
70
59
|
const eq = app.eventQueue;
|
|
71
|
-
if (!eq
|
|
60
|
+
if (!(eq == null ? void 0 : eq.events)) return [];
|
|
72
61
|
const events = eq.events;
|
|
73
62
|
const items = [];
|
|
74
63
|
for (const [channel] of events.entries()) {
|
|
75
|
-
const
|
|
64
|
+
const workerProcessName = (0, import_worker_processes.resolveWorkerProcessName)(channel);
|
|
65
|
+
const known = (0, import_worker_processes.getWorkerProcessDefinition)(workerProcessName);
|
|
76
66
|
items.push({
|
|
77
67
|
name: channel,
|
|
78
68
|
label: (known == null ? void 0 : known.label) ?? channel,
|
|
79
69
|
description: (known == null ? void 0 : known.description) ?? `EventQueue channel: ${channel}`,
|
|
80
70
|
type: "event-queue",
|
|
81
|
-
pending: null
|
|
71
|
+
pending: null,
|
|
72
|
+
workerProcessName: known == null ? void 0 : known.name
|
|
82
73
|
});
|
|
83
74
|
}
|
|
84
75
|
return items;
|
|
85
76
|
}
|
|
77
|
+
function scanKnownWorkerModes(app) {
|
|
78
|
+
const pluginManager = app.pm;
|
|
79
|
+
if (!(pluginManager == null ? void 0 : pluginManager.get)) return [];
|
|
80
|
+
const hasPlugin = (name) => {
|
|
81
|
+
var _a;
|
|
82
|
+
try {
|
|
83
|
+
return Boolean((_a = pluginManager.get) == null ? void 0 : _a.call(pluginManager, name));
|
|
84
|
+
} catch {
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
return import_worker_processes.WORKER_PROCESS_DEFINITIONS.filter(
|
|
89
|
+
(definition) => definition.common && !definition.sandbox && (!definition.pluginName || hasPlugin(definition.pluginName))
|
|
90
|
+
).map((definition) => ({
|
|
91
|
+
name: definition.name,
|
|
92
|
+
label: definition.label,
|
|
93
|
+
description: definition.description,
|
|
94
|
+
type: definition.kind === "redis-list" ? "redis-list" : "event-queue",
|
|
95
|
+
pending: null,
|
|
96
|
+
workerProcessName: definition.name
|
|
97
|
+
}));
|
|
98
|
+
}
|
|
86
99
|
async function scanRedisQueues(app) {
|
|
87
|
-
var _a;
|
|
88
100
|
const redis = (0, import_redis.getRedisClient)(app);
|
|
89
101
|
if (!redis) {
|
|
90
102
|
return [];
|
|
@@ -93,8 +105,8 @@ async function scanRedisQueues(app) {
|
|
|
93
105
|
const items = [];
|
|
94
106
|
for (const pattern of REDIS_QUEUE_PATTERNS) {
|
|
95
107
|
try {
|
|
96
|
-
const
|
|
97
|
-
const keyList =
|
|
108
|
+
const result = await redis.sendCommand(["SCAN", "0", "MATCH", pattern, "COUNT", "200"]);
|
|
109
|
+
const keyList = Array.isArray(result == null ? void 0 : result[1]) ? result[1] : [];
|
|
98
110
|
for (const key of keyList) {
|
|
99
111
|
if (seen.has(key)) continue;
|
|
100
112
|
seen.add(key);
|
|
@@ -110,7 +122,8 @@ async function scanRedisQueues(app) {
|
|
|
110
122
|
label: desc.label,
|
|
111
123
|
description: desc.description,
|
|
112
124
|
type: "redis-list",
|
|
113
|
-
pending
|
|
125
|
+
pending,
|
|
126
|
+
workerProcessName: desc.workerProcessName
|
|
114
127
|
});
|
|
115
128
|
}
|
|
116
129
|
} catch {
|
|
@@ -120,6 +133,7 @@ async function scanRedisQueues(app) {
|
|
|
120
133
|
}
|
|
121
134
|
async function scanQueues(app) {
|
|
122
135
|
const eventQueues = scanEventQueue(app);
|
|
136
|
+
const knownWorkerModes = scanKnownWorkerModes(app);
|
|
123
137
|
const redisQueues = await scanRedisQueues(app);
|
|
124
138
|
const seenNames = /* @__PURE__ */ new Set();
|
|
125
139
|
const merged = [];
|
|
@@ -127,6 +141,12 @@ async function scanQueues(app) {
|
|
|
127
141
|
merged.push(q);
|
|
128
142
|
seenNames.add(q.name);
|
|
129
143
|
}
|
|
144
|
+
for (const q of knownWorkerModes) {
|
|
145
|
+
if (!seenNames.has(q.name)) {
|
|
146
|
+
merged.push(q);
|
|
147
|
+
seenNames.add(q.name);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
130
150
|
for (const q of redisQueues) {
|
|
131
151
|
if (!seenNames.has(q.name)) {
|
|
132
152
|
merged.push(q);
|
|
@@ -43,13 +43,9 @@ __export(node_exports, {
|
|
|
43
43
|
});
|
|
44
44
|
module.exports = __toCommonJS(node_exports);
|
|
45
45
|
var import_os = __toESM(require("os"));
|
|
46
|
+
var import_worker_processes = require("../../shared/worker-processes");
|
|
46
47
|
function isWorkerMode(workerMode) {
|
|
47
|
-
|
|
48
|
-
if (!mode || mode === "main" || mode === "app") return false;
|
|
49
|
-
if (mode === "-") return false;
|
|
50
|
-
const topics = mode.split(",").map((t) => t.trim()).filter(Boolean);
|
|
51
|
-
if (topics.includes("!")) return false;
|
|
52
|
-
return true;
|
|
48
|
+
return (0, import_worker_processes.isWorkerOnlyMode)(workerMode ?? process.env.WORKER_MODE);
|
|
53
49
|
}
|
|
54
50
|
function getNodeRoleFrom(opts) {
|
|
55
51
|
if (opts.appRole === "app" || opts.appRole === "worker" || opts.appRole === "sandbox") {
|
|
@@ -67,7 +63,7 @@ function getLocalRole() {
|
|
|
67
63
|
}
|
|
68
64
|
function getLocalNodeId(app) {
|
|
69
65
|
const port = process.env.APP_PORT || "unknown";
|
|
70
|
-
const mode = process.env.WORKER_MODE || "main";
|
|
66
|
+
const mode = (0, import_worker_processes.normalizeWorkerMode)(process.env.WORKER_MODE) || "main";
|
|
71
67
|
const appName = process.env.APP_NAME || (app == null ? void 0 : app.name) || "main";
|
|
72
68
|
return `${appName}_${mode}_${import_os.default.hostname()}_${port}_${process.pid}`;
|
|
73
69
|
}
|