plugin-cluster-manager 1.1.16 → 1.1.17

package/src/server/adapters/redis-event-queue-adapter.ts

@@ -0,0 +1,189 @@
+import type { Application, IEventQueueAdapter, QueueEventOptions, QueueMessageOptions } from '@nocobase/server';
+import { createClient } from 'redis';
+import { randomUUID } from 'crypto';
+import { workerModeServesProcess } from '../../shared/worker-processes';
+
+const DEFAULT_INTERVAL_MS = 250;
+const DEFAULT_CONCURRENCY = 1;
+const DEFAULT_ACK_TIMEOUT_MS = 15_000;
+const REDIS_QUEUE_PREFIX = 'nocobase:event-queue';
+
+type RedisClient = ReturnType<typeof createClient>;
+
+type StoredMessage = {
+  id: string;
+  content: unknown;
+  options?: QueueMessageOptions;
+};
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function createTimeoutSignal(timeout: number): AbortSignal {
+  if (typeof AbortSignal !== 'undefined' && typeof AbortSignal.timeout === 'function') {
+    return AbortSignal.timeout(timeout);
+  }
+  const controller = new AbortController();
+  setTimeout(() => controller.abort(), timeout);
+  return controller.signal;
+}
+
+export class RedisEventQueueAdapter implements IEventQueueAdapter {
+  private client: RedisClient;
+  private connected = false;
+  private events = new Map<string, QueueEventOptions>();
+  private reading = new Map<string, Set<Promise<void>>>();
+  private consuming = new Set<string>();
+
+  constructor(
+    private readonly options: {
+      app: Application;
+      url: string;
+    },
+  ) {
+    this.client = createClient({ url: options.url });
+    this.client.on('error', (error) => {
+      this.options.app.logger.error(`[RedisEventQueueAdapter] Redis error: ${error.message}`);
+    });
+  }
+
+  isConnected(): boolean {
+    return this.connected;
+  }
+
+  async connect(): Promise<void> {
+    if (this.connected) return;
+    if (!this.client.isOpen) {
+      await this.client.connect();
+    }
+    this.connected = true;
+    for (const channel of this.events.keys()) {
+      this.startConsumer(channel);
+    }
+    this.options.app.logger.info('[RedisEventQueueAdapter] Connected');
+  }
+
+  async close(): Promise<void> {
+    this.connected = false;
+    const batches = Array.from(this.reading.values()).flatMap((items) => Array.from(items));
+    if (batches.length) {
+      await Promise.allSettled(batches);
+    }
+    if (this.client.isOpen) {
+      await this.client.quit().catch(() => this.client.disconnect());
+    }
+    this.options.app.logger.info('[RedisEventQueueAdapter] Closed');
+  }
+
+  subscribe(channel: string, event: QueueEventOptions): void {
+    this.events.set(channel, event);
+    if (this.connected) {
+      this.startConsumer(channel);
+    }
+  }
+
+  unsubscribe(channel: string): void {
+    this.events.delete(channel);
+  }
+
+  async publish(channel: string, content: unknown, options: QueueMessageOptions = {}): Promise<void> {
+    if (!this.connected) {
+      throw new Error('redis event queue is not connected');
+    }
+    const message: StoredMessage = {
+      id: randomUUID(),
+      content,
+      options: {
+        ...options,
+        timestamp: Date.now(),
+      },
+    };
+    await this.client.rPush(this.getQueueKey(channel), JSON.stringify(message));
+  }
+
+  private getQueueKey(channel: string) {
+    return `${REDIS_QUEUE_PREFIX}:${channel}`;
+  }
+
+  private startConsumer(channel: string) {
+    if (this.consuming.has(channel)) return;
+    this.consuming.add(channel);
+    return this.consume(channel)
+      .catch((error) => {
+        this.options.app.logger.error(`[RedisEventQueueAdapter] Consumer failed for ${channel}: ${error.message}`);
+      })
+      .finally(() => {
+        this.consuming.delete(channel);
+        if (this.connected && this.events.has(channel)) {
+          this.startConsumer(channel);
+        }
+      });
+  }
+
+  private async consume(channel: string) {
+    while (this.connected && this.events.has(channel)) {
+      const event = this.events.get(channel);
+      if (event && this.canProcess(channel, event)) {
+        this.read(channel, event);
+      }
+      await sleep(event?.interval || DEFAULT_INTERVAL_MS);
+    }
+  }
+
+  private canProcess(channel: string, event: QueueEventOptions) {
+    if (!workerModeServesProcess(process.env.WORKER_MODE, channel)) {
+      return false;
+    }
+    return event.idle();
+  }
+
+  private read(channel: string, event: QueueEventOptions) {
+    const active = this.reading.get(channel) || new Set<Promise<void>>();
+    this.reading.set(channel, active);
+
+    const available = (event.concurrency || DEFAULT_CONCURRENCY) - active.size;
+    for (let index = 0; index < available; index += 1) {
+      const promise = this.readOne(channel, event).finally(() => active.delete(promise));
+      active.add(promise);
+    }
+  }
+
+  private async readOne(channel: string, event: QueueEventOptions) {
+    const raw = await this.client.lPop(this.getQueueKey(channel));
+    if (!raw) return;
+
+    let message: StoredMessage;
+    try {
+      message = JSON.parse(raw);
+    } catch (error) {
+      this.options.app.logger.warn(`[RedisEventQueueAdapter] Dropped invalid message from ${channel}`, error);
+      return;
+    }
+
+    await this.process(channel, event, message);
+  }
+
+  private async process(channel: string, event: QueueEventOptions, message: StoredMessage) {
+    const { timeout = DEFAULT_ACK_TIMEOUT_MS, maxRetries = 0, retried = 0 } = message.options || {};
+    try {
+      await event.process(message.content, {
+        id: message.id,
+        retried,
+        signal: createTimeoutSignal(timeout),
+        queueOptions: message.options,
+      });
+    } catch (error) {
+      if (maxRetries > 0 && retried < maxRetries) {
+        await this.publish(channel, message.content, {
+          ...message.options,
+          timeout,
+          maxRetries,
+          retried: retried + 1,
+        });
+        return;
+      }
+      this.options.app.logger.error(`[RedisEventQueueAdapter] Message failed on ${channel}`, error);
+    }
+  }
+}

package/src/server/adapters/redis-node-registry.ts

@@ -1,5 +1,5 @@
 import os from 'os';
-import { scanKeys, getRedisClient } from '../utils/redis';
+import { scanKeys, getRedisClient, isClusterRedisConfigured } from '../utils/redis';
 import { getLocalNodeId } from '../utils/node';
 
 export class RedisNodeRegistry {
@@ -7,6 +7,10 @@ export class RedisNodeRegistry {
   private readonly ttlSecs = 30; // 30 seconds TTL
   private readonly intervalMs = 10000; // Heartbeat every 10 seconds
   private readonly keyPrefix = 'cluster-manager:nodes:';
+  private warnedMissingRedis = false;
+  private lastHeartbeatAt: number | null = null;
+  private lastHeartbeatError: string | null = null;
+  private lastReadError: string | null = null;
 
   constructor(private app: any) {}
 
@@ -33,7 +37,16 @@ export class RedisNodeRegistry {
 
   private async heartbeat() {
     const redis = getRedisClient(this.app);
-    if (!redis) return;
+    if (!redis) {
+      this.lastHeartbeatError = 'Redis is not configured for cluster node discovery';
+      if (!this.warnedMissingRedis) {
+        this.warnedMissingRedis = true;
+        this.app.logger.warn(
+          '[RedisNodeRegistry] Redis is not configured; Cluster Nodes can only show the local fallback node.',
+        );
+      }
+      return;
+    }
 
     // Unique identifier combining hostname, port, pid, mode, and appName to handle multiple workers on the same host
     const port = process.env.APP_PORT || 'unknown';
@@ -69,6 +82,8 @@ export class RedisNodeRegistry {
           arch: process.arch,
           uptime: process.uptime(),
           workerMode: mode,
+          appRole: process.env.APP_ROLE || '',
+          isSandbox: process.env.SKILL_HUB_SANDBOX === 'true',
           appPort: port,
           clusterMode: process.env.CLUSTER_MODE || '',
         },
@@ -90,18 +105,27 @@ export class RedisNodeRegistry {
 
     try {
       await redis.sendCommand(['SET', key, JSON.stringify(metadata), 'EX', this.ttlSecs.toString()]);
+      this.lastHeartbeatAt = Date.now();
+      this.lastHeartbeatError = null;
     } catch (err: any) {
+      this.lastHeartbeatError = err.message;
       this.app.logger.error(`[RedisNodeRegistry] Heartbeat failed: ${err.message}`);
     }
   }
 
   public async getNodes(): Promise<any[]> {
     const redis = getRedisClient(this.app);
-    if (!redis) return [];
+    if (!redis) {
+      this.lastReadError = 'Redis is not configured for cluster node discovery';
+      return [];
+    }
 
     try {
       const rawKeys = await scanKeys(redis, `${this.keyPrefix}*`);
-      if (rawKeys.length === 0) return [];
+      if (rawKeys.length === 0) {
+        this.lastReadError = null;
+        return [];
+      }
 
       const values = await redis.sendCommand(['MGET', ...rawKeys]);
 
@@ -117,10 +141,26 @@ export class RedisNodeRegistry {
           }
         }
       }
+      this.lastReadError = null;
       return nodes;
     } catch (err: any) {
+      this.lastReadError = err.message;
       this.app.logger.error(`[RedisNodeRegistry] Error fetching nodes: ${err.message}`);
       return [];
     }
   }
+
+  public getStatus() {
+    const redis = getRedisClient(this.app);
+    return {
+      configured: isClusterRedisConfigured(this.app),
+      connected: Boolean(redis),
+      keyPrefix: this.keyPrefix,
+      ttlSecs: this.ttlSecs,
+      intervalMs: this.intervalMs,
+      lastHeartbeatAt: this.lastHeartbeatAt,
+      lastHeartbeatError: this.lastHeartbeatError,
+      lastReadError: this.lastReadError,
+    };
+  }
 }

package/src/server/collections/orchestrator-stacks.ts

@@ -44,6 +44,12 @@ export default {
       interface: 'json',
       uiSchema: { title: 'Environment Variables' },
     },
+    {
+      name: 'workerMode',
+      type: 'string',
+      interface: 'input',
+      uiSchema: { title: 'Worker Mode', 'x-component': 'Input' },
+    },
     {
       name: 'volumes',
       type: 'json',

package/src/server/collections/worker-queue-mappings.ts

@@ -5,8 +5,8 @@
  * When a stack has assigned queues, the orchestrator adapter sets
  * WORKER_MODE=<comma-separated-queue-names> on new containers.
  *
- * If no mappings exist for a stack, WORKER_MODE=* is preserved
- * (backwards compatibility).
+ * Explicit orchestratorStacks.workerMode takes precedence. If no explicit
+ * mode exists, mappings can still provide WORKER_MODE for legacy stacks.
  */
 export default {
   name: 'workerQueueMappings',
@@ -67,7 +67,7 @@ export default {
         'x-component': 'Select',
         'x-component-props': {
           allowClear: true,
-          placeholder: 'Unassigned (worker runs all queues)',
+          placeholder: 'Unassigned (fallback only)',
         },
       },
     },

package/src/server/orchestrator/PackageManager.ts

@@ -8,6 +8,17 @@ import Application from '@nocobase/server';
 /** Allow only safe package name characters: letters, digits, dash, underscore, dot, @, /, [, ] */
 const SAFE_PKG_RE = /^(?:[a-zA-Z0-9_.@/-]|\[|\])+$/;
 const INSTALL_CHANNEL = 'cluster-manager.install-packages';
+const APT_ENV: NodeJS.ProcessEnv = {
+  DEBIAN_FRONTEND: 'noninteractive',
+  DEBCONF_NONINTERACTIVE_SEEN: 'true',
+  APT_LISTCHANGES_FRONTEND: 'none',
+  TERM: 'dumb',
+  NEEDRESTART_MODE: 'a',
+  UCF_FORCE_CONFOLD: '1',
+  UCF_FORCE_CONFFOLD: '1',
+};
+const APT_DPKG_OPTIONS = ['-o', 'Dpkg::Options::=--force-confdef', '-o', 'Dpkg::Options::=--force-confold'];
+const DPKG_CONFIGURE_OPTIONS = ['--force-confdef', '--force-confold', '--configure', '-a'];
 
 type TargetRole = 'app' | 'worker' | 'sandbox' | 'all';
 type PackageKind = 'apt' | 'npm' | 'python';
@@ -164,22 +175,27 @@ export class PackageManager {
       };
 
       // Step 1: APT packages
-      if (missingPackages.apt.length > 0) {
+      if (safePackages.apt.length > 0) {
         if (registryConfig.aptMirrorUrl) {
           const aptMirrorUrl = sanitizeHttpUrl(registryConfig.aptMirrorUrl, 'APT mirror URL');
           logs.push(`Applying APT mirror: ${redactUrl(aptMirrorUrl)}`);
           await this.configureAptMirror(aptMirrorUrl, logs);
         }
 
-        await this.updateInstallStatus('running', 20, 'Installing APT packages...', logs);
-        await this.runCommand('apt-get', ['update', '-qq'], 'Updating APT package index...', logs, 1200000);
-        await this.runCommand(
-          'apt-get',
-          ['install', '-y', '--no-install-recommends', ...missingPackages.apt],
-          'Installing APT packages...',
-          logs,
-          1200000,
-        );
+        await this.updateInstallStatus('running', 20, 'Preparing APT/dpkg...', logs);
+        await this.repairAptState(logs);
+        if (missingPackages.apt.length > 0) {
+          await this.updateInstallStatus('running', 25, 'Installing APT packages...', logs);
+          await this.runCommand('apt-get', ['update', '-qq'], 'Updating APT package index...', logs, 1200000, APT_ENV);
+          await this.runCommand(
+            'apt-get',
+            [...APT_DPKG_OPTIONS, 'install', '-y', '--no-install-recommends', ...missingPackages.apt],
+            'Installing APT packages...',
+            logs,
+            1200000,
+            APT_ENV,
+          );
+        }
       }
 
       // Step 2: NPM packages (Global)
@@ -277,12 +293,13 @@ export class PackageManager {
     label: string,
     logs: string[],
     timeoutMs = 1200000,
+    env?: NodeJS.ProcessEnv,
   ): Promise<void> {
     logs.push(`RUNNING: ${formatCommand(command, args)}`);
     logs.push(`${label}`);
 
     await new Promise<void>((resolve, reject) => {
-      const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+      const child = spawn(command, args, { stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, ...env } });
       let stdout = '';
       let stderr = '';
       let settled = false;
@@ -324,6 +341,26 @@ export class PackageManager {
     });
   }
 
+  private async repairAptState(logs: string[]): Promise<void> {
+    logs.push('Preparing APT/dpkg in non-interactive mode...');
+    await this.runCommand(
+      'dpkg',
+      DPKG_CONFIGURE_OPTIONS,
+      'Repairing interrupted dpkg configuration...',
+      logs,
+      1200000,
+      APT_ENV,
+    );
+    await this.runCommand(
+      'apt-get',
+      [...APT_DPKG_OPTIONS, '-f', 'install', '-y', '--no-install-recommends'],
+      'Repairing incomplete APT dependencies...',
+      logs,
+      1200000,
+      APT_ENV,
+    );
+  }
+
   private async getMissingPackages(kind: PackageKind, packages: string[], logs: string[]): Promise<string[]> {
     const missing: string[] = [];
     for (const pkg of packages) {

package/src/server/orchestrator/types.ts

@@ -24,8 +24,8 @@ export interface ScaleResult {
 }
 
 export interface ContainerStats {
-  cpu: number;       // percentage 0-100
-  memory: number;    // bytes
+  cpu: number; // percentage 0-100
+  memory: number; // bytes
   memoryLimit: number;
   networkRx: number; // bytes
   networkTx: number; // bytes
@@ -38,11 +38,12 @@ export interface StackConfig {
   image: string;
   command?: string;
   envVars?: Record<string, string>;
+  workerMode?: string;
   volumes?: string[];
   networks?: string[];
   resourceLimits?: {
-    memory?: string;  // "1536Mi" | "2Gi"
-    cpu?: string;     // "1" | "500m"
+    memory?: string; // "1536Mi" | "2Gi"
+    cpu?: string; // "1" | "500m"
   };
   replicas: number;
   desiredReplicas: number;

package/src/server/plugin.ts

@@ -7,11 +7,12 @@ import { redisActions } from './actions/redis-monitor';
 import { aclCacheActions, createAclCacheMiddleware } from './actions/acl-cache';
 import { clusterActions, readLocalLogs } from './actions/cluster-nodes';
 import { getRedisClient } from './utils/redis';
-import { getLocalNodeId, isWorkerMode } from './utils/node';
+import { getLocalNodeId, getLocalRole, isWorkerMode } from './utils/node';
 import { eventQueueActions } from './actions/event-queue-monitor';
 import { lockActions } from './actions/lock-monitor';
 import { cacheMonitorActions } from './actions/cache-monitor';
 import { RedisPubSubAdapter } from './adapters/redis-pubsub-adapter';
+import { RedisEventQueueAdapter } from './adapters/redis-event-queue-adapter';
 import { RedisNodeRegistry } from './adapters/redis-node-registry';
 import { RedisLockAdapter } from './adapters/redis-lock-adapter';
 import { orchestratorActions } from './actions/orchestrator';
@@ -137,6 +138,7 @@ export class PluginClusterManagerServer extends Plugin {
 
     // Register Redis PubSub adapter if URL is configured and no adapter already set
     this.registerPubSubAdapter();
+    await this.registerEventQueueAdapter();
 
     // Register missing Redis Lock adapter if running bare open-source core
     const lockMgr = this.app.lockManager as any;
@@ -410,6 +412,38 @@ export class PluginClusterManagerServer extends Plugin {
     this.app.logger.info('[cluster-manager] Redis PubSub adapter registered');
   }
 
+  private async registerEventQueueAdapter() {
+    const enabled = process.env.QUEUE_ADAPTER === 'redis' || Boolean(process.env.QUEUE_ADAPTER_REDIS_URL);
+    if (!enabled) {
+      return;
+    }
+
+    const url = process.env.QUEUE_ADAPTER_REDIS_URL || process.env.REDIS_URL;
+    if (!url) {
+      this.app.logger.warn('[cluster-manager] QUEUE_ADAPTER=redis but QUEUE_ADAPTER_REDIS_URL/REDIS_URL is not set');
+      return;
+    }
+
+    const eventQueue = this.app.eventQueue as any;
+    const existingAdapter = eventQueue?.adapter;
+    const existingName = existingAdapter?.constructor?.name;
+    if (existingAdapter && existingName !== 'MemoryEventQueueAdapter') {
+      this.app.logger.info(`[cluster-manager] EventQueue adapter already registered (${existingName}), skipping`);
+      return;
+    }
+
+    const adapter = new RedisEventQueueAdapter({ app: this.app, url });
+    const wasConnected = Boolean(eventQueue?.isConnected?.());
+    if (wasConnected) {
+      await eventQueue.close();
+    }
+    eventQueue.setAdapter(adapter);
+    if (wasConnected) {
+      await eventQueue.connect();
+    }
+    this.app.logger.info('[cluster-manager] Redis EventQueue adapter registered');
+  }
+
   /**
    * Initialize the Container Orchestrator subsystem.
    * Config is loaded from DB (orchestratorSettings collection) first,
@@ -484,10 +518,10 @@ export class PluginClusterManagerServer extends Plugin {
     // Leader election runs on app nodes only. Worker-only pods still load the
     // plugin for monitoring/package installation, but they must not become the
     // Kubernetes orchestrator leader.
-    const workerOnlyNode = this.isWorkerOnlyNode();
+    const nonAppNode = this.isNonAppNode();
     this.leaderElection = new LeaderElection(this.app, {
-      enabled: !workerOnlyNode,
-      disabledReason: workerOnlyNode ? 'Worker-only nodes do not run orchestrator write operations.' : '',
+      enabled: !nonAppNode,
+      disabledReason: nonAppNode ? 'Non-app nodes do not run orchestrator write operations.' : '',
     });
     await this.leaderElection.init();
 
@@ -561,8 +595,8 @@ export class PluginClusterManagerServer extends Plugin {
     }
   }
 
-  private isWorkerOnlyNode(): boolean {
-    return isWorkerMode(process.env.WORKER_MODE);
+  private isNonAppNode(): boolean {
+    return getLocalRole() !== 'app';
   }
 }