plugin-cluster-manager 1.1.15 → 1.1.17

package/src/server/queue-scanner.ts

@@ -1,14 +1,15 @@
 /**
- * QueueScanner — discovers all registered queues in the system.
- *
- * Two sources:
- *   1. EventQueue events (registered via app.eventQueue.subscribe)
- *   2. Redis List-based queues (convention *:plugin-*:queue)
- *
- * Used by the Queue Assignment UI to let admins map queues to worker stacks.
+ * QueueScanner discovers process keys and queue aliases that can be assigned
+ * to worker stacks. The worker deployment should receive process keys in
+ * WORKER_MODE; physical EventQueue/Redis names are exposed as aliases only.
  */
 
 import type { Application } from '@nocobase/server';
+import {
+  getWorkerProcessDefinition,
+  resolveWorkerProcessName,
+  WORKER_PROCESS_DEFINITIONS,
+} from '../shared/worker-processes';
 import { getRedisClient } from './utils/redis';
 
 export type QueueItem = {
@@ -17,39 +18,27 @@ export type QueueItem = {
   description: string;
   type: 'event-queue' | 'redis-list';
   pending: number | null;
+  workerProcessName?: string;
 };
 
-const KNOWN_QUEUE_LABELS: Record<string, { label: string; description: string }> = {
-  'workflow:process': {
-    label: 'Workflow',
-    description: 'Process workflow executions (plugin-workflow)',
-  },
-  'async-task:process': {
-    label: 'Async Tasks',
-    description: 'Execute async tasks (plugin-async-task-manager)',
-  },
-  'knowledge-base:document-vectorize': {
-    label: 'Document Vectorization',
-    description: 'Vectorize knowledge base documents (plugin-knowledge-base)',
-  },
-  'git-review:process': {
-    label: 'Git Review',
-    description: 'AI code review jobs (plugin-git-manager)',
-  },
-  'build-guide:process': {
-    label: 'Build Guide',
-    description: 'Build user guide pages (plugin-build-guide-block)',
-  },
-  'build-ui-template:process': {
-    label: 'Build UI Template',
-    description: 'Build UI template pages (plugin-build-ui-template)',
-  },
-};
-
-/** Redis key patterns for List-based queues (same as event-queue-monitor.ts) */
-const REDIS_QUEUE_PATTERNS = ['*:plugin-git-manager:review:queue', '*:plugin-build-guide-block:build:queue'];
+const REDIS_QUEUE_PATTERNS = [
+  '*:plugin-git-manager:review:queue',
+  '*:plugin-build-guide-block:build:queue',
+  '*:plugin-build-visualization-block:build:queue',
+  'file-preview-auth.ocr.queue',
+];
+
+function describeRedisQueueKey(key: string): { label: string; description: string; workerProcessName?: string } {
+  const workerProcessName = resolveWorkerProcessName(key);
+  const definition = getWorkerProcessDefinition(workerProcessName);
+  if (definition) {
+    return {
+      label: definition.label,
+      description: definition.description,
+      workerProcessName: definition.name,
+    };
+  }
 
-function describeRedisQueueKey(key: string): { label: string; description: string } {
   const parts = String(key).split(':');
   const plugin = parts[parts.length - 3] || 'unknown';
   const queue = parts[parts.length - 2] || key;
@@ -59,33 +48,54 @@ function describeRedisQueueKey(key: string): { label: string; description: strin
   };
 }
 
-/**
- * Discover all queues from EventQueue subscribers.
- */
 function scanEventQueue(app: Application): QueueItem[] {
   const eq = (app as any).eventQueue;
-  if (!eq || !eq.events) return [];
+  if (!eq?.events) return [];
 
   const events: Map<string, { concurrency?: number; interval?: number; shared?: boolean }> = eq.events;
   const items: QueueItem[] = [];
 
   for (const [channel] of events.entries()) {
-    const known = KNOWN_QUEUE_LABELS[channel];
+    const workerProcessName = resolveWorkerProcessName(channel);
+    const known = getWorkerProcessDefinition(workerProcessName);
     items.push({
       name: channel,
       label: known?.label ?? channel,
       description: known?.description ?? `EventQueue channel: ${channel}`,
       type: 'event-queue',
       pending: null,
+      workerProcessName: known?.name,
     });
   }
 
   return items;
 }
 
-/**
- * Discover Redis List-based queues via SCAN.
- */
+function scanKnownWorkerModes(app: Application): QueueItem[] {
+  const pluginManager = (app as unknown as { pm?: { get?: (name: string) => unknown } }).pm;
+  if (!pluginManager?.get) return [];
+
+  const hasPlugin = (name: string) => {
+    try {
+      return Boolean(pluginManager.get?.(name));
+    } catch {
+      return false;
+    }
+  };
+
+  return WORKER_PROCESS_DEFINITIONS.filter(
+    (definition) =>
+      definition.common && !definition.sandbox && (!definition.pluginName || hasPlugin(definition.pluginName)),
+  ).map((definition) => ({
+    name: definition.name,
+    label: definition.label,
+    description: definition.description,
+    type: definition.kind === 'redis-list' ? 'redis-list' : ('event-queue' as const),
+    pending: null,
+    workerProcessName: definition.name,
+  }));
+}
+
 async function scanRedisQueues(app: Application): Promise<QueueItem[]> {
   const redis = getRedisClient(app);
   if (!redis) {
@@ -97,8 +107,8 @@ async function scanRedisQueues(app: Application): Promise<QueueItem[]> {
 
   for (const pattern of REDIS_QUEUE_PATTERNS) {
     try {
-      const keys: string[] = await redis.sendCommand(['SCAN', '0', 'MATCH', pattern, 'COUNT', '200']);
-      const keyList: string[] = typeof keys[1]?.length === 'number' ? keys[1] : [];
+      const result = await redis.sendCommand(['SCAN', '0', 'MATCH', pattern, 'COUNT', '200']);
+      const keyList: string[] = Array.isArray(result?.[1]) ? result[1] : [];
 
       for (const key of keyList) {
         if (seen.has(key)) continue;
@@ -118,6 +128,7 @@ async function scanRedisQueues(app: Application): Promise<QueueItem[]> {
           description: desc.description,
           type: 'redis-list',
           pending,
+          workerProcessName: desc.workerProcessName,
         });
       }
     } catch {
@@ -128,14 +139,11 @@ async function scanRedisQueues(app: Application): Promise<QueueItem[]> {
   return items;
 }
 
-/**
- * Full queue scan — merges EventQueue + Redis results.
- */
 export async function scanQueues(app: Application): Promise<{ queues: QueueItem[]; total: number }> {
   const eventQueues = scanEventQueue(app);
+  const knownWorkerModes = scanKnownWorkerModes(app);
   const redisQueues = await scanRedisQueues(app);
 
-  // Deduplicate: if a queue name appears in both sources, prefer EventQueue
   const seenNames = new Set<string>();
   const merged: QueueItem[] = [];
 
@@ -143,6 +151,12 @@ export async function scanQueues(app: Application): Promise<{ queues: QueueItem[
     merged.push(q);
     seenNames.add(q.name);
   }
+  for (const q of knownWorkerModes) {
+    if (!seenNames.has(q.name)) {
+      merged.push(q);
+      seenNames.add(q.name);
+    }
+  }
   for (const q of redisQueues) {
     if (!seenNames.has(q.name)) {
       merged.push(q);

package/src/server/utils/node.ts

@@ -1,4 +1,5 @@
 import os from 'os';
+import { isWorkerOnlyMode, normalizeWorkerMode } from '../../shared/worker-processes';
 
 export type NodeRole = 'app' | 'worker' | 'sandbox';
 
@@ -15,15 +16,7 @@ export type NodeRole = 'app' | 'worker' | 'sandbox';
  * HTTP, so it is treated as an app node.
  */
 export function isWorkerMode(workerMode?: string): boolean {
-  const mode = (workerMode ?? process.env.WORKER_MODE ?? '').trim();
-  if (!mode || mode === 'main' || mode === 'app') return false;
-  if (mode === '-') return false;
-  const topics = mode
-    .split(',')
-    .map((t) => t.trim())
-    .filter(Boolean);
-  if (topics.includes('!')) return false;
-  return true;
+  return isWorkerOnlyMode(workerMode ?? process.env.WORKER_MODE);
 }
 
 /**
@@ -55,7 +48,7 @@ export function getLocalRole(): NodeRole {
  */
 export function getLocalNodeId(app: any): string {
   const port = process.env.APP_PORT || 'unknown';
-  const mode = process.env.WORKER_MODE || 'main';
+  const mode = normalizeWorkerMode(process.env.WORKER_MODE) || 'main';
   const appName = process.env.APP_NAME || app?.name || 'main';
   return `${appName}_${mode}_${os.hostname()}_${port}_${process.pid}`;
 }

package/src/server/utils/redis.ts

@@ -3,15 +3,52 @@ import { createClient } from 'redis';
 
 let globalRedisClient: any = null;
 
+const CLUSTER_REDIS_CONNECTION = 'cluster-manager:registry';
+
+const CLUSTER_REDIS_ENV_CANDIDATES = [
+  'NODE_REGISTRY_REDIS_URL',
+  'CLUSTER_MANAGER_REDIS_URL',
+  'REDIS_URL',
+  'CACHE_REDIS_URL',
+  'PUBSUB_ADAPTER_REDIS_URL',
+  'QUEUE_ADAPTER_REDIS_URL',
+  'LOCK_ADAPTER_REDIS_URL',
+];
+
+export function getClusterRedisUrl() {
+  for (const key of CLUSTER_REDIS_ENV_CANDIDATES) {
+    const value = process.env[key];
+    if (value?.trim()) {
+      return value.trim();
+    }
+  }
+  return '';
+}
+
+export function isClusterRedisConfigured(app?: any) {
+  if (getClusterRedisUrl()) {
+    return true;
+  }
+
+  const manager = app?.redisConnectionManager;
+  return Boolean(manager?.getConnection?.());
+}
+
 export function getRedisClient(app?: any) {
   if (globalRedisClient) return globalRedisClient;
 
+  const url = getClusterRedisUrl();
+
   if (app?.redisConnectionManager) {
+    if (url) {
+      const conn = app.redisConnectionManager.getConnection(CLUSTER_REDIS_CONNECTION, { connectionString: url });
+      if (conn) return conn;
+    }
+
     const conn = app.redisConnectionManager.getConnection();
     if (conn) return conn;
   }
 
-  const url = process.env.REDIS_URL || process.env.CACHE_REDIS_URL || process.env.PUBSUB_ADAPTER_REDIS_URL;
   if (!url) return null;
 
   globalRedisClient = createClient({ url });
@@ -53,9 +90,7 @@ export async function scanKeys(redis: any, pattern: string, batchSize = 200): Pr
   let cursor = '0';
 
   do {
-    const result = await redis.sendCommand([
-      'SCAN', cursor, 'MATCH', pattern, 'COUNT', String(batchSize),
-    ]);
+    const result = await redis.sendCommand(['SCAN', cursor, 'MATCH', pattern, 'COUNT', String(batchSize)]);
 
     // result is [nextCursor, [...keys]]
     cursor = String(result[0]);

package/src/shared/worker-processes.ts

@@ -0,0 +1,216 @@
+export type WorkerProcessKind = 'event-queue' | 'redis-list' | 'pubsub';
+
+export type WorkerProcessDefinition = {
+  name: string;
+  label: string;
+  description: string;
+  kind: WorkerProcessKind;
+  aliases?: string[];
+  redisKeySuffixes?: string[];
+  pluginName?: string;
+  sandbox?: boolean;
+  common?: boolean;
+};
+
+export const WORKER_PROCESS_DEFINITIONS: WorkerProcessDefinition[] = [
+  {
+    name: 'workflow:process',
+    label: 'Workflow',
+    description: 'Process workflow executions',
+    kind: 'event-queue',
+    aliases: ['workflow.pendingExecution', '@nocobase/plugin-workflow.pendingExecution'],
+    common: true,
+  },
+  {
+    name: 'async-task:process',
+    label: 'Async Tasks',
+    description: 'Execute async tasks',
+    kind: 'event-queue',
+    aliases: ['async-task-manager.task', '@nocobase/plugin-async-task-manager.task'],
+    common: true,
+  },
+  {
+    name: 'notification:send',
+    label: 'Notifications',
+    description: 'Send notification messages',
+    kind: 'event-queue',
+    aliases: ['notification-manager.send', '@nocobase/plugin-notification-manager.send'],
+    common: true,
+  },
+  {
+    name: 'knowledge-base:document-vectorize',
+    label: 'Document Vectorization',
+    description: 'Vectorize knowledge base documents',
+    kind: 'event-queue',
+    aliases: ['knowledge-base:document-vectorize'],
+    pluginName: 'plugin-knowledge-base',
+    common: true,
+  },
+  {
+    name: 'git-review:process',
+    label: 'Git Review',
+    description: 'AI code review jobs',
+    kind: 'redis-list',
+    aliases: ['plugin-git-manager.review'],
+    redisKeySuffixes: [':plugin-git-manager:review:queue'],
+    pluginName: 'plugin-git-manager',
+    common: true,
+  },
+  {
+    name: 'build-guide:process',
+    label: 'Build Guide',
+    description: 'Build user guide pages',
+    kind: 'redis-list',
+    aliases: ['plugin-build-guide-block.build'],
+    redisKeySuffixes: [':plugin-build-guide-block:build:queue'],
+    pluginName: 'plugin-build-guide-block',
+    common: true,
+  },
+  {
+    name: 'build-visualization:process',
+    label: 'Build Visualization',
+    description: 'Build visualization blocks',
+    kind: 'redis-list',
+    aliases: ['plugin-build-visualization-block.build'],
+    redisKeySuffixes: [':plugin-build-visualization-block:build:queue'],
+    pluginName: 'plugin-build-visualization-block',
+    common: true,
+  },
+  {
+    name: 'build-ui-template:process',
+    label: 'Build UI Template',
+    description: 'Build UI template pages',
+    kind: 'event-queue',
+    aliases: ['plugin-build-ui-template.build'],
+    pluginName: 'plugin-build-ui-template',
+    common: true,
+  },
+  {
+    name: 'file-preview-auth:ocr',
+    label: 'File Preview OCR',
+    description: 'Run OCR extraction for authenticated file previews',
+    kind: 'redis-list',
+    aliases: ['file-preview-auth.ocr.queue'],
+    redisKeySuffixes: ['file-preview-auth.ocr.queue'],
+    pluginName: 'plugin-file-preview-auth',
+    common: true,
+  },
+  {
+    name: 'skill-hub:sandbox',
+    label: 'Skill Sandbox',
+    description: 'Execute Skill Hub sandbox tasks',
+    kind: 'pubsub',
+    aliases: ['skill-hub.task'],
+    pluginName: 'plugin-agent-orchestrator',
+    sandbox: true,
+  },
+];
+
+const PROCESS_BY_NAME = new Map(WORKER_PROCESS_DEFINITIONS.map((definition) => [definition.name, definition]));
+
+const ALIAS_TO_PROCESS = new Map<string, string>();
+for (const definition of WORKER_PROCESS_DEFINITIONS) {
+  ALIAS_TO_PROCESS.set(definition.name, definition.name);
+  for (const alias of definition.aliases || []) {
+    ALIAS_TO_PROCESS.set(alias, definition.name);
+  }
+}
+
+function splitWorkerModeTokens(value?: string | string[]): string[] {
+  const rawTokens = Array.isArray(value) ? value : String(value || '').split(',');
+  return rawTokens.map((token) => String(token).trim()).filter(Boolean);
+}
+
+function stripKnownPrefixes(token: string): string[] {
+  const candidates = [token];
+
+  const dotIndex = token.indexOf('.');
+  if (dotIndex > 0) {
+    candidates.push(token.slice(dotIndex + 1));
+  }
+
+  const colonIndex = token.indexOf(':');
+  if (colonIndex > 0) {
+    candidates.push(token.slice(colonIndex + 1));
+  }
+
+  return Array.from(new Set(candidates));
+}
+
+export function resolveWorkerProcessName(token: string): string {
+  const value = String(token || '').trim();
+  if (!value || value === '*' || value === '!' || value === '-') {
+    return value;
+  }
+
+  const directProcessName = ALIAS_TO_PROCESS.get(value);
+  if (directProcessName) {
+    return directProcessName;
+  }
+
+  for (const candidate of stripKnownPrefixes(value)) {
+    const processName = ALIAS_TO_PROCESS.get(candidate);
+    if (processName) {
+      return processName;
+    }
+  }
+
+  for (const definition of WORKER_PROCESS_DEFINITIONS) {
+    if ((definition.redisKeySuffixes || []).some((suffix) => value.endsWith(suffix))) {
+      return definition.name;
+    }
+  }
+
+  return value;
+}
+
+export function normalizeWorkerMode(value?: string | string[]): string | undefined {
+  const resolved = splitWorkerModeTokens(value).map(resolveWorkerProcessName);
+  if (resolved.includes('*')) {
+    return '*';
+  }
+
+  const unique = Array.from(new Set(resolved.filter(Boolean)));
+  return unique.length ? unique.join(',') : undefined;
+}
+
+export function workerModeTokens(value?: string | string[]): string[] {
+  const normalized = normalizeWorkerMode(value);
+  return normalized ? normalized.split(',') : [];
+}
+
+export function workerModeServesProcess(workerMode: string | undefined, processNameOrAlias: string): boolean {
+  const mode = normalizeWorkerMode(workerMode);
+  if (!mode) return true;
+  if (mode === 'main' || mode === 'app') return true;
+  if (mode === '-') return false;
+  if (mode === '*') return true;
+
+  const processName = resolveWorkerProcessName(processNameOrAlias);
+  return mode.split(',').includes(processName);
+}
+
+export function isWorkerOnlyMode(workerMode?: string): boolean {
+  const mode = normalizeWorkerMode(workerMode);
+  if (!mode || mode === 'main' || mode === 'app' || mode === '-') return false;
+  return !mode.split(',').includes('!');
+}
+
+export function isAppServingMode(workerMode?: string): boolean {
+  const mode = normalizeWorkerMode(workerMode);
+  if (!mode || mode === 'main' || mode === 'app') return true;
+  return mode.split(',').includes('!');
+}
+
+export function getWorkerProcessDefinition(nameOrAlias: string): WorkerProcessDefinition | undefined {
+  return PROCESS_BY_NAME.get(resolveWorkerProcessName(nameOrAlias));
+}
+
+export function getCommonWorkerProcesses(): WorkerProcessDefinition[] {
+  return WORKER_PROCESS_DEFINITIONS.filter((definition) => definition.common && !definition.sandbox);
+}
+
+export function getWorkerProcessByChannel(channel: string): WorkerProcessDefinition | undefined {
+  const processName = resolveWorkerProcessName(channel);
+  return PROCESS_BY_NAME.get(processName);
+}