plugin-cluster-manager 1.1.15 → 1.1.17

package/src/server/actions/doctor.ts

@@ -9,6 +9,12 @@ import type { ContainerInfo, StackConfig } from '../orchestrator/types';
 import { getLocalNodeId, getNodeRoleFrom } from '../utils/node';
 import { getRedisClient, scanKeys } from '../utils/redis';
 import { packagesFromConfig, type CustomPackageMap, type WorkerPackageMap } from '../../shared/packages';
+import {
+  getWorkerProcessDefinition,
+  normalizeWorkerMode,
+  resolveWorkerProcessName,
+  WORKER_PROCESS_DEFINITIONS,
+} from '../../shared/worker-processes';
 
 const ACTIVE_RUN_KEY = 'cluster-manager:doctor:active';
 const RESPONSE_KEY_PREFIX = 'cluster-manager:doctor-response:';
@@ -1112,29 +1118,91 @@ async function getQueueDiagnostics(app: Application) {
     | { constructor?: { name?: string }; queues?: Map<string, unknown[]> }
     | undefined;
   const channels = [];
+  const requiredProcesses = new Set<string>();
   for (const [channel, options] of eventQueue.events || new Map()) {
     let pending: number | null = null;
     if (adapter?.queues && eventQueue.getFullChannel) {
       const fullChannel = eventQueue.getFullChannel(channel, options.shared);
       pending = adapter.queues.get(fullChannel)?.length || 0;
     }
+    const processName = resolveWorkerProcessName(channel);
+    const definition = getWorkerProcessDefinition(processName);
+    if (definition && !definition.sandbox) {
+      requiredProcesses.add(definition.name);
+    }
     channels.push({
       channel,
+      workerProcessName: definition?.name,
       concurrency: options.concurrency || 1,
       interval: options.interval || 250,
       pending,
     });
   }
 
+  for (const definition of WORKER_PROCESS_DEFINITIONS) {
+    if (definition.common && definition.pluginName) {
+      try {
+        if ((app as any).pm?.get?.(definition.pluginName)) {
+          requiredProcesses.add(definition.name);
+        }
+      } catch {
+        // Ignore plugin-manager lookup errors.
+      }
+    }
+  }
+
+  const coverage = await getWorkerProcessCoverage(app, Array.from(requiredProcesses));
+
   return {
     available: true,
     connected: eventQueue.isConnected?.() || false,
     adapter: adapter?.constructor?.name || 'unknown',
     channels,
+    coverage,
     totalPending: channels.reduce((sum, item) => sum + (item.pending || 0), 0),
   };
 }
 
+async function getWorkerProcessCoverage(app: Application, requiredProcesses: string[]) {
+  const result = {
+    required: requiredProcesses,
+    covered: [] as string[],
+    missing: [] as string[],
+    wildcard: false,
+    stacks: [] as Array<{ id: unknown; name: unknown; workerMode: string }>,
+  };
+
+  try {
+    const repo = app.db.getRepository('orchestratorStacks');
+    const stacks = await repo.find({ filter: { enabled: true } });
+
+    const covered = new Set<string>();
+    for (const stack of stacks) {
+      const envVars = stack.get?.('envVars') as { WORKER_MODE?: string } | undefined;
+      const workerMode =
+        normalizeWorkerMode((stack.get?.('workerMode') as string | undefined) || envVars?.WORKER_MODE) || '*';
+      result.stacks.push({
+        id: stack.get?.('id'),
+        name: stack.get?.('name'),
+        workerMode,
+      });
+      if (workerMode === '*') {
+        result.wildcard = true;
+      }
+      for (const token of workerMode.split(',').filter(Boolean)) {
+        covered.add(token);
+      }
+    }
+
+    result.covered = Array.from(covered);
+    result.missing = result.wildcard ? [] : requiredProcesses.filter((processName) => !covered.has(processName));
+  } catch {
+    result.missing = requiredProcesses;
+  }
+
+  return result;
+}
+
 async function getOrchestratorDiagnostics(app: Application, options: DoctorSnapshotOptions) {
   const plugin = getApp(app).pm?.get('plugin-cluster-manager') as {
     orchestrator?: {
@@ -1214,6 +1282,7 @@ function buildRecommendations(params: {
   packageDrifts: number;
   redisAvailable: boolean;
   databaseOk: boolean;
+  queueCoverageMissing?: string[];
 }) {
   const recommendations = [];
   if (!params.redisAvailable) {
@@ -1258,6 +1327,13 @@ function buildRecommendations(params: {
       message: 'One or more worker nodes are missing configured packages or have failed package initialization.',
     });
   }
+  if (params.queueCoverageMissing?.length) {
+    recommendations.push({
+      level: 'warning',
+      code: 'worker_process_coverage_missing',
+      message: `No explicit worker stack covers: ${params.queueCoverageMissing.join(', ')}.`,
+    });
+  }
   if (params.topErrors > 0) {
     recommendations.push({
       level: 'warning',
@@ -1301,6 +1377,7 @@ async function buildDoctorReport(app: Application, run: Record<string, unknown>,
     packageDrifts: packageDiagnostics.packageDrifts.length,
     redisAvailable: Boolean(redisDiagnostics.available),
     databaseOk: Boolean(databaseDiagnostics.ping.ok),
+    queueCoverageMissing: queueDiagnostics.coverage?.missing || [],
   });
   const criticalFindings = recommendations.filter((item) => item.level === 'critical').length;
   const warningFindings = recommendations.filter((item) => item.level === 'warning').length;

package/src/server/actions/event-queue-monitor.ts

@@ -2,7 +2,13 @@ import { Context } from '@nocobase/actions';
 import { scanKeys } from '../utils/redis';
 
 const REDIS_QUEUE_CONNECTION = 'cluster-manager:queue-monitor';
-const REDIS_QUEUE_PATTERNS = ['*:plugin-git-manager:review:queue', '*:plugin-build-guide-block:build:queue'];
+const REDIS_QUEUE_PATTERNS = [
+  '*:plugin-git-manager:review:queue',
+  '*:plugin-build-guide-block:build:queue',
+  '*:plugin-build-visualization-block:build:queue',
+  'file-preview-auth.ocr.queue',
+  'nocobase:event-queue:*',
+];
 
 function getQueueRedisUrl() {
   return process.env.QUEUE_ADAPTER_REDIS_URL || process.env.REDIS_URL;
@@ -19,10 +25,16 @@ async function getQueueRedis(ctx: Context) {
 
 function knownRedisQueueKeys(ctx: Context) {
   const appName = (ctx.app as any).name || process.env.APP_NAME || 'main';
-  return [`${appName}:plugin-git-manager:review:queue`, `${appName}:plugin-build-guide-block:build:queue`];
+  return [
+    `${appName}:plugin-git-manager:review:queue`,
+    `${appName}:plugin-build-guide-block:build:queue`,
+    `${appName}:plugin-build-visualization-block:build:queue`,
+    'file-preview-auth.ocr.queue',
+  ];
 }
 
 function isKnownRedisQueueKey(key: string) {
+  if (key.startsWith('nocobase:event-queue:')) return true;
   return REDIS_QUEUE_PATTERNS.some((pattern) => {
     const suffix = pattern.replace('*:', '');
     return key === suffix || key.endsWith(`:${suffix}`);
@@ -30,6 +42,25 @@ function isKnownRedisQueueKey(key: string) {
 }
 
 function describeRedisQueueKey(key: string) {
+  if (key === 'file-preview-auth.ocr.queue') {
+    return {
+      appName: 'main',
+      plugin: 'plugin-file-preview-auth',
+      queue: 'ocr',
+      channel: key,
+    };
+  }
+
+  if (key.startsWith('nocobase:event-queue:')) {
+    const channel = key.slice('nocobase:event-queue:'.length);
+    return {
+      appName: channel.split('.')[0] || 'main',
+      plugin: 'event-queue',
+      queue: channel,
+      channel,
+    };
+  }
+
   const parts = String(key).split(':');
   const queue = parts[parts.length - 2] || key;
   const plugin = parts[parts.length - 3] || 'unknown';
@@ -96,7 +127,7 @@ function parseRedisQueueMessage(raw: string, key: string, index: number) {
   } catch {
     // Keep the raw string for non-JSON messages.
   }
-  const queuedAt = content?.queuedAt ? Date.parse(content.queuedAt) : null;
+  const queuedAt = content?.queuedAt ? Date.parse(content.queuedAt) : content?.options?.timestamp || null;
   return {
     id: `${key}:${index}`,
     index,

package/src/server/actions/orchestrator.ts

@@ -8,6 +8,11 @@
 import { Context } from '@nocobase/actions';
 import { getRedisClient } from '../utils/redis';
 import type { IOrchestratorAdapter, StackConfig } from '../orchestrator/types';
+import { normalizeWorkerMode } from '../../shared/worker-processes';
+
+type QueueMappingRecord = {
+  get: (key: string) => unknown;
+};
 
 /** Helper: get orchestrator adapter from plugin instance */
 function getAdapter(ctx: Context): IOrchestratorAdapter {
@@ -54,6 +59,33 @@ async function assertManagedContainer(
   }
 }
 
+function applyWorkerMode(stack: StackConfig, workerMode: string) {
+  stack.workerMode = workerMode;
+  stack.envVars = {
+    ...(stack.envVars || {}),
+    APP_ROLE: stack.envVars?.APP_ROLE || 'worker',
+    WORKER_MODE: workerMode,
+    SKILL_HUB_SANDBOX: stack.envVars?.SKILL_HUB_SANDBOX || 'false',
+  };
+}
+
+async function resolveMappedWorkerMode(ctx: Context, stack: StackConfig): Promise<string | undefined> {
+  const mappingsRepo = ctx.db.getRepository('workerQueueMappings');
+  const assigned = (await mappingsRepo.find({
+    filter: {
+      stackId: stack.id,
+      enabled: true,
+    },
+  })) as QueueMappingRecord[];
+
+  return normalizeWorkerMode(
+    assigned
+      .map((mapping) => String(mapping.get('queueName') || ''))
+      .filter(Boolean)
+      .join(','),
+  );
+}
+
 export const orchestratorActions = {
   /**
    * GET /workerOrchestrator:ping
@@ -122,9 +154,9 @@ export const orchestratorActions = {
    * Body: { stackId: 1, replicas: 3 }
    * Leader-only
    *
-   * Before scaling, resolves queue-to-stack mappings and injects
-   * WORKER_MODE into the stack's envVars so new containers only
-   * process assigned queues.
+   * Before scaling, resolves the stack-level worker mode and injects
+   * WORKER_MODE into envVars so new containers process only selected queues.
+   * Queue mappings remain as a fallback for legacy stacks.
    */
   async scale(ctx: Context, next: () => Promise<void>) {
     assertLeader(ctx);
@@ -136,44 +168,32 @@ export const orchestratorActions = {
 
     const stack = await getStack(ctx, stackId);
 
-    // ── Resolve queue assignments for this stack ──
-    try {
-      const mappingsRepo = ctx.db.getRepository('workerQueueMappings');
-      if (mappingsRepo) {
-        const assigned = await mappingsRepo.find({
-          filter: {
-            stackId: stack.id,
-            enabled: true,
-          },
-        });
-
-        const queueNames = assigned.map((m: any) => m.get('queueName') as string).filter(Boolean);
-
-        if (queueNames.length > 0) {
-          const workerMode = queueNames.join(',');
-          ctx.app.logger.info(
-            `[Orchestrator] Injecting WORKER_MODE=${workerMode} for stack "${stack.name}" (${queueNames.length} queue(s) assigned)`,
-          );
-          // Merge into envVars; adapter code merges envVars over inherited env
-          stack.envVars = {
-            ...(stack.envVars || {}),
-            WORKER_MODE: workerMode,
-          };
+    const stackWorkerMode = normalizeWorkerMode(stack.workerMode);
+    const envWorkerMode = normalizeWorkerMode(stack.envVars?.WORKER_MODE);
+
+    if (stackWorkerMode) {
+      applyWorkerMode(stack, stackWorkerMode);
+      ctx.app.logger.info(`[Orchestrator] Using stack WORKER_MODE=${stackWorkerMode} for "${stack.name}"`);
+    } else if (envWorkerMode && envWorkerMode !== '*') {
+      applyWorkerMode(stack, envWorkerMode);
+      ctx.app.logger.info(`[Orchestrator] Using env WORKER_MODE=${envWorkerMode} for "${stack.name}"`);
+    } else {
+      try {
+        const mappedWorkerMode = await resolveMappedWorkerMode(ctx, stack);
+
+        if (mappedWorkerMode) {
+          applyWorkerMode(stack, mappedWorkerMode);
+          ctx.app.logger.info(`[Orchestrator] Using mapped WORKER_MODE=${mappedWorkerMode} for "${stack.name}"`);
         } else {
-          // No specific assignment → default to all queues (backwards compatible)
-          stack.envVars = {
-            ...(stack.envVars || {}),
-            WORKER_MODE: '*',
-          };
+          const fallbackWorkerMode = envWorkerMode || '*';
+          applyWorkerMode(stack, fallbackWorkerMode);
+          ctx.app.logger.info(`[Orchestrator] Using fallback WORKER_MODE=${fallbackWorkerMode} for "${stack.name}"`);
         }
+      } catch (err: any) {
+        const fallbackWorkerMode = envWorkerMode || '*';
+        ctx.app.logger.debug(`[Orchestrator] Queue mappings not available: ${err.message}`);
+        applyWorkerMode(stack, fallbackWorkerMode);
       }
-    } catch (err: any) {
-      // If workerQueueMappings table doesn't exist yet, fall back gracefully
-      ctx.app.logger.debug(`[Orchestrator] Queue mappings not available: ${err.message}`);
-      stack.envVars = {
-        ...(stack.envVars || {}),
-        WORKER_MODE: '*',
-      };
     }
 
     const result = await adapter.scale(stack, Number(replicas));

package/src/server/actions/queue-mappings.ts

@@ -42,6 +42,7 @@ export const queueMappingsActions = {
         .filter((q) => !mappedNames.has(q.name))
         .map((q) => ({
           name: q.name,
+          workerProcessName: q.workerProcessName,
           type: q.type,
           label: q.label,
           description: q.description,

package/src/server/actions/tasks.ts

@@ -1,142 +1,142 @@
-import { Context } from '@nocobase/actions';
-
-export const tasksActions = {
-  async list(ctx: Context, next: () => Promise<void>) {
-    const repo = ctx.db.getRepository('asyncTasks');
-    const { page = 1, pageSize = 20, statusFilter } = ctx.action.params;
-
-    const filter: any = {};
-    if (statusFilter !== undefined && statusFilter !== '') {
-       filter.status = statusFilter === 'null' ? null : Number(statusFilter);
-    }
-
-    const [rows, count] = await repo.findAndCount({
-      filter,
-      appends: ['createdBy'],
-      offset: (page - 1) * pageSize,
-      limit: pageSize,
-      sort: ['-createdAt'],
-    });
-
-    ctx.body = {
-      data: rows,
-      meta: { count, page: Number(page), pageSize: Number(pageSize) },
-    };
-    await next();
-  },
-
-  async cancel(ctx: Context, next: () => Promise<void>) {
-    const { filterByTk } = ctx.action.params;
-    if (!filterByTk) {
-      ctx.throw(400, 'Task ID is required');
-    }
-
-    const repo = ctx.db.getRepository('asyncTasks');
-    const task = await repo.findOne({ filterByTk });
-    if (!task) {
-      ctx.throw(404, 'Task not found');
-    }
-
-    // Status: RUNNING = 0, PENDING = null
-    const status = task.get('status');
-    if (status !== 0 && status !== null) {
-      ctx.throw(400, 'Task is not running or pending');
-    }
-
-    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
-    ctx.app.logger.info(`[cluster-manager] Canceling task ${filterByTk} by user ${user}`);
-
-    // Try to cancel via pub/sub for cross-instance support
-    const pluginName = '@nocobase/plugin-async-task-manager';
-    try {
-      await ctx.app.pubSubManager.publish(`${pluginName}.task.cancel`, JSON.stringify({
-        taskId: filterByTk,
-      }));
-    } catch {
-      // Fallback: direct DB update
-    }
-
-    await repo.update({
-      filterByTk,
-      values: {
-        status: -2, // CANCELED
-        doneAt: new Date(),
-      },
-    });
-
-    ctx.body = { success: true };
-    await next();
-  },
-
-  async retry(ctx: Context, next: () => Promise<void>) {
-    const { filterByTk } = ctx.action.params;
-    if (!filterByTk) {
-      ctx.throw(400, 'Task ID is required');
-    }
-
-    const repo = ctx.db.getRepository('asyncTasks');
-    const task = await repo.findOne({ filterByTk });
-    if (!task) {
-      ctx.throw(404, 'Task not found');
-    }
-
-    const status = task.get('status');
-    // Only retry failed (-1) or canceled (-2) tasks
-    if (status !== -1 && status !== -2) {
-      ctx.throw(400, 'Only failed or canceled tasks can be retried');
-    }
-
-    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
-    ctx.app.logger.info(`[cluster-manager] Retrying task ${filterByTk} by user ${user}`);
-
-    await repo.update({
-      filterByTk,
-      values: {
-        status: null, // PENDING
-        result: null,
-        progressCurrent: 0,
-        startedAt: null,
-        doneAt: null,
-      },
-    });
-
-    // Re-queue the task
-    const pluginName = '@nocobase/plugin-async-task-manager';
-    try {
-      await ctx.app.eventQueue.publish(`${pluginName}.task`, {
-        taskId: filterByTk,
-      });
-    } catch {
-      // Queue may not be available
-    }
-
-    ctx.body = { success: true };
-    await next();
-  },
-
-  async purge(ctx: Context, next: () => Promise<void>) {
-    const { days = 0 } = ctx.action.params.values || ctx.action.params;
-    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
-    ctx.app.logger.info(`[cluster-manager] Purging tasks (days=${days}) by user ${user}`);
-    const repo = ctx.db.getRepository('asyncTasks');
-    
-    const filter: any = {
-      $and: [
-        { status: { $ne: 0 } },
-        { status: { $ne: null } }
-      ]
-    };
-
-    if (days && Number(days) > 0) {
-      const date = new Date();
-      date.setDate(date.getDate() - Number(days));
-      filter.createdAt = {
-        $lt: date.toISOString(),
-      };
-    }
-
-    const count = await repo.destroy({ filter });
-    ctx.body = { success: true, deletedCount: count };
-    await next();
-  },
-};
+import { Context } from '@nocobase/actions';
+
+export const tasksActions = {
+  async list(ctx: Context, next: () => Promise<void>) {
+    const repo = ctx.db.getRepository('asyncTasks');
+    const { page = 1, pageSize = 20, statusFilter } = ctx.action.params;
+
+    const filter: any = {};
+    if (statusFilter !== undefined && statusFilter !== '') {
+      filter.status = statusFilter === 'null' ? null : Number(statusFilter);
+    }
+
+    const [rows, count] = await repo.findAndCount({
+      filter,
+      appends: ['createdBy'],
+      offset: (page - 1) * pageSize,
+      limit: pageSize,
+      sort: ['-createdAt'],
+    });
+
+    ctx.body = {
+      data: rows,
+      meta: { count, page: Number(page), pageSize: Number(pageSize) },
+    };
+    await next();
+  },
+
+  async cancel(ctx: Context, next: () => Promise<void>) {
+    const { filterByTk } = ctx.action.params;
+    if (!filterByTk) {
+      ctx.throw(400, 'Task ID is required');
+    }
+
+    const repo = ctx.db.getRepository('asyncTasks');
+    const task = await repo.findOne({ filterByTk });
+    if (!task) {
+      ctx.throw(404, 'Task not found');
+    }
+
+    // Status: RUNNING = 0, PENDING = null
+    const status = task.get('status');
+    if (status !== 0 && status !== null) {
+      ctx.throw(400, 'Task is not running or pending');
+    }
+
+    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
+    ctx.app.logger.info(`[cluster-manager] Canceling task ${filterByTk} by user ${user}`);
+
+    // Try to cancel via pub/sub for cross-instance support
+    const pluginName = '@nocobase/plugin-async-task-manager';
+    try {
+      await ctx.app.pubSubManager.publish(
+        `${pluginName}.task.cancel`,
+        JSON.stringify({
+          taskId: filterByTk,
+        }),
+      );
+    } catch {
+      // Fallback: direct DB update
+    }
+
+    await repo.update({
+      filterByTk,
+      values: {
+        status: -2, // CANCELED
+        doneAt: new Date(),
+      },
+    });
+
+    ctx.body = { success: true };
+    await next();
+  },
+
+  async retry(ctx: Context, next: () => Promise<void>) {
+    const { filterByTk } = ctx.action.params;
+    if (!filterByTk) {
+      ctx.throw(400, 'Task ID is required');
+    }
+
+    const repo = ctx.db.getRepository('asyncTasks');
+    const task = await repo.findOne({ filterByTk });
+    if (!task) {
+      ctx.throw(404, 'Task not found');
+    }
+
+    const status = task.get('status');
+    // Only retry failed (-1) or canceled (-2) tasks
+    if (status !== -1 && status !== -2) {
+      ctx.throw(400, 'Only failed or canceled tasks can be retried');
+    }
+
+    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
+    ctx.app.logger.info(`[cluster-manager] Retrying task ${filterByTk} by user ${user}`);
+
+    await repo.update({
+      filterByTk,
+      values: {
+        status: null, // PENDING
+        result: null,
+        progressCurrent: 0,
+        startedAt: null,
+        doneAt: null,
+      },
+    });
+
+    // Re-queue the task
+    const pluginName = '@nocobase/plugin-async-task-manager';
+    try {
+      await ctx.app.eventQueue.publish(`${pluginName}.task`, {
+        id: filterByTk,
+      });
+    } catch {
+      // Queue may not be available
+    }
+
+    ctx.body = { success: true };
+    await next();
+  },
+
+  async purge(ctx: Context, next: () => Promise<void>) {
+    const { days = 0 } = ctx.action.params.values || ctx.action.params;
+    const user = ctx.state?.currentUser?.nickname || ctx.state?.currentUser?.id || 'unknown';
+    ctx.app.logger.info(`[cluster-manager] Purging tasks (days=${days}) by user ${user}`);
+    const repo = ctx.db.getRepository('asyncTasks');
+
+    const filter: any = {
+      $and: [{ status: { $ne: 0 } }, { status: { $ne: null } }],
+    };
+
+    if (days && Number(days) > 0) {
+      const date = new Date();
+      date.setDate(date.getDate() - Number(days));
+      filter.createdAt = {
+        $lt: date.toISOString(),
+      };
+    }
+
+    const count = await repo.destroy({ filter });
+    ctx.body = { success: true, deletedCount: count };
+    await next();
+  },
+};