plugin-agent-orchestrator 1.0.6 → 1.0.14

package/src/server/migrations/20260503000000-add-orchestrator-trace-fields.ts

@@ -0,0 +1,32 @@
+import { Migration } from '@nocobase/server';
+
+export default class AddOrchestratorTraceFieldsToSkillExecutions extends Migration {
+  on = 'afterLoad';
+  appVersion = '>=0.1.0';
+
+  async up() {
+    const queryInterface = this.db.sequelize.getQueryInterface();
+    const tablePrefix = this.db.options.tablePrefix || '';
+    const tableName = `${tablePrefix}skillExecutions`;
+    const tableExists = await queryInterface.tableExists(tableName).catch(() => false);
+    if (!tableExists) return;
+
+    const tableDesc = await queryInterface.describeTable(tableName);
+    const addIfMissing = async (name: string) => {
+      if (tableDesc[name]) return;
+      await queryInterface.addColumn(tableName, name, {
+        type: 'VARCHAR(100)',
+        allowNull: true,
+      });
+    };
+
+    await addIfMissing('orchestratorRootRunId');
+    await addIfMissing('orchestratorSpanId');
+    await addIfMissing('orchestratorParentSpanId');
+    await addIfMissing('orchestratorToolCallId');
+  }
+
+  async down() {
+    // No rollback: keeping nullable trace-link columns is safe.
+  }
+}

package/src/server/plugin.ts

@@ -2,20 +2,33 @@ import { Plugin } from '@nocobase/server';
 import path from 'path';
 import { createDelegateToolsProvider } from './tools/delegate-task';
 import { registerTracingResource } from './resources/tracing';
+import SkillHubSubFeature from './skill-hub/plugin';
 
 export class PluginAgentOrchestratorServer extends Plugin {
-  async afterAdd() {}
+  skillHub: SkillHubSubFeature;
+
+  async afterAdd() {
+    this.skillHub = new SkillHubSubFeature(this);
+  }
 
   async beforeLoad() {
     // Import collection definitions
     this.db.import({ directory: path.resolve(__dirname, 'collections') });
+
+    this.db.addMigrations({
+      namespace: this.name,
+      directory: path.resolve(__dirname, 'migrations'),
+      context: { plugin: this },
+    });
   }
 
   async load() {
+    await this.skillHub.load();
+
     // --- ACL ---
     this.app.acl.registerSnippet({
       name: `pm.${this.name}`,
-      actions: ['orchestratorConfig:*'],
+      actions: ['orchestratorConfig:*', 'orchestratorTracing:*', 'agentExecutionSpans:*', 'skillDefinitions:*', 'skillExecutions:*', 'skillHub:*', 'skillWorkerConfigs:*'],
     });
 
     // --- Register Dynamic Tools ---
@@ -29,18 +42,53 @@ export class PluginAgentOrchestratorServer extends Plugin {
     // Custom read-only resource for the Swarm Tracing admin page.
     registerTracingResource(this);
 
+    // --- Log Retention ---
+    // Daily prune of orchestratorLogs / agentExecutionSpans to keep tables bounded.
+    // Override window via env: ORCHESTRATOR_LOG_RETENTION_DAYS (default 30).
+    this.app.cronJobManager.addJob({
+      cronTime: '0 30 2 * * *',
+      onTick: async () => {
+        try {
+          const days = Number(process.env.ORCHESTRATOR_LOG_RETENTION_DAYS || 30);
+          if (!Number.isFinite(days) || days <= 0) return;
+          const cutoff = new Date(Date.now() - days * 86400000);
+          const repo = this.db.getRepository('orchestratorLogs');
+          const spansRepo = this.db.getRepository('agentExecutionSpans');
+          const deletedLogs = repo
+            ? await repo.destroy({
+                filter: { createdAt: { $lt: cutoff.toISOString() } },
+              })
+            : 0;
+          const deletedSpans = spansRepo
+            ? await spansRepo.destroy({
+                filter: { createdAt: { $lt: cutoff.toISOString() } },
+              })
+            : 0;
+          this.app.log.info(
+            `[AgentOrchestrator] Pruned ${deletedLogs} orchestratorLogs and ${deletedSpans} agentExecutionSpans rows older than ${days} day(s).`,
+          );
+        } catch (e) {
+          this.app.log.error('[AgentOrchestrator] Log retention job failed', e);
+        }
+      },
+    });
+
     // NOTE: The createReactAgent approach does NOT create aiConversation records,
     // so there is no need for a middleware to hide "headless" conversations.
     // If future versions need conversation logging, add it here.
   }
 
   async install() {
-    // No seed data needed on first install
+    await this.skillHub.install();
   }
 
   async afterEnable() {}
   async afterDisable() {}
   async remove() {}
+  
+  async beforeStop() {
+    await this.skillHub.beforeStop();
+  }
 }
 
 export default PluginAgentOrchestratorServer;

package/src/server/resources/tracing.ts

@@ -1,5 +1,108 @@
 import { Plugin } from '@nocobase/server';
 
+function toPlain(row: any) {
+  return row?.toJSON?.() || row;
+}
+
+function normalizeSpanFilter(filter: any = {}) {
+  const next = { ...filter };
+  if (next.subAgentUsername) {
+    next.employeeUsername = next.subAgentUsername;
+    delete next.subAgentUsername;
+  }
+  return next;
+}
+
+function spanTitle(row: any) {
+  if (row.title) return row.title;
+  if (row.type === 'sub_agent') return `${row.leaderUsername || '-'} -> ${row.employeeUsername || '-'}`;
+  return row.toolName || row.type || 'span';
+}
+
+function formatSpanListRow(raw: any) {
+  const row = toPlain(raw);
+  const metadata = row.metadata || {};
+  const input = row.input || {};
+  return {
+    id: row.id,
+    rootRunId: row.rootRunId,
+    parentSpanId: row.parentSpanId,
+    type: row.type,
+    leaderUsername: row.leaderUsername,
+    subAgentUsername: row.employeeUsername,
+    employeeUsername: row.employeeUsername,
+    toolName: row.toolName,
+    task: input.task || metadata.task || row.title || '',
+    context: input.context || '',
+    result: row.output,
+    status: row.status,
+    depth: metadata.depth ?? 0,
+    durationMs: row.durationMs,
+    error: row.error,
+    userId: row.userId,
+    createdAt: row.createdAt || row.startedAt,
+    traceCount: metadata.traceCount || 0,
+    messageCount: Array.isArray(metadata.messages) ? metadata.messages.length : metadata.messageCount || 0,
+    hasUnifiedTrace: true,
+  };
+}
+
+function buildSpanTree(rows: any[]) {
+  const plainRows = rows.map(toPlain);
+  const byId = new Map<string, any>();
+  const roots: any[] = [];
+
+  for (const row of plainRows) {
+    const node = { ...row, children: [] };
+    byId.set(String(row.id), node);
+  }
+
+  for (const row of byId.values()) {
+    if (row.parentSpanId && byId.has(String(row.parentSpanId))) {
+      byId.get(String(row.parentSpanId)).children.push(row);
+    } else {
+      roots.push(row);
+    }
+  }
+
+  const sortTree = (items: any[]) => {
+    items.sort(
+      (a, b) =>
+        new Date(a.startedAt || a.createdAt || 0).getTime() -
+        new Date(b.startedAt || b.createdAt || 0).getTime(),
+    );
+    for (const item of items) sortTree(item.children || []);
+  };
+  sortTree(roots);
+  return roots;
+}
+
+function flattenSpanTimeline(rows: any[]) {
+  return rows
+    .map(toPlain)
+    .sort(
+      (a, b) =>
+        new Date(a.startedAt || a.createdAt || 0).getTime() -
+        new Date(b.startedAt || b.createdAt || 0).getTime(),
+    )
+    .map((row) => {
+      const input = row.input || {};
+      const metadata = row.metadata || {};
+      return {
+        type: row.type,
+        at: row.startedAt || row.createdAt,
+        title: spanTitle(row),
+        toolName: row.toolName,
+        args: row.type === 'tool' || row.type === 'skill' ? input : undefined,
+        status: row.status,
+        content: row.output || row.error || input.task || metadata.summary || '',
+        spanId: row.id,
+        parentSpanId: row.parentSpanId,
+        skillExecutionId: row.skillExecutionId,
+      };
+    });
+}
+
 /**
  * Custom resource for the Swarm Tracing admin UI (Phase 5).
  * Queries the dedicated orchestratorLogs collection instead of
@@ -15,11 +118,41 @@ export function registerTracingResource(plugin: Plugin) {
        * List all delegation execution logs.
        */
       async list(ctx, next) {
-        const repo = ctx.db.getRepository('orchestratorLogs');
-        const { page = 1, pageSize = 50, sort = ['-createdAt'] } = ctx.action.params;
+        const { page = 1, pageSize = 50, sort = ['-createdAt'], filter = {} } = ctx.action.params;
 
         try {
+          const spanRepo = ctx.db.getRepository('agentExecutionSpans');
+          if (spanRepo) {
+            const spanFilter = {
+              ...normalizeSpanFilter(filter),
+              parentSpanId: null,
+              type: 'sub_agent',
+            };
+            const [spanRows, spanCount] = await spanRepo.findAndCount({
+              filter: spanFilter,
+              sort,
+              offset: (Number(page) - 1) * Number(pageSize),
+              limit: Number(pageSize),
+            });
+
+            if (spanCount > 0) {
+              ctx.body = {
+                data: spanRows.map(formatSpanListRow),
+                meta: {
+                  count: spanCount,
+                  page: Number(page),
+                  pageSize: Number(pageSize),
+                  totalPage: Math.ceil(spanCount / Number(pageSize)),
+                },
+              };
+              await next();
+              return;
+            }
+          }
+
+          const repo = ctx.db.getRepository('orchestratorLogs');
           const [rows, count] = await repo.findAndCount({
+            filter,
             sort,
             offset: (Number(page) - 1) * Number(pageSize),
             limit: Number(pageSize),
@@ -32,6 +165,7 @@ export function registerTracingResource(plugin: Plugin) {
               subAgentUsername: row.subAgentUsername,
               toolName: row.toolName,
               task: row.task,
+              context: row.context,
               result: row.result,
               status: row.status,
               depth: row.depth,
@@ -39,6 +173,9 @@ export function registerTracingResource(plugin: Plugin) {
               error: row.error,
               userId: row.userId,
               createdAt: row.createdAt,
+              traceCount: Array.isArray(row.trace) ? row.trace.length : 0,
+              messageCount: Array.isArray(row.messages) ? row.messages.length : 0,
+              hasUnifiedTrace: false,
             })),
             meta: {
               count,
@@ -47,9 +184,10 @@ export function registerTracingResource(plugin: Plugin) {
               totalPage: Math.ceil(count / Number(pageSize)),
             },
           };
-        } catch (e) {
+        } catch (e: any) {
+          // Bubble up so the admin sees an error toast instead of an empty grid.
           ctx.log.error('[AgentOrchestrator] Tracing list error', e);
-          ctx.body = { data: [], meta: { count: 0 } };
+          ctx.throw(500, e?.message || 'Failed to load tracing logs');
         }
 
         await next();
@@ -59,7 +197,7 @@ export function registerTracingResource(plugin: Plugin) {
        * Get a single delegation log by ID.
        */
       async get(ctx, next) {
-        const { filterByTk } = ctx.action.params;
+        const { filterByTk, source } = ctx.action.params;
 
         if (!filterByTk) {
           ctx.throw(400, 'id is required');
@@ -67,15 +205,52 @@ export function registerTracingResource(plugin: Plugin) {
         }
 
         try {
+          const spanRepo = ctx.db.getRepository('agentExecutionSpans');
+          if (spanRepo && source !== 'log') {
+            const span = await spanRepo.findOne({
+              filter: { id: filterByTk },
+            });
+
+            if (span) {
+              const plainSpan = toPlain(span);
+              const rows = await spanRepo.find({
+                filter: { rootRunId: plainSpan.rootRunId },
+                sort: ['createdAt'],
+              });
+              const tree = buildSpanTree(rows);
+              const timeline = flattenSpanTimeline(rows);
+              const rootRow = formatSpanListRow(plainSpan);
+              const metadata = plainSpan.metadata || {};
+
+              ctx.body = {
+                data: {
+                  ...rootRow,
+                  input: plainSpan.input,
+                  output: plainSpan.output,
+                  metadata,
+                  trace: timeline,
+                  messages: metadata.messages || [],
+                  children: tree,
+                },
+              };
+              await next();
+              return;
+            }
+          }
+
           const repo = ctx.db.getRepository('orchestratorLogs');
-          const log = await repo.findOne({
-            filter: { id: filterByTk },
-          });
+          const log =
+            source === 'span'
+              ? null
+              : await repo.findOne({
+                  filter: { id: filterByTk },
+                });
 
-          ctx.body = { data: log };
-        } catch (e) {
+          const plainLog = log?.toJSON?.() || log;
+          ctx.body = { data: plainLog ? { ...plainLog, hasUnifiedTrace: false } : null };
+        } catch (e: any) {
           ctx.log.error('[AgentOrchestrator] Tracing get error', e);
-          ctx.body = { data: null };
+          ctx.throw(500, e?.message || 'Failed to load tracing log');
         }
 
         await next();
@@ -83,9 +258,5 @@ export function registerTracingResource(plugin: Plugin) {
     },
   });
 
-  // ACL: allow admin access to tracing endpoints
-  app.acl.registerSnippet({
-    name: `pm.${plugin.name}.tracing`,
-    actions: ['orchestratorTracing:list', 'orchestratorTracing:get'],
-  });
+  // ACL: access is granted by the parent plugin's snippet (pm.plugin-agent-orchestrator → orchestratorTracing:*)
 }

package/src/server/services/CodeValidator.ts

@@ -0,0 +1,159 @@
+interface ForbiddenPattern {
+  pattern: RegExp;
+  reason: string;
+}
+
+const DANGEROUS_NODE_PATTERNS: ForbiddenPattern[] = [
+  { pattern: /require\s*\(\s*['"]child_process['"]\s*\)/, reason: 'child_process module not allowed' },
+  { pattern: /require\s*\(\s*['"]cluster['"]\s*\)/, reason: 'cluster module not allowed' },
+  { pattern: /require\s*\(\s*['"]dgram['"]\s*\)/, reason: 'dgram module not allowed' },
+  { pattern: /require\s*\(\s*['"]net['"]\s*\)/, reason: 'net module not allowed' },
+  { pattern: /require\s*\(\s*['"]http['"]\s*\)/, reason: 'http module not allowed' },
+  { pattern: /require\s*\(\s*['"]https['"]\s*\)/, reason: 'https module not allowed' },
+  { pattern: /require\s*\(\s*['"]vm['"]\s*\)/, reason: 'vm module not allowed' },
+  { pattern: /process\.exit/, reason: 'process.exit not allowed' },
+  { pattern: /process\.env(?!\s*\.OUTPUT_DIR)/, reason: 'process.env access not allowed (use OUTPUT_DIR only)' },
+  { pattern: /process\.kill/, reason: 'process.kill not allowed' },
+];
+
+const DANGEROUS_PYTHON_PATTERNS: ForbiddenPattern[] = [
+  { pattern: /import\s+subprocess/, reason: 'subprocess module not allowed' },
+  { pattern: /from\s+subprocess\s+import/, reason: 'subprocess module not allowed' },
+  { pattern: /import\s+shutil/, reason: 'shutil module not allowed' },
+  { pattern: /__import__\s*\(/, reason: '__import__ not allowed' },
+  { pattern: /os\.system\s*\(/, reason: 'os.system not allowed' },
+  { pattern: /os\.popen\s*\(/, reason: 'os.popen not allowed' },
+  { pattern: /os\.exec\w*\s*\(/, reason: 'os.exec* not allowed' },
+  { pattern: /os\.spawn\w*\s*\(/, reason: 'os.spawn* not allowed' },
+  { pattern: /\beval\s*\(/, reason: 'eval not allowed' },
+  { pattern: /\bexec\s*\(/, reason: 'exec not allowed' },
+  { pattern: /\bcompile\s*\(/, reason: 'compile not allowed' },
+];
+
+/** Built-in Node.js modules that are always allowed in sandbox code */
+const NODE_BUILTINS = [
+  'fs', 'path', 'os', 'crypto', 'util', 'stream', 'buffer',
+  'querystring', 'url', 'assert', 'events', 'string_decoder', 'zlib',
+];
+
+/** Built-in Python modules that are always allowed in sandbox code */
+const PYTHON_BUILTINS = [
+  'os', 'sys', 'json', 'math', 'datetime', 'collections', 'itertools',
+  'functools', 'pathlib', 'typing', 'io', 'csv', 're', 'string', 'textwrap',
+  'decimal', 'fractions', 'random', 'statistics', 'copy', 'enum', 'dataclasses',
+  'abc', 'contextlib', 'operator', 'time', 'calendar', 'locale', 'struct',
+  'hashlib', 'base64', 'binascii', 'codecs', 'unicodedata', 'pprint',
+  'warnings', 'traceback', 'logging', 'unittest', 'argparse',
+  'tempfile', 'xml', 'zipfile',
+  // Pre-installed local packages (trusted, bundled with plugin)
+  'svg_to_pptx',
+];
+
+/**
+ * Maps PyPI package names to their Python import names
+ * (only where they differ from the package name).
+ */
+const PYTHON_IMPORT_NAME_MAP: Record<string, string> = {
+  'python-docx': 'docx',
+  'python-pptx': 'pptx',
+  'Pillow': 'PIL',
+  'pyyaml': 'yaml',
+};
+
+export class CodeValidator {
+  /**
+   * Check code against forbidden patterns (dangerous modules/functions).
+   * @throws CodeValidationError if a forbidden pattern is found.
+   */
+  validate(code: string, language: 'node' | 'python'): void {
+    const patterns = language === 'node'
+      ? DANGEROUS_NODE_PATTERNS
+      : DANGEROUS_PYTHON_PATTERNS;
+
+    for (const { pattern, reason } of patterns) {
+      if (pattern.test(code)) {
+        throw new CodeValidationError(reason, pattern.source);
+      }
+    }
+  }
+
+  /**
+   * Validate that code only imports packages in the whitelist.
+   * Called after the basic forbidden pattern check.
+   * Skips validation if whitelist is empty (env not initialized yet).
+   *
+   * @param code - The code to validate
+   * @param language - 'node' or 'python'
+   * @param whitelist - Array of allowed package names (from skillWorkerConfigs.packageWhitelist)
+   */
+  validateImports(code: string, language: 'node' | 'python', whitelist: string[]): void {
+    if (!whitelist?.length) return; // Skip if env not initialized
+
+    if (language === 'node') {
+      this.validateNodeImports(code, whitelist);
+    } else if (language === 'python') {
+      this.validatePythonImports(code, whitelist);
+    }
+  }
+
+  /**
+   * Check Node.js require() calls against the whitelist.
+   * Built-in modules (fs, path, etc.) are always allowed.
+   */
+  private validateNodeImports(code: string, whitelist: string[]): void {
+    // Match require statements with string literal arguments (non-relative paths)
+    const requires = [...code.matchAll(/require\s*\(\s*['"]([^'"./][^'"]*)['"]\s*\)/g)];
+
+    for (const match of requires) {
+      const raw = match[1];
+      // Handle scoped packages (e.g. '@org/lib') and subpaths (e.g. 'lib/utils')
+      const pkgName = raw.startsWith('@')
+        ? raw.split('/').slice(0, 2).join('/')
+        : raw.split('/')[0];
+
+      if (NODE_BUILTINS.includes(pkgName)) continue;
+      if (whitelist.includes(pkgName)) continue;
+
+      throw new CodeValidationError(
+        `Package "${pkgName}" is not in the allowed whitelist. Allowed: ${whitelist.join(', ')}`,
+        `require('${pkgName}')`,
+      );
+    }
+  }
+
+  /**
+   * Check Python import/from statements against the whitelist.
+   * Built-in modules (os, sys, json, etc.) are always allowed.
+   * Handles PyPI→import name mapping (e.g., python-docx → docx, Pillow → PIL).
+   */
+  private validatePythonImports(code: string, whitelist: string[]): void {
+    // Match: import pkg, from pkg import ..., import pkg.sub
+    const imports = [...code.matchAll(/(?:^|\n)\s*(?:import|from)\s+([a-zA-Z_][a-zA-Z0-9_]*)/g)];
+
+    // Build allowed import names from whitelist
+    const allowedImports = new Set([
+      ...PYTHON_BUILTINS,
+      ...whitelist.map((p) => PYTHON_IMPORT_NAME_MAP[p] || p),
+    ]);
+
+    for (const match of imports) {
+      const pkg = match[1];
+      if (allowedImports.has(pkg)) continue;
+
+      throw new CodeValidationError(
+        `Module "${pkg}" is not in the allowed whitelist. Allowed: ${[...allowedImports].join(', ')}`,
+        `import ${pkg}`,
+      );
+    }
+  }
+}
+
+export class CodeValidationError extends Error {
+  constructor(
+    message: string,
+    public matchedPattern: string,
+  ) {
+    super(`Code validation failed: ${message}`);
+    this.name = 'CodeValidationError';
+  }
+}

package/src/server/services/ExecutionSpanService.ts

@@ -0,0 +1,106 @@
+export const ORCHESTRATOR_TRACE_CONTEXT_KEY = '__orchestratorTraceContext';
+
+export type OrchestratorTraceContext = {
+  rootRunId: string;
+  spanId?: string;
+  parentSpanId?: string;
+  toolCallId?: string;
+  leaderUsername?: string;
+  employeeUsername?: string;
+  toolName?: string;
+};
+
+type SpanValues = {
+  rootRunId: string;
+  parentSpanId?: string;
+  type: string;
+  status: string;
+  leaderUsername?: string;
+  employeeUsername?: string;
+  toolName?: string;
+  title?: string;
+  input?: any;
+  output?: string;
+  error?: string;
+  durationMs?: number;
+  startedAt?: Date;
+  endedAt?: Date;
+  orchestratorLogId?: string | number;
+  skillExecutionId?: string | number;
+  metadata?: any;
+  userId?: string | number;
+};
+
+function toPlain(record: any) {
+  return record?.toJSON?.() || record;
+}
+
+export class ExecutionSpanService {
+  constructor(private readonly plugin: any) {}
+
+  async create(values: SpanValues) {
+    try {
+      const repo = this.plugin.db.getRepository('agentExecutionSpans');
+      if (!repo) return null;
+
+      const record = await repo.create({
+        values: {
+          ...values,
+          startedAt: values.startedAt || new Date(),
+          createdAt: new Date(),
+        },
+      });
+      return toPlain(record);
+    } catch (error) {
+      this.plugin.app.log?.warn?.('[AgentOrchestrator] Failed to create execution span', error);
+      return null;
+    }
+  }
+
+  async update(spanId: string | number | undefined, values: Record<string, any>) {
+    if (!spanId) return null;
+    try {
+      const repo = this.plugin.db.getRepository('agentExecutionSpans');
+      if (!repo) return null;
+      await repo.update({
+        filterByTk: spanId,
+        values: {
+          ...values,
+          updatedAt: new Date(),
+        },
+      });
+      return { id: spanId };
+    } catch (error) {
+      this.plugin.app.log?.warn?.(`[AgentOrchestrator] Failed to update execution span ${spanId}`, error);
+      return null;
+    }
+  }
+
+  async finish(
+    spanId: string | number | undefined,
+    status: 'success' | 'error' | 'canceled' | 'timeout',
+    startedAt: number,
+    values: Record<string, any> = {},
+  ) {
+    return this.update(spanId, {
+      ...values,
+      status,
+      endedAt: new Date(),
+      durationMs: Date.now() - startedAt,
+    });
+  }
+}
+
+export function getOrchestratorTraceContext(ctx: any): OrchestratorTraceContext | null {
+  return (
+    ctx?.[ORCHESTRATOR_TRACE_CONTEXT_KEY] ||
+    ctx?.state?.orchestratorTraceContext ||
+    ctx?.runtime?.context?.orchestratorTraceContext ||
+    null
+  );
+}
+
+export function setOrchestratorTraceContext(ctx: any, traceContext: OrchestratorTraceContext) {
+  ctx[ORCHESTRATOR_TRACE_CONTEXT_KEY] = traceContext;
+  return ctx;
+}