plugin-agent-orchestrator 1.0.13 → 1.0.14

package/src/server/services/SkillRepositoryService.ts

@@ -0,0 +1,142 @@
+import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync } from 'fs';
+import { resolve, relative } from 'path';
+import AdmZip from 'adm-zip';
+import { parseSkillMarkdown } from '../skill-hub/utils/json-fields';
+export interface SkillPackageMetadata {
+  path: string;
+  metadata: Record<string, any>;
+  instructions: string;
+  code: string | null;
+}
+
+export class SkillRepositoryService {
+  private baseDir: string;
+
+  constructor(storagePath: string) {
+    this.baseDir = resolve(storagePath, 'skills');
+    if (!existsSync(this.baseDir)) {
+      mkdirSync(this.baseDir, { recursive: true });
+    }
+  }
+
+  /**
+   * Extract a zip file to the repository.
+   * Returns parsed metadata from SKILL.md / skill.yaml
+   */
+  async extractSkillPackage(skillName: string, zipFilePath: string) {
+    const targetDir = resolve(this.baseDir, skillName);
+    
+    // Clean target dir if exists
+    if (existsSync(targetDir)) {
+      rmSync(targetDir, { recursive: true, force: true });
+    }
+    mkdirSync(targetDir, { recursive: true });
+
+    // Extract
+    const zip = new AdmZip(zipFilePath);
+    zip.extractAllTo(targetDir, true);
+
+    return this.readSkillPackage(targetDir);
+  }
+
+  getSkillPath(skillName: string) {
+    return resolve(this.baseDir, skillName);
+  }
+
+  getSkillCode(skillName: string): string | null {
+    const dir = this.getSkillPath(skillName);
+    if (!existsSync(dir)) return null;
+
+    return this.getSkillCodeFromDir(dir);
+  }
+
+  copySkillPackageTo(skillName: string, destDir: string) {
+    const srcDir = this.getSkillPath(skillName);
+    this.copyDirectoryTo(srcDir, destDir);
+  }
+
+  readSkillPackage(packageDir: string): SkillPackageMetadata {
+    const rootDir = resolve(packageDir);
+    if (!existsSync(rootDir)) {
+      return {
+        path: rootDir,
+        metadata: {},
+        instructions: '',
+        code: null,
+      };
+    }
+
+    let metadata: Record<string, any> = {};
+    let instructions = '';
+    const skillMdPath = resolve(rootDir, 'SKILL.md');
+
+    if (existsSync(skillMdPath)) {
+      const content = readFileSync(skillMdPath, 'utf8');
+      const parsed = parseSkillMarkdown(content);
+      metadata = parsed.metadata;
+      instructions = parsed.body;
+    }
+
+    instructions += this.aggregateOtherMarkdownFiles(rootDir);
+
+    return {
+      path: rootDir,
+      metadata,
+      instructions: instructions.trim(),
+      code: this.getSkillCodeFromDir(rootDir),
+    };
+  }
+
+  copyDirectoryTo(srcDir: string, destDir: string) {
+    if (!existsSync(srcDir)) return;
+
+    cpSync(srcDir, destDir, {
+      recursive: true,
+      force: true,
+      filter: (src) => {
+        const name = src.split(/[\\/]/).pop();
+        return !['node_modules', '.git', '__pycache__'].includes(name || '') && !src.endsWith('.pyc');
+      },
+    });
+  }
+
+
+
+  private getSkillCodeFromDir(dir: string): string | null {
+    if (existsSync(resolve(dir, 'index.py'))) {
+      return readFileSync(resolve(dir, 'index.py'), 'utf8');
+    }
+    if (existsSync(resolve(dir, 'index.js'))) {
+      return readFileSync(resolve(dir, 'index.js'), 'utf8');
+    }
+    if (existsSync(resolve(dir, 'main.py'))) {
+      return readFileSync(resolve(dir, 'main.py'), 'utf8');
+    }
+
+    return null;
+  }
+
+  private aggregateOtherMarkdownFiles(dir: string, baseDir = dir): string {
+    let combined = '';
+    
+    if (!existsSync(dir)) return combined;
+    
+    const items = readdirSync(dir);
+    for (const item of items) {
+      const fullPath = resolve(dir, item);
+      const stat = statSync(fullPath);
+      
+      if (stat.isDirectory()) {
+        if (item !== 'node_modules' && item !== '.git') {
+          combined += this.aggregateOtherMarkdownFiles(fullPath, baseDir);
+        }
+      } else if (stat.isFile() && item.toLowerCase().endsWith('.md') && item.toUpperCase() !== 'SKILL.md') {
+        const relPath = relative(baseDir, fullPath);
+        const content = readFileSync(fullPath, 'utf8');
+        combined += `\n\n--- Content from ${relPath} ---\n\n${content}`;
+      }
+    }
+    
+    return combined;
+  }
+}

package/src/server/services/WorkerEnvManager.ts

@@ -0,0 +1,113 @@
+import Application from '@nocobase/server';
+import { Database } from '@nocobase/database';
+import { parseJsonText, stringifyJsonText } from '../skill-hub/utils/json-fields';
+
+type WorkerEnvConfig = {
+  npmRegistryUrl?: string;
+  npmAuthToken?: string;
+  pypiIndexUrl?: string;
+  pypiTrustedHost?: string;
+  aptMirrorUrl?: string;
+  aptGpgKeyUrl?: string;
+  customPackages?: {
+    python?: string[];
+    node?: string[];
+    apt?: string[];
+  };
+};
+
+const DEFAULT_WHITELIST: Record<'python' | 'node' | 'apt', string[]> = {
+  python: [
+    'python-docx',
+    'openpyxl',
+    'pandas',
+    'matplotlib',
+    'Pillow',
+    'reportlab',
+    'jinja2',
+    'pyyaml',
+    'tabulate',
+    'xlsxwriter',
+    'python-pptx',
+  ],
+  node: ['xlsx', 'docx', 'pdfkit', 'csv-parse', 'archiver', 'sharp', 'lodash', 'dayjs'],
+  apt: ['python3', 'python3-pip', 'python3-venv'],
+};
+
+export class WorkerEnvManager {
+  constructor(
+    private readonly app: Application,
+    private readonly db: Database,
+    private readonly storagePath: string,
+  ) {}
+
+  async getOrCreateConfig() {
+    const repo = this.db.getRepository('skillWorkerConfigs');
+    let config = await repo.findOne();
+    if (config) return config;
+
+    config = await repo.create({
+      values: {
+        retentionHours: 24,
+        initStatus: 'idle',
+        initProgressPercent: 0,
+        packageWhitelist: stringifyJsonText(DEFAULT_WHITELIST, DEFAULT_WHITELIST),
+        customPackages: stringifyJsonText({ python: [], node: [], apt: [] }, { python: [], node: [], apt: [] }),
+      },
+    });
+    return config;
+  }
+
+  async initEnvironment(config: WorkerEnvConfig) {
+    const current = await this.getOrCreateConfig();
+    await current.update({
+      initStatus: 'running',
+      initProgressPercent: 0,
+      initProgressLog: 'Queued sandbox environment refresh',
+      lastInitLog: '',
+      customPackages: stringifyJsonText(config.customPackages || { python: [], node: [], apt: [] }, {
+        python: [],
+        node: [],
+        apt: [],
+      }),
+    });
+
+    await this.app.pubSubManager.publish('skill-hub.init-env', {
+      ...config,
+      storagePath: this.storagePath,
+      queuedAt: new Date().toISOString(),
+    });
+
+    return 'Sandbox environment refresh queued on available workers.';
+  }
+
+  async executeInit(payload: WorkerEnvConfig) {
+    const customPackages = payload.customPackages || { python: [], node: [], apt: [] };
+    const whitelist = {
+      python: Array.from(new Set([...(DEFAULT_WHITELIST.python || []), ...(customPackages.python || [])])),
+      node: Array.from(new Set([...(DEFAULT_WHITELIST.node || []), ...(customPackages.node || [])])),
+      apt: Array.from(new Set([...(DEFAULT_WHITELIST.apt || []), ...(customPackages.apt || [])])),
+    };
+
+    await this.app.pubSubManager.publish('skill-hub.init-env.progress', {
+      percent: 25,
+      log: 'Resolved sandbox package whitelist',
+    });
+
+    await this.app.pubSubManager.publish('skill-hub.init-env.progress', {
+      percent: 75,
+      log: 'Sandbox runtime uses the local worker environment',
+    });
+
+    await this.app.pubSubManager.publish('skill-hub.init-env.done', {
+      status: 'succeeded',
+      log:
+        'Sandbox environment is ready on this worker. Package installation is managed by the worker image/runtime; whitelist was refreshed.',
+      whitelist,
+    });
+  }
+
+  parsePackageWhitelist(config: any) {
+    return parseJsonText(config?.get?.('packageWhitelist') ?? config?.packageWhitelist, DEFAULT_WHITELIST);
+  }
+}