skill-any-code 1.0.0

package/src/infrastructure/worker-pool/parse.worker.impl.ts

@@ -0,0 +1,135 @@
+import * as path from 'path'
+import { DirectoryAnalysis, FileAnalysis, LLMConfig, ModificationLog } from '../../common/types'
+import { OpenAIClient } from '../llm/openai.client'
+import { LLMUsageTracker } from '../llm/llm.usage.tracker'
+import { CodeSplitter } from '../splitter/code.splitter'
+import { FileHashCache } from '../cache/file.hash.cache'
+import { LLMAnalysisService } from '../../application/services/llm.analysis.service'
+import * as crypto from 'crypto'
+
+export type WorkerUsageDelta = {
+  totalPromptTokens: number
+  totalCompletionTokens: number
+  totalTokens: number
+  totalCalls: number
+}
+
+export async function parseFile(
+  filePath: string,
+  fileContent: string,
+  fileHash: string,
+  language?: string,
+  llmConfig?: LLMConfig
+): Promise<{ analysis: FileAnalysis; usage: WorkerUsageDelta }> {
+  if (!llmConfig) {
+    throw new Error('LLM config is required for file parsing')
+  }
+
+  // 初始化LLM相关服务
+  const tracker = new LLMUsageTracker()
+  const llmClient = new OpenAIClient(llmConfig, tracker)
+  const fileSplitter = new CodeSplitter(llmClient)
+  const cache = new FileHashCache({
+    cacheDir: llmConfig.cache_dir,
+    maxSizeMb: llmConfig.cache_max_size_mb,
+  })
+  const llmAnalysisService = new LLMAnalysisService(llmClient, fileSplitter, cache, llmConfig)
+
+  const result = await llmAnalysisService.analyzeFile(filePath, fileContent, fileHash)
+  return { analysis: result, usage: tracker.getStats() }
+}
+
+export async function aggregateDirectory(
+  dirPath: string,
+  payload: {
+    childrenDirs: Array<{ name: string; summary: string; description?: string }>
+    childrenFiles: Array<{ name: string; summary: string; description?: string }>
+  },
+  llmConfig?: LLMConfig
+): Promise<{ description: string; summary: string; usage: WorkerUsageDelta }> {
+  const name = path.basename(dirPath)
+  const childrenDirsPayload = (payload?.childrenDirs ?? []).map(d => ({
+    name: d.name,
+    summary: d.summary,
+    description: d.description ?? d.summary,
+  }))
+  const childrenFilesPayload = (payload?.childrenFiles ?? []).map(f => ({
+    name: f.name,
+    summary: f.summary,
+    description: f.description ?? f.summary,
+  }))
+
+  // LLM 不可用时回退到可解释的聚合文案
+  if (!llmConfig || !llmConfig.base_url || !llmConfig.model) {
+    const fileCount = childrenFilesPayload.length
+    const dirCount = childrenDirsPayload.length
+    const fallback = `The "${name}" directory contains ${fileCount} file(s) and ${dirCount} subdirectory(ies) and helps organize related source code and modules.`
+    return { description: fallback, summary: fallback, usage: { totalPromptTokens: 0, totalCompletionTokens: 0, totalTokens: 0, totalCalls: 0 } }
+  }
+
+  // 初始化LLM相关服务（worker 内做本任务 token 统计，返回给主线程聚合）
+  const tracker = new LLMUsageTracker()
+  const llmClient = new OpenAIClient(llmConfig, tracker)
+  const fileSplitter = new CodeSplitter(llmClient)
+  const cache = new FileHashCache({
+    cacheDir: llmConfig.cache_dir,
+    maxSizeMb: llmConfig.cache_max_size_mb,
+  })
+  const llmAnalysisService = new LLMAnalysisService(llmClient, fileSplitter, cache, llmConfig)
+
+  try {
+    const dirResultFromLLM = await llmAnalysisService.analyzeDirectory(childrenDirsPayload, childrenFilesPayload)
+    return {
+      description: dirResultFromLLM.description,
+      summary: dirResultFromLLM.summary,
+      usage: tracker.getStats(),
+    }
+  } catch {
+    const fileCount = childrenFilesPayload.length
+    const dirCount = childrenDirsPayload.length
+    const fallback = `The "${name}" directory contains ${fileCount} file(s) and ${dirCount} subdirectory(ies) and helps organize related source code and modules.`
+    return { description: fallback, summary: fallback, usage: tracker.getStats() }
+  }
+}
+
+export async function validateResult(parentResult: DirectoryAnalysis, childResult: FileAnalysis | DirectoryAnalysis): Promise<{
+  valid: boolean
+  corrections?: Partial<FileAnalysis | DirectoryAnalysis>
+  log?: ModificationLog
+}> {
+  // 简单的校验逻辑：检查子项是否在父项的结构中存在
+  const existsInStructure = parentResult.structure.some(item => item.name === childResult.name)
+  const corrections: Partial<FileAnalysis | DirectoryAnalysis> = {}
+  let log: ModificationLog | undefined
+
+  if (!existsInStructure) {
+    // 修正：将子项添加到父项的结构中
+    ;(corrections as Partial<DirectoryAnalysis>).structure = [
+      ...parentResult.structure,
+      {
+        name: childResult.name,
+        type: childResult.type,
+        description: childResult.summary.substring(0, 100),
+      },
+    ]
+
+    log = {
+      id: crypto.randomUUID(),
+      timestamp: new Date().toISOString(),
+      path: parentResult.path,
+      type: 'omission',
+      originalContent: JSON.stringify(parentResult.structure),
+      correctedContent: JSON.stringify((corrections as Partial<DirectoryAnalysis>).structure),
+      reason: `Child item ${childResult.name} missing from directory structure`,
+    }
+
+    return {
+      valid: false,
+      corrections,
+      log,
+    }
+  }
+
+  return { valid: true }
+}
+

package/src/infrastructure/worker-pool/parse.worker.ts

@@ -0,0 +1,9 @@
+import * as workerpool from 'workerpool'
+import { aggregateDirectory, parseFile, validateResult } from './parse.worker.impl'
+
+// 注册Worker方法
+workerpool.worker({
+  parseFile,
+  aggregateDirectory,
+  validateResult
+})

package/src/infrastructure/worker-pool/worker-pool.service.ts

@@ -0,0 +1,173 @@
+import * as workerpool from 'workerpool'
+import * as path from 'path'
+import * as fs from 'fs'
+import { IWorkerPoolService } from '../../domain/interfaces'
+import { FileAnalysis, DirectoryAnalysis, ModificationLog, LLMConfig } from '../../common/types'
+import { AppError, ErrorCode } from '../../common/errors'
+import { DEFAULT_CONCURRENCY } from '../../common/constants'
+import os from 'os'
+import { aggregateDirectory, parseFile, validateResult } from './parse.worker.impl'
+
+export interface DirectoryAggregationPayload {
+  childrenDirs: Array<{ name: string; summary: string; description?: string }>
+  childrenFiles: Array<{ name: string; summary: string; description?: string }>
+}
+
+export interface DirectoryAggregationLLMResult {
+  description: string
+  summary: string
+}
+
+type WorkerUsageDelta = {
+  totalPromptTokens: number
+  totalCompletionTokens: number
+  totalTokens: number
+  totalCalls: number
+}
+
+export class WorkerPoolService implements IWorkerPoolService {
+  private pool?: workerpool.Pool
+  private pendingTasks: Promise<any>[] = []
+  private currentConcurrency: number
+  private llmConfig: LLMConfig
+  private readonly localMode: boolean
+
+  constructor(llmConfig: LLMConfig, concurrency: number = DEFAULT_CONCURRENCY) {
+    this.currentConcurrency = concurrency
+    this.llmConfig = {
+      ...llmConfig,
+      cache_dir: WorkerPoolService.expandTilde(llmConfig.cache_dir),
+    }
+
+    const workerScript = path.join(__dirname, 'parse.worker.js')
+    this.localMode = !fs.existsSync(workerScript)
+
+    // Jest/ts-jest 直接执行 src 时，parse.worker.js 不存在。此时回退到“本进程执行”以保证核心逻辑可测。
+    // 生产构建（dist）与 CLI 运行时，parse.worker.js 存在，继续使用 worker threads 提升性能。
+    if (!this.localMode) {
+      this.pool = workerpool.pool(workerScript, {
+        maxWorkers: concurrency,
+        workerType: 'thread',
+      })
+    }
+  }
+
+  private static expandTilde(p: string): string {
+    if (!p) return p
+    if (p.startsWith('~') && (p.length === 1 || p[1] === '/' || p[1] === '\\')) {
+      return path.join(os.homedir(), p.slice(1))
+    }
+    return p
+  }
+
+  async submitFileAnalysisTask(
+    filePath: string,
+    fileContent: string,
+    fileHash: string,
+    language?: string
+  ): Promise<{ analysis: FileAnalysis; usage: WorkerUsageDelta }> {
+    if (this.localMode) {
+      const task = parseFile(filePath, fileContent, fileHash, language, this.llmConfig)
+      this.pendingTasks.push(task)
+      try {
+        return await task
+      } finally {
+        this.pendingTasks = this.pendingTasks.filter(t => t !== task)
+      }
+    }
+    try {
+      const task = this.pool!.exec('parseFile', [filePath, fileContent, fileHash, language, this.llmConfig])
+      this.pendingTasks.push(task)
+      
+      const result = await task
+      // 移除已完成的任务
+      this.pendingTasks = this.pendingTasks.filter(t => t !== task)
+      
+      return result
+    } catch (e) {
+      throw new AppError(ErrorCode.WORKER_SCHEDULE_FAILED, 'File analysis task failed', (e as Error).message)
+    }
+  }
+
+  async submitDirectoryAggregationTask(
+    dirPath: string,
+    payload: DirectoryAggregationPayload
+  ): Promise<DirectoryAggregationLLMResult & { usage: WorkerUsageDelta }> {
+    if (this.localMode) {
+      const task = aggregateDirectory(dirPath, payload, this.llmConfig)
+      this.pendingTasks.push(task)
+      try {
+        return await task
+      } finally {
+        this.pendingTasks = this.pendingTasks.filter(t => t !== task)
+      }
+    }
+    try {
+      const task = this.pool!.exec('aggregateDirectory', [dirPath, payload, this.llmConfig])
+      this.pendingTasks.push(task)
+      
+      const result = await task
+      this.pendingTasks = this.pendingTasks.filter(t => t !== task)
+      
+      return result
+    } catch (e) {
+      throw new AppError(ErrorCode.WORKER_SCHEDULE_FAILED, 'Directory aggregation task failed', (e as Error).message)
+    }
+  }
+
+  async submitValidationTask(parentResult: DirectoryAnalysis, childResult: FileAnalysis | DirectoryAnalysis): Promise<{
+    valid: boolean
+    corrections?: Partial<FileAnalysis | DirectoryAnalysis>
+    log?: ModificationLog
+  }> {
+    if (this.localMode) {
+      const task = validateResult(parentResult, childResult)
+      this.pendingTasks.push(task)
+      try {
+        return await task
+      } finally {
+        this.pendingTasks = this.pendingTasks.filter(t => t !== task)
+      }
+    }
+    try {
+      const task = this.pool!.exec('validateResult', [parentResult, childResult])
+      this.pendingTasks.push(task)
+      
+      const result = await task
+      this.pendingTasks = this.pendingTasks.filter(t => t !== task)
+      
+      return result
+    } catch (e) {
+      throw new AppError(ErrorCode.WORKER_SCHEDULE_FAILED, 'Validation task failed', (e as Error).message)
+    }
+  }
+
+  setConcurrency(concurrency: number): void {
+    this.currentConcurrency = concurrency
+    if (this.localMode) {
+      return
+    }
+    // 立即终止旧 worker，避免遗留线程占用资源
+    void this.pool!.terminate(true).catch(() => {})
+    this.pool = workerpool.pool(path.join(__dirname, 'parse.worker.js'), {
+      maxWorkers: concurrency,
+      workerType: 'thread'
+    })
+  }
+
+  async waitAll(): Promise<void> {
+    await Promise.all(this.pendingTasks)
+  }
+
+  cancelAll(): void {
+    void this.pool?.terminate(true).catch(() => {})
+    this.pendingTasks = []
+  }
+
+  async terminate(force: boolean = true): Promise<void> {
+    if (this.pool) {
+      await this.pool.terminate(force)
+    }
+    this.pendingTasks = []
+  }
+}

package/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "commonjs",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "moduleResolution": "node",
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
+    "baseUrl": ".",
+    "paths": {
+      "@/*": ["src/*"]
+    },
+    "types": ["node", "jest"]
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "coverage", "tests"]
+}

package/tsconfig.test.json

@@ -0,0 +1,5 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["src/**/*", "tests/**/*"],
+  "exclude": ["node_modules", "dist", "coverage"]
+}