spec-agent 1.0.3

package/src/commands/scan.ts

@@ -0,0 +1,568 @@
+import * as path from 'path';
+import { Command } from 'commander';
+import { Logger } from '../utils/logger';
+import { 
+  ensureDir, 
+  fileExists, 
+  getFileSize, 
+  formatSize, 
+  parseSize,
+  findFiles,
+  writeJson,
+  readFileContent
+} from '../utils/file';
+import { parseDocument, analyzeBase64Images } from '../services/document-parser';
+import { 
+  analyzeDocumentStructure, 
+  splitByLLMStructure,
+  describeEmbeddedImages,
+  getLLMConfigForPurpose,
+  validateLLMConfig 
+} from '../services/llm';
+import { Manifest, Chunk } from '../types';
+
+interface ScanOptions {
+  input?: string;
+  stdin?: boolean;
+  output: string;
+  chunkSize: string;
+  format: string;
+  dryRun?: boolean;
+  yes?: boolean;
+  /**
+   * Minimum chunk size - chunks smaller than this will be merged with neighbors
+   * Default: 10KB
+   */
+  minChunkSize?: string;
+  /**
+   * Use LLM to analyze document structure for intelligent chunking
+   * Default: true
+   */
+  llmChunking?: boolean;
+  /**
+   * Fail immediately if LLM chunking fails (no fallback)
+   * Default: false
+   */
+  strictLlm?: boolean;
+}
+
+const SCAN_EXIT_CODE = {
+  INPUT_ERROR: 1,
+  CONFIG_ERROR: 2,
+  LLM_STRICT_ERROR: 3,
+  RUNTIME_ERROR: 10,
+} as const;
+
+export async function scanCommand(options: ScanOptions, command: Command): Promise<void> {
+  const logger = new Logger();
+  
+  try {
+    // Validate input
+    if (!options.input && !options.stdin) {
+      logger.error('[E_SCAN_INPUT] --input is required (or use --stdin)');
+      logger.info('  spec-agent scan --input <path> --output <path>');
+      logger.info("  Run 'spec-agent scan --help' for details.");
+      process.exit(SCAN_EXIT_CODE.INPUT_ERROR);
+    }
+
+    let inputFiles: string[] = [];
+    
+    if (options.stdin) {
+      // Read from stdin
+      const stdinContent = await readStdin();
+      inputFiles = stdinContent.split('\n').filter(f => f.trim());
+    } else if (options.input) {
+      const inputPath = path.resolve(options.input);
+      
+      if (!(await fileExists(inputPath))) {
+        logger.error(`[E_SCAN_INPUT] Input path not found: ${options.input}`);
+        process.exit(SCAN_EXIT_CODE.INPUT_ERROR);
+      }
+
+      const stats = await ensureDir(inputPath).then(() => 
+        require('fs').promises.stat(inputPath)
+      ).catch(() => null);
+      
+      if (!stats) {
+        logger.error(`[E_SCAN_INPUT] Cannot access path: ${options.input}`);
+        process.exit(SCAN_EXIT_CODE.INPUT_ERROR);
+      }
+
+      if (stats.isDirectory()) {
+        // Find all supported files in directory
+        const patterns = ['**/*.md', '**/*.pdf', '**/*.docx', '**/*.html', '**/*.txt'];
+        for (const pattern of patterns) {
+          const files = await findFiles(pattern, inputPath);
+          inputFiles.push(...files);
+        }
+      } else {
+        inputFiles = [inputPath];
+      }
+    }
+
+    if (inputFiles.length === 0) {
+      logger.error('[E_SCAN_INPUT] No input files found');
+      process.exit(SCAN_EXIT_CODE.INPUT_ERROR);
+    }
+
+    const chunkSizeBytes = parseSize(options.chunkSize);
+    const totalSize = (await Promise.all(
+      inputFiles.map(f => getFileSize(f))
+    )).reduce((sum, s) => sum + s, 0);
+    
+    logger.info(`Found ${inputFiles.length} files, total size: ${formatSize(totalSize)}`);
+
+    // Parse documents and create content-based chunks
+    let rawChunks: Array<{ content: string; sourceFiles: string[]; title?: string }> = [];
+    const useLLMChunking = options.llmChunking !== false; // Default true
+    const llmConfig = useLLMChunking ? getLLMConfigForPurpose('scan') : null;
+    let llmChunkingSuccessFiles = 0;
+    let llmChunkingFallbackFiles = 0;
+    let imageAssetsDetected = 0;
+    let imageAssetsDescribed = 0;
+    if (useLLMChunking && llmConfig) {
+      // LLM chunking is enabled by default; fail fast so we don't silently degrade
+      // to regex-only behavior on complex documents.
+      try {
+        validateLLMConfig(llmConfig);
+      } catch (error) {
+        logger.error(`[E_SCAN_CONFIG] ${error instanceof Error ? error.message : String(error)}`);
+        process.exit(SCAN_EXIT_CODE.CONFIG_ERROR);
+      }
+    }
+
+    for (const filePath of inputFiles) {
+      logger.info(`Parsing ${path.basename(filePath)}...`);
+      
+      try {
+        const parsed = await parseDocument(filePath);
+        let contentForChunking = parsed.content;
+        imageAssetsDetected += parsed.images?.length || 0;
+        let imageSummaries: Record<string, string> = {};
+
+        if (parsed.images && parsed.images.length > 0) {
+          logger.info(`  检测到 ${parsed.images.length} 张嵌入图片`);
+          if (llmConfig && llmConfig.apiKey) {
+            imageSummaries = await describeEmbeddedImages(parsed.images, llmConfig, logger);
+            imageAssetsDescribed += Object.keys(imageSummaries).length;
+          } else {
+            logger.warn('  未配置 LLM，图片仅保留占位信息，不含语义摘要');
+          }
+        }
+        
+        // Log if base64 images were found and removed
+        const base64Info = analyzeBase64Images(contentForChunking);
+        if (base64Info.count > 0) {
+          logger.info(`  Remaining base64 refs: ${base64Info.count} (~${formatSize(base64Info.estimatedSize)})`);
+        }
+
+        // Try LLM-driven chunking first
+        let contentChunks: Array<{ content: string; title?: string }> = [];
+        
+        if (useLLMChunking && llmConfig) {
+          try {
+            logger.info(`  使用 LLM 分析文档结构...`);
+            const structure = await analyzeDocumentStructure(contentForChunking, llmConfig, logger);
+            
+            logger.info(`  识别到 ${structure.sections.length} 个章节，建议分组: ${structure.suggestedGroups.map(g => g.name).join(', ')}`);
+            
+            const llmChunks = splitByLLMStructure(contentForChunking, structure, chunkSizeBytes);
+            contentChunks = llmChunks.map(c => ({ content: c.content, title: c.title }));
+            llmChunkingSuccessFiles++;
+            
+            logger.info(`  LLM 智能切分: ${contentChunks.length} 个 chunks`);
+          } catch (llmError) {
+            if (options.strictLlm) {
+              throw new Error(`[E_SCAN_LLM_STRICT] LLM 严格模式下切分失败: ${llmError instanceof Error ? llmError.message : String(llmError)}`);
+            }
+            logger.warn(`  LLM 分析失败，回退到大小切分: ${llmError instanceof Error ? llmError.message : String(llmError)}`);
+            // Keep a safe fallback when LLM transiently fails. We avoid heading-regex
+            // fallback here because it is brittle on complex mixed-format documents.
+            const sizeChunks = forceSplitBySize(contentForChunking, chunkSizeBytes);
+            contentChunks = sizeChunks.map(c => ({ content: c }));
+            llmChunkingFallbackFiles++;
+          }
+        } else {
+          // Rule-based chunking
+          const ruleChunks = splitByStructure(contentForChunking, chunkSizeBytes);
+          contentChunks = ruleChunks.map(c => ({ content: c }));
+        }
+        
+        for (const chunk of contentChunks) {
+          const enrichedContent = appendImageSummariesToChunk(chunk.content, imageSummaries);
+          rawChunks.push({
+            content: enrichedContent,
+            sourceFiles: [filePath],
+            title: chunk.title,
+          });
+        }
+      } catch (error) {
+        logger.warn(`Failed to parse ${path.basename(filePath)}, using fallback`);
+        rawChunks.push({
+          content: `[Error parsing file: ${path.basename(filePath)}]`,
+          sourceFiles: [filePath],
+        });
+      }
+    }
+
+    // Merge small chunks to reduce total count
+    const minChunkSizeBytes = options.minChunkSize ? parseSize(options.minChunkSize) : 10 * 1024; // Default 10KB
+    rawChunks = mergeSmallChunks(rawChunks, chunkSizeBytes, minChunkSizeBytes);
+    
+    logger.info(`最终: ${rawChunks.length} 个 chunks (最大: ${options.chunkSize}, 最小合并: ${formatSize(minChunkSizeBytes)})`);
+    if (useLLMChunking) {
+      logger.info(`LLM 切分统计: 成功 ${llmChunkingSuccessFiles} 文件, 回退 ${llmChunkingFallbackFiles} 文件`);
+    }
+    if (imageAssetsDetected > 0) {
+      logger.info(`图片语义统计: 检测 ${imageAssetsDetected} 张, 已摘要 ${imageAssetsDescribed} 张`);
+    }
+
+    // Prepare chunks directory
+    const outputDir = path.dirname(path.resolve(options.output));
+    const chunksDir = path.join(outputDir, 'chunks');
+    await ensureDir(chunksDir);
+
+    // Write chunks to files
+    const chunks: Chunk[] = [];
+    for (let i = 0; i < rawChunks.length; i++) {
+      const rawChunk = rawChunks[i];
+      const chunkFileName = `chunk_${i}.txt`;
+      const chunkFilePath = path.join(chunksDir, chunkFileName);
+      
+      await require('fs-extra').writeFile(chunkFilePath, rawChunk.content, 'utf-8');
+      
+      chunks.push({
+        id: i,
+        sourceFiles: rawChunk.sourceFiles,
+        size: Buffer.byteLength(rawChunk.content, 'utf-8'),
+        contentPath: chunkFilePath,
+      });
+    }
+
+    // Preview mode
+    if (options.dryRun) {
+      logger.info('Dry run mode - manifest preview:');
+      for (const chunk of chunks.slice(0, 20)) {
+        const preview = chunk.content 
+          ? `${path.basename(chunk.sourceFiles[0])} (${formatSize(chunk.size)})`
+          : `${chunk.sourceFiles.length} files, ${formatSize(chunk.size)}`;
+        logger.info(`  Chunk ${chunk.id}: ${preview}`);
+      }
+      if (chunks.length > 20) {
+        logger.info(`  ... and ${chunks.length - 20} more chunks`);
+      }
+      return;
+    }
+
+    // Create manifest
+    const manifest: Manifest = {
+      version: '1.0.0',
+      createdAt: new Date().toISOString(),
+      totalFiles: inputFiles.length,
+      totalSize: totalSize,
+      chunkSize: chunkSizeBytes,
+      chunks: chunks,
+    };
+
+    // Write manifest
+    const outputPath = path.resolve(options.output);
+    await ensureDir(path.dirname(outputPath));
+    await writeJson(outputPath, manifest);
+
+    logger.success(`Manifest created: ${outputPath}`);
+    logger.json({
+      status: 'success',
+      totalFiles: inputFiles.length,
+      totalSize: formatSize(totalSize),
+      chunks: chunks.length,
+      chunkSize: options.chunkSize,
+      llmChunkingEnabled: useLLMChunking,
+      llmChunkingSuccessFiles,
+      llmChunkingFallbackFiles,
+      imageAssetsDetected,
+      imageAssetsDescribed,
+      manifestPath: outputPath,
+    });
+
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    logger.error(`Scan failed: ${message}`);
+    if (message.includes('[E_SCAN_LLM_STRICT]')) {
+      process.exit(SCAN_EXIT_CODE.LLM_STRICT_ERROR);
+    }
+    process.exit(SCAN_EXIT_CODE.RUNTIME_ERROR);
+  }
+}
+
+/**
+ * Split Markdown content by document structure (headers, sections)
+ * Optimized for Markdown format with hierarchical heading structure
+ */
+function splitByStructure(content: string, maxSize: number): string[] {
+  // Parse document structure to identify sections
+  const sections = parseMarkdownStructure(content);
+  
+  const chunks: string[] = [];
+  let currentChunk: string[] = [];
+  let currentSize = 0;
+
+  for (const section of sections) {
+    const sectionSize = Buffer.byteLength(section.content, 'utf-8');
+    
+    // If a single section is larger than maxSize, we need to split it
+    if (sectionSize > maxSize) {
+      // First, flush current chunk if any
+      if (currentChunk.length > 0) {
+        chunks.push(currentChunk.join('\n\n'));
+        currentChunk = [];
+        currentSize = 0;
+      }
+      
+      // Split the large section by paragraphs
+      const subChunks = splitLargeSection(section, maxSize);
+      chunks.push(...subChunks);
+      continue;
+    }
+    
+    // Check if adding this section would exceed maxSize
+    if (currentSize + sectionSize > maxSize && currentChunk.length > 0) {
+      // Start a new chunk
+      chunks.push(currentChunk.join('\n\n'));
+      currentChunk = [section.content];
+      currentSize = sectionSize;
+    } else {
+      currentChunk.push(section.content);
+      currentSize += sectionSize;
+    }
+  }
+
+  // Don't forget the last chunk
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk.join('\n\n'));
+  }
+
+  return chunks.length > 0 ? chunks : [content];
+}
+
+/**
+ * Parse Markdown document into structured sections
+ * Each section starts with a heading and includes all content until the next heading
+ */
+interface MarkdownSection {
+  level: number;
+  title: string;
+  content: string;
+}
+
+function parseMarkdownStructure(content: string): MarkdownSection[] {
+  const lines = content.split('\n');
+  const sections: MarkdownSection[] = [];
+  
+  let currentSection: MarkdownSection | null = null;
+  let currentContent: string[] = [];
+  
+  for (const line of lines) {
+    // Check if this line is a heading
+    const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);
+    
+    if (headingMatch) {
+      // Save previous section
+      if (currentSection) {
+        currentSection.content = currentContent.join('\n').trim();
+        sections.push(currentSection);
+      } else if (currentContent.length > 0) {
+        // Content before first heading - treat as intro section
+        sections.push({
+          level: 0,
+          title: '',
+          content: currentContent.join('\n').trim(),
+        });
+      }
+      
+      // Start new section
+      const level = headingMatch[1].length;
+      const title = headingMatch[2].trim();
+      currentSection = {
+        level,
+        title,
+        content: '',
+      };
+      currentContent = [line];
+    } else {
+      currentContent.push(line);
+    }
+  }
+  
+  // Don't forget the last section
+  if (currentSection) {
+    currentSection.content = currentContent.join('\n').trim();
+    sections.push(currentSection);
+  } else if (currentContent.length > 0) {
+    sections.push({
+      level: 0,
+      title: '',
+      content: currentContent.join('\n').trim(),
+    });
+  }
+  
+  return sections;
+}
+
+/**
+ * Split a large section by paragraphs while trying to preserve context
+ */
+function splitLargeSection(section: MarkdownSection, maxSize: number): string[] {
+  const chunks: string[] = [];
+  const paragraphs = section.content.split(/\n\n+/);
+  
+  let currentChunk: string[] = [];
+  let currentSize = 0;
+  
+  // Include the heading in each chunk for context
+  const headingPrefix = section.level > 0 ? `${'#'.repeat(section.level)} ${section.title}\n\n` : '';
+  const prefixSize = Buffer.byteLength(headingPrefix, 'utf-8');
+  
+  for (const paragraph of paragraphs) {
+    const paraSize = Buffer.byteLength(paragraph, 'utf-8');
+    
+    if (currentSize + paraSize + prefixSize > maxSize && currentChunk.length > 0) {
+      // Start new chunk with heading prefix
+      chunks.push(headingPrefix + currentChunk.join('\n\n'));
+      currentChunk = [paragraph];
+      currentSize = paraSize;
+    } else {
+      currentChunk.push(paragraph);
+      currentSize += paraSize;
+    }
+  }
+  
+  if (currentChunk.length > 0) {
+    chunks.push(headingPrefix + currentChunk.join('\n\n'));
+  }
+  
+  return chunks;
+}
+
+/**
+ * Force split content by size only (no structure awareness)
+ */
+function forceSplitBySize(content: string, maxSize: number): string[] {
+  const chunks: string[] = [];
+  let position = 0;
+
+  while (position < content.length) {
+    let end = Math.min(position + maxSize, content.length);
+    
+    // Try to find a newline to split at
+    if (end < content.length) {
+      const nextNewline = content.indexOf('\n', end - 100);
+      if (nextNewline !== -1 && nextNewline < end + 100) {
+        end = nextNewline + 1;
+      }
+    }
+
+    chunks.push(content.slice(position, end));
+    position = end;
+  }
+
+  return chunks;
+}
+
+/**
+ * Merge small chunks with their neighbors to reduce total chunk count
+ * - Chunks smaller than minSize are candidates for merging
+ * - They will be merged with the next chunk if possible
+ * - If the next chunk would exceed maxSize, merge with previous instead
+ */
+function mergeSmallChunks(
+  chunks: Array<{ content: string; sourceFiles: string[]; title?: string }>,
+  maxSize: number,
+  minSize: number
+): Array<{ content: string; sourceFiles: string[]; title?: string }> {
+  if (chunks.length <= 1) return chunks;
+  
+  const merged: Array<{ content: string; sourceFiles: string[]; title?: string }> = [];
+  let current = chunks[0];
+  
+  for (let i = 1; i < chunks.length; i++) {
+    const next = chunks[i];
+    const currentSize = Buffer.byteLength(current.content, 'utf-8');
+    const nextSize = Buffer.byteLength(next.content, 'utf-8');
+    
+    // If current chunk is small, try to merge with next
+    if (currentSize < minSize) {
+      const combinedSize = currentSize + nextSize + 2; // +2 for separator
+      
+      if (combinedSize <= maxSize) {
+        // Merge with next chunk
+        current.content = current.content + '\n\n' + next.content;
+        current.sourceFiles = [...new Set([...current.sourceFiles, ...next.sourceFiles])];
+        // Combine titles if both have them
+        if (current.title && next.title && current.title !== next.title) {
+          current.title = `${current.title} + ${next.title}`;
+        } else if (next.title && !current.title) {
+          current.title = next.title;
+        }
+        // Continue to next iteration - might merge more
+      } else {
+        // Can't merge with next, keep current as-is and move to next
+        merged.push(current);
+        current = next;
+      }
+    } else {
+      // Current chunk is big enough, keep it
+      merged.push(current);
+      current = next;
+    }
+  }
+  
+  // Don't forget the last chunk
+  merged.push(current);
+  
+  return merged;
+}
+
+function appendImageSummariesToChunk(content: string, imageSummaries: Record<string, string>): string {
+  if (!imageSummaries || Object.keys(imageSummaries).length === 0) {
+    return content;
+  }
+
+  const idMatches = content.match(/图片引用\s+((?:IMG|PDFIMG)\d{4})/g) || [];
+  const imageIds = Array.from(new Set(idMatches.map(item => item.replace(/.*\s+/, '').trim())));
+  if (imageIds.length === 0) {
+    return content;
+  }
+
+  const summaryLines: string[] = [];
+  for (const imageId of imageIds) {
+    const summary = imageSummaries[imageId];
+    if (summary) {
+      summaryLines.push(`- ${imageId}: ${summary}`);
+    }
+  }
+
+  if (summaryLines.length === 0) {
+    return content;
+  }
+
+  return `${content}\n\n### 图片语义补充\n${summaryLines.join('\n')}`;
+}
+
+function readStdin(): Promise<string> {
+  return new Promise((resolve) => {
+    let data = '';
+    process.stdin.setEncoding('utf8');
+    process.stdin.on('data', (chunk) => {
+      data += chunk;
+    });
+    process.stdin.on('end', () => {
+      resolve(data);
+    });
+    // If stdin is empty (not piped), resolve with empty string after short delay
+    setTimeout(() => {
+      if (data === '') {
+        resolve('');
+      }
+    }, 100);
+  });
+}

package/src/commands/status.ts

@@ -0,0 +1,129 @@
+import * as path from 'path';
+import { Command } from 'commander';
+import { Logger } from '../utils/logger';
+import { fileExists, readJson } from '../utils/file';
+import { WorkspaceStatus } from '../types';
+
+interface StatusOptions {
+  workspace: string;
+  format: string;
+}
+
+const PHASES = ['scan', 'analyze', 'merge', 'plan', 'dispatch'] as const;
+type Phase = typeof PHASES[number];
+
+export async function statusCommand(options: StatusOptions, command: Command): Promise<void> {
+  const logger = new Logger();
+  const workspacePath = path.resolve(options.workspace);
+  
+  try {
+    const status = await checkWorkspaceStatus(workspacePath);
+
+    if (options.format === 'json') {
+      logger.json(status);
+      return;
+    }
+
+    // Text output
+    logger.info(`Workspace: ${workspacePath}`);
+    logger.info('');
+    logger.info('Phases:');
+
+    for (const phase of PHASES) {
+      const phaseStatus = status.phases[phase];
+      const icon = phaseStatus.completed ? '✔' : '○';
+      let details = '';
+      if (phase === 'analyze' && 'chunksAnalyzed' in phaseStatus && phaseStatus.chunksAnalyzed !== undefined) {
+        details = `- ${phaseStatus.chunksAnalyzed} chunks analyzed`;
+      } else if ('output' in phaseStatus && phaseStatus.output) {
+        details = `- ${phaseStatus.output}`;
+      }
+      logger.info(`  ${icon} ${phase.padEnd(10)} ${details}`);
+    }
+
+    if (status.canResumeFrom) {
+      logger.info('');
+      logger.info(`Can resume from: ${status.canResumeFrom}`);
+      logger.info(`  spec-agent pipeline --from ${status.canResumeFrom} --output ${options.workspace}`);
+    }
+
+  } catch (error) {
+    logger.error(`Status check failed: ${error instanceof Error ? error.message : String(error)}`);
+    process.exit(1);
+  }
+}
+
+async function checkWorkspaceStatus(workspacePath: string): Promise<WorkspaceStatus> {
+  const status: WorkspaceStatus = {
+    workspace: workspacePath,
+    phases: {
+      scan: { completed: false },
+      analyze: { completed: false },
+      merge: { completed: false },
+      plan: { completed: false },
+      dispatch: { completed: false },
+    },
+  };
+
+  // Check each phase
+  for (const phase of PHASES) {
+    const doneFile = path.join(workspacePath, `.${phase}_done`);
+    const phaseOutput = getPhaseOutput(phase);
+    const outputPath = path.join(workspacePath, phaseOutput);
+    const phaseState = status.phases[phase];
+
+    if (await fileExists(doneFile)) {
+      phaseState.completed = true;
+      if (phase !== 'analyze') {
+        (phaseState as { completed: boolean; output?: string }).output = phaseOutput;
+      }
+
+      // Additional details for specific phases
+      if (phase === 'analyze') {
+        try {
+          const summariesDir = path.join(workspacePath, 'summaries');
+          const files = await import('../utils/file').then(m => m.findFiles('chunk_*_summary.json', summariesDir));
+          (phaseState as { completed: boolean; chunksAnalyzed?: number }).chunksAnalyzed = files.length;
+        } catch {
+          // Ignore errors
+        }
+      }
+    } else if (await fileExists(outputPath)) {
+      // Output exists but not marked done
+      phaseState.completed = true;
+      if (phase !== 'analyze') {
+        (phaseState as { completed: boolean; output?: string }).output = phaseOutput;
+      }
+    }
+  }
+
+  // Determine where we can resume from
+  for (let i = PHASES.length - 1; i >= 0; i--) {
+    const phase = PHASES[i];
+    if (status.phases[phase].completed) {
+      // Can resume from next phase
+      if (i < PHASES.length - 1) {
+        status.canResumeFrom = PHASES[i + 1];
+      }
+      break;
+    }
+  }
+
+  // If nothing done, can start from scan
+  if (!status.phases.scan.completed) {
+    status.canResumeFrom = 'scan';
+  }
+
+  return status;
+}
+
+function getPhaseOutput(phase: Phase): string {
+  const outputs: Record<Phase, string> = {
+    scan: 'manifest.json',
+    analyze: 'summaries/',
+    merge: 'spec_summary.json',
+    plan: 'task_plan.json',
+    dispatch: 'dispatch_plan.json',
+  };
+  return outputs[phase];
+}