spec-agent 1.0.3

package/src/index.ts

@@ -0,0 +1,137 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { scanCommand } from './commands/scan';
+import { analyzeCommand } from './commands/analyze';
+import { mergeCommand } from './commands/merge';
+import { planCommand } from './commands/plan';
+import { dispatchCommand } from './commands/dispatch';
+import { learnCommand } from './commands/learn';
+import { pipelineCommand } from './commands/pipeline';
+import { statusCommand } from './commands/status';
+import { cleanCommand } from './commands/clean';
+import { doctorCommand } from './commands/doctor';
+
+const program = new Command();
+
+program
+  .name('spec-agent')
+  .description('Multi-agent CLI tool for breaking down large requirement documents')
+  .version('1.0.0');
+
+program
+  .command('scan')
+  .description('Scan documents and create chunk manifest')
+  .option('-i, --input <path>', 'Input file or directory')
+  .option('--stdin', 'Read file paths from stdin')
+  .option('-o, --output <path>', 'Output path for manifest.json', 'manifest.json')
+  .option('-c, --chunk-size <size>', 'Maximum chunk size for LLM analysis (default: 200kb)', '200kb')
+  .option('--min-chunk-size <size>', 'Minimum chunk size - smaller chunks will be merged (default: 10kb)', '10kb')
+  .option('--no-llm-chunking', 'Disable LLM-driven document structure analysis (use rule-based only)')
+  .option('--strict-llm', 'Fail if LLM chunking fails (no fallback)')
+  .option('-f, --format <format>', 'Input format: md, pdf, docx, html, auto', 'auto')
+  .option('--dry-run', 'Preview scan plan without creating manifest')
+  .option('-y, --yes', 'Skip confirmation prompts')
+  .action(scanCommand);
+
+program
+  .command('analyze')
+  .description('Analyze document chunks in parallel using multiple agents')
+  .option('-m, --manifest <path>', 'Path to manifest.json', 'manifest.json')
+  .option('-o, --output <dir>', 'Output directory for summaries', 'summaries')
+  .option('-a, --agents <count>', 'Number of parallel agents', 'auto')
+  .option('--chunks <indices>', 'Comma-separated chunk indices to analyze')
+  .option('--focus <type>', 'Analysis focus: full, features, data-model, api, pages', 'full')
+  .option('--apply-learned', 'Apply learned patterns from previous runs')
+  .option('--retries <count>', 'Retry attempts per failed chunk (default: 1)', '1')
+  .option('--budget-tokens <count>', 'Max total tokens for analyze run (0 = unlimited)', '0')
+  .option('--dry-run', 'Preview analysis plan without running')
+  .option('-y, --yes', 'Skip confirmation prompts')
+  .action(analyzeCommand);
+
+program
+  .command('merge')
+  .description('Merge chunk summaries into unified spec')
+  .option('-s, --summaries <dir>', 'Directory containing summaries', 'summaries')
+  .option('-o, --output <path>', 'Output path for spec_summary.json', 'spec_summary.json')
+  .option('--strategy <strategy>', 'Merge strategy: conservative, aggressive', 'conservative')
+  .option('--dry-run', 'Preview merge plan without running')
+  .option('-y, --yes', 'Skip confirmation prompts')
+  .action(mergeCommand);
+
+program
+  .command('plan')
+  .description('Generate executable task plan from spec')
+  .option('-s, --spec <path>', 'Path to spec_summary.json', 'spec_summary.json')
+  .option('-o, --output <path>', 'Output path for task_plan.json', 'task_plan.json')
+  .option('-t, --type <type>', 'Output type: prototype, code, docs', 'prototype')
+  .option('--framework <fw>', 'Target framework: vue3, react, html', 'vue3')
+  .option('-p, --parallel <count>', 'Max parallel tasks', '3')
+  .option('--dry-run', 'Preview task plan without generating')
+  .option('-y, --yes', 'Skip confirmation prompts')
+  .action(planCommand);
+
+program
+  .command('dispatch')
+  .description('Dispatch tasks to specialized agents')
+  .option('-p, --plan <path>', 'Path to task_plan.json', 'task_plan.json')
+  .option('-o, --output <path>', 'Output path for dispatch_plan.json', 'dispatch_plan.json')
+  .option('-a, --agents <mapping>', 'Agent type mapping, e.g. frontend:2,backend:2')
+  .option('--strategy <strategy>', 'Dispatch strategy: balanced, skill-first, load-first', 'balanced')
+  .option('--dry-run', 'Preview dispatch plan without executing')
+  .option('-y, --yes', 'Skip confirmation prompts')
+  .action(dispatchCommand);
+
+program
+  .command('learn')
+  .description('Learn and accumulate project-specific patterns')
+  .option('-w, --workspace <dir>', 'Workspace directory', '.')
+  .option('--from <phase>', 'Learn from: summaries, plan, dispatch', 'summaries')
+  .option('--pattern <name>', 'Pattern name to learn')
+  .option('--rule <value>', 'Pattern rule value')
+  .option('--list', 'List all learned patterns')
+  .option('--export <path>', 'Export patterns to JSON file')
+  .option('--apply', 'Apply learned patterns to current workspace')
+  .action(learnCommand);
+
+program
+  .command('pipeline')
+  .description('Run full pipeline: scan → analyze → merge → plan → dispatch')
+  .option('-i, --input <path>', 'Input file or directory')
+  .option('-o, --output <dir>', 'Output directory', 'workspace')
+  .option('-a, --agents <count>', 'Max parallel agents', 'auto')
+  .option('-c, --chunk-size <size>', 'Max chunk size for LLM analysis (default: 200kb)', '200kb')
+  .option('--min-chunk-size <size>', 'Minimum chunk size - smaller chunks will be merged (default: 10kb)', '10kb')
+  .option('--analyze-retries <count>', 'Retry attempts per failed analyze chunk (default: 1)', '1')
+  .option('--analyze-budget-tokens <count>', 'Max total tokens for analyze in pipeline (0 = unlimited)', '0')
+  .option('--strict-llm', 'Fail if LLM chunking fails (no fallback)')
+  .option('--framework <fw>', 'Target framework', 'vue3')
+  .option('--stop-at <phase>', 'Stop after phase: scan, analyze, merge, plan, dispatch')
+  .option('--from <phase>', 'Resume from phase: scan, analyze, merge, plan, dispatch')
+  .option('--dry-run', 'Preview full pipeline without executing')
+  .option('-y, --yes', 'Skip all confirmation prompts')
+  .action(pipelineCommand);
+
+program
+  .command('status')
+  .description('Check workspace status')
+  .option('-w, --workspace <dir>', 'Workspace directory', '.')
+  .option('--format <format>', 'Output format: text, json', 'text')
+  .action(statusCommand);
+
+program
+  .command('clean')
+  .description('Clean workspace intermediate files')
+  .option('-w, --workspace <dir>', 'Workspace directory', '.')
+  .option('--dry-run', 'Preview what would be cleaned')
+  .option('-y, --yes', 'Skip confirmation')
+  .action(cleanCommand);
+
+program
+  .command('doctor')
+  .description('Run environment and configuration health checks')
+  .option('-w, --workspace <dir>', 'Workspace directory', '.')
+  .option('--check-llm', 'Run a lightweight real LLM connectivity test')
+  .option('--format <format>', 'Output format: text, json', 'text')
+  .action(doctorCommand);
+
+program.parse();

package/src/services/document-parser.ts

@@ -0,0 +1,548 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { execFile } from 'child_process';
+import { promisify } from 'util';
+
+const execFileAsync = promisify(execFile);
+
+export interface ParsedDocument {
+  content: string;
+  format: 'markdown' | 'text' | 'html';
+  images?: EmbeddedImage[];
+  metadata: {
+    title?: string;
+    author?: string;
+    pages?: number;
+    wordCount?: number;
+  };
+}
+
+export interface EmbeddedImage {
+  id: string;
+  alt: string;
+  mimeType: string;
+  estimatedSize: number;
+  dataUri: string;
+}
+
+/**
+ * Parse a document file and extract text content as Markdown
+ * Supports: .md, .txt, .html, .pdf, .docx
+ * 
+ * All formats are normalized to Markdown for consistent chunking:
+ * - Headings become # ## ### 
+ * - Lists become - or 1. 
+ * - Tables become Markdown tables
+ */
+export async function parseDocument(filePath: string): Promise<ParsedDocument> {
+  const ext = path.extname(filePath).toLowerCase();
+  const buffer = await fs.readFile(filePath);
+
+  switch (ext) {
+    case '.md':
+      return parseMarkdownFile(buffer);
+    case '.txt':
+      return parseTextFile(buffer);
+    case '.html':
+    case '.htm':
+      return parseHtmlToMarkdown(buffer);
+    case '.pdf':
+      return parsePdfToMarkdown(buffer, filePath);
+    case '.docx':
+      return parseDocxToMarkdown(buffer);
+    default:
+      // Try to read as text
+      return parseTextFile(buffer);
+  }
+}
+
+function parseMarkdownFile(buffer: Buffer): ParsedDocument {
+  const content = buffer.toString('utf-8');
+  const normalized = normalizeMarkdown(content);
+  const extracted = extractEmbeddedImages(normalized);
+  return {
+    content: extracted.content,
+    format: 'markdown',
+    images: extracted.images,
+    metadata: {
+      wordCount: content.split(/\s+/).length,
+    },
+  };
+}
+
+function parseTextFile(buffer: Buffer): ParsedDocument {
+  const content = buffer.toString('utf-8');
+  const normalized = normalizeMarkdown(content);
+  const extracted = extractEmbeddedImages(normalized);
+  return {
+    content: extracted.content,
+    format: 'markdown',
+    images: extracted.images,
+    metadata: {
+      wordCount: content.split(/\s+/).length,
+    },
+  };
+}
+
+/**
+ * Normalize content to proper Markdown format
+ * - Ensures consistent heading syntax
+ * - Normalizes list markers
+ * - Fixes spacing around headers
+ * - Removes base64 images (replaces with placeholder)
+ */
+function normalizeMarkdown(content: string): string {
+  return content
+    // Ensure space after # for headers
+    .replace(/^(#{1,6})([^\s#])/gm, '$1 $2')
+    // Normalize list markers (convert * to -)
+    .replace(/^(\s*)\*[ \t]/gm, '$1- ')
+    // Ensure blank line before headers
+    .replace(/([^\n])\n(#{1,6}\s)/g, '$1\n\n$2')
+    // Remove excessive blank lines (max 2)
+    .replace(/\n{4,}/g, '\n\n\n')
+    .trim();
+}
+
+export function extractEmbeddedImages(content: string): { content: string; images: EmbeddedImage[] } {
+  const images: EmbeddedImage[] = [];
+  let imageIndex = 1;
+
+  const withMarkdownImages = content.replace(
+    /!\[([^\]]*)\]\((data:image\/([^;]+);base64,([A-Za-z0-9+/=]+))\)/g,
+    (_match, altText: string, dataUri: string, mimeSubType: string, base64Data: string) => {
+      const id = `IMG${String(imageIndex++).padStart(4, '0')}`;
+      const estimatedSize = Math.round(base64Data.length * 0.75);
+      const mimeType = `image/${mimeSubType}`;
+      const alt = (altText || '').trim();
+      images.push({ id, alt, mimeType, estimatedSize, dataUri });
+      return `\n[图片引用 ${id} | alt="${alt || '无'}" | ${mimeType} | ${estimatedSize} bytes]\n`;
+    }
+  );
+
+  const withHtmlImages = withMarkdownImages.replace(
+    /<img[^>]*src="(data:image\/([^;]+);base64,([A-Za-z0-9+/=]+))"[^>]*>/gi,
+    (match, dataUri: string, mimeSubType: string, base64Data: string) => {
+      const altMatch = match.match(/\salt="([^"]*)"/i);
+      const alt = altMatch?.[1]?.trim() || '';
+      const id = `IMG${String(imageIndex++).padStart(4, '0')}`;
+      const estimatedSize = Math.round(base64Data.length * 0.75);
+      const mimeType = `image/${mimeSubType}`;
+      images.push({ id, alt, mimeType, estimatedSize, dataUri });
+      return `\n[图片引用 ${id} | alt="${alt || '无'}" | ${mimeType} | ${estimatedSize} bytes]\n`;
+    }
+  );
+
+  return {
+    content: withHtmlImages,
+    images,
+  };
+}
+
+/**
+ * Check if content contains base64 images and estimate their size
+ */
+export function analyzeBase64Images(content: string): { count: number; estimatedSize: number } {
+  const base64Pattern = /data:image\/[^;]+;base64,([A-Za-z0-9+/=]+)/g;
+  let match;
+  let count = 0;
+  let totalLength = 0;
+  
+  while ((match = base64Pattern.exec(content)) !== null) {
+    count++;
+    totalLength += match[1].length;
+  }
+  
+  // Base64 is ~4/3 of binary size, so multiply by 0.75 to get approximate binary size
+  const estimatedSize = Math.round(totalLength * 0.75);
+  
+  return { count, estimatedSize };
+}
+
+async function parseHtmlToMarkdown(buffer: Buffer): Promise<ParsedDocument> {
+  const html = buffer.toString('utf-8');
+  
+  // Convert HTML to Markdown with structure preservation
+  const markdown = convertHtmlToMarkdown(html);
+  
+  const extracted = extractEmbeddedImages(markdown);
+  return {
+    content: extracted.content,
+    format: 'markdown',
+    images: extracted.images,
+    metadata: {
+      wordCount: markdown.split(/\s+/).length,
+    },
+  };
+}
+
+/**
+ * Convert HTML to Markdown while preserving document structure
+ */
+function convertHtmlToMarkdown(html: string): string {
+  let md = html;
+  
+  // Remove script and style tags with content
+  md = md.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
+  md = md.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '');
+  
+  // Convert headings
+  md = md.replace(/<h1[^>]*>([\s\S]*?)<\/h1>/gi, '\n\n# $1\n\n');
+  md = md.replace(/<h2[^>]*>([\s\S]*?)<\/h2>/gi, '\n\n## $1\n\n');
+  md = md.replace(/<h3[^>]*>([\s\S]*?)<\/h3>/gi, '\n\n### $1\n\n');
+  md = md.replace(/<h4[^>]*>([\s\S]*?)<\/h4>/gi, '\n\n#### $1\n\n');
+  md = md.replace(/<h5[^>]*>([\s\S]*?)<\/h5>/gi, '\n\n##### $1\n\n');
+  md = md.replace(/<h6[^>]*>([\s\S]*?)<\/h6>/gi, '\n\n###### $1\n\n');
+  
+  // Convert paragraphs
+  md = md.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '\n\n$1\n\n');
+  
+  // Convert line breaks
+  md = md.replace(/<br\s*\/?>/gi, '\n');
+  
+  // Convert strong/b and em/i
+  md = md.replace(/<(strong|b)[^>]*>([\s\S]*?)<\/(strong|b)>/gi, '**$2**');
+  md = md.replace(/<(em|i)[^>]*>([\s\S]*?)<\/(em|i)>/gi, '*$2*');
+  
+  // Convert code
+  md = md.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`');
+  md = md.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '\n\n```\n$1\n```\n\n');
+  
+  // Convert unordered lists
+  md = md.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, (match, content) => {
+    return '\n\n' + content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n') + '\n';
+  });
+  
+  // Convert ordered lists
+  md = md.replace(/<ol[^>]*>([\s\S]*?)<\/ol>/gi, (match, content) => {
+    let index = 1;
+    return '\n\n' + content.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, itemContent: string) => `${index++}. ${itemContent.trim()}\n`) + '\n';
+  });
+  
+  // Convert tables
+  md = md.replace(/<table[^>]*>([\s\S]*?)<\/table>/gi, (match, content) => {
+    let tableMd = '\n\n';
+    const rows = content.match(/<tr[^>]*>([\s\S]*?)<\/tr>/gi) || [];
+    
+    rows.forEach((row: string, rowIndex: number) => {
+      const cells = row.match(/<t[dh][^>]*>([\s\S]*?)<\/t[dh]>/gi) || [];
+      const cellContents = cells.map((cell: string) => 
+        cell.replace(/<[^>]+>/g, '').trim()
+      );
+      
+      if (cellContents.length > 0) {
+        tableMd += '| ' + cellContents.join(' | ') + ' |\n';
+        // Add separator after header row
+        if (rowIndex === 0) {
+          tableMd += '|' + cellContents.map(() => ' --- |').join('') + '\n';
+        }
+      }
+    });
+    
+    return tableMd + '\n';
+  });
+  
+  // Convert links
+  md = md.replace(/<a[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi, '[$2]($1)');
+  
+  // Convert images
+  md = md.replace(/<img[^>]+src="([^"]+)"[^>]*alt="([^"]*)"[^>]*\/?>/gi, '![$2]($1)');
+  md = md.replace(/<img[^>]+alt="([^"]*)"[^>]*src="([^"]+)"[^>]*\/?>/gi, '![$1]($2)');
+  md = md.replace(/<img[^>]+src="([^"]+)"[^>]*\/?>/gi, '![]($1)');
+  
+  // Remove remaining HTML tags but keep content
+  md = md.replace(/<[^>]+>/g, '');
+  
+  // Decode HTML entities
+  md = md.replace(/&amp;/g, '&');
+  md = md.replace(/&lt;/g, '<');
+  md = md.replace(/&gt;/g, '>');
+  md = md.replace(/&quot;/g, '"');
+  md = md.replace(/&#39;/g, "'");
+  md = md.replace(/&nbsp;/g, ' ');
+  md = md.replace(/&mdash;/g, '—');
+  md = md.replace(/&ndash;/g, '–');
+  md = md.replace(/&hellip;/g, '...');
+  
+  // Clean up excessive whitespace
+  md = md.replace(/\n{4,}/g, '\n\n\n');
+  
+  return md.trim();
+}
+
+function parseEnvInt(name: string, fallback: number): number {
+  const raw = process.env[name];
+  if (!raw) return fallback;
+  const parsed = parseInt(raw, 10);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+async function extractPdfPageImages(pdfPath: string, maxPages: number): Promise<EmbeddedImage[]> {
+  if (maxPages <= 0) {
+    return [];
+  }
+
+  const tempRoot = path.join(
+    process.cwd(),
+    '.spec-agent-tmp',
+    'pdf-pages',
+    `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`
+  );
+  const outPrefix = path.join(tempRoot, 'page');
+  await fs.ensureDir(tempRoot);
+
+  try {
+    await execFileAsync('pdftoppm', [
+      '-png',
+      '-f', '1',
+      '-l', String(maxPages),
+      pdfPath,
+      outPrefix,
+    ]);
+
+    const files = await fs.readdir(tempRoot);
+    const pngFiles = files
+      .filter(f => /^page-\d+\.png$/i.test(f))
+      .sort((a, b) => {
+        const ai = parseInt(a.match(/\d+/)?.[0] || '0', 10);
+        const bi = parseInt(b.match(/\d+/)?.[0] || '0', 10);
+        return ai - bi;
+      });
+
+    const images: EmbeddedImage[] = [];
+    let imageIdx = 1;
+    for (const file of pngFiles) {
+      const imagePath = path.join(tempRoot, file);
+      const buffer = await fs.readFile(imagePath);
+      const base64 = buffer.toString('base64');
+      const pageNo = parseInt(file.match(/\d+/)?.[0] || String(imageIdx), 10);
+      images.push({
+        id: `PDFIMG${String(imageIdx++).padStart(4, '0')}`,
+        alt: `PDF第${pageNo}页`,
+        mimeType: 'image/png',
+        estimatedSize: buffer.length,
+        dataUri: `data:image/png;base64,${base64}`,
+      });
+    }
+    return images;
+  } catch {
+    // pdftoppm unavailable or conversion failed.
+    return [];
+  } finally {
+    await fs.remove(tempRoot).catch(() => undefined);
+  }
+}
+
+async function parsePdfToMarkdown(buffer: Buffer, filePath?: string): Promise<ParsedDocument> {
+  try {
+    // Dynamic import to avoid loading if not needed
+    const pdfParse = await import('pdf-parse');
+    const result = await pdfParse.default(buffer);
+    
+    // Convert PDF text to structured Markdown
+    // PDF text often has page breaks and layout artifacts that need cleaning
+    const structuredContent = structurePdfContent(result.text);
+    
+    const maxPdfImagePages = Math.max(0, parseEnvInt('PDF_IMAGE_PAGE_LIMIT', 8));
+    const pageImages = filePath
+      ? await extractPdfPageImages(filePath, Math.min(result.numpages, maxPdfImagePages))
+      : [];
+
+    const imageHeaders = pageImages.map(
+      image => `[图片引用 ${image.id} | alt="${image.alt || '无'}" | ${image.mimeType} | ${image.estimatedSize} bytes]`
+    );
+    const contentWithPageImages = imageHeaders.length > 0
+      ? `${imageHeaders.join('\n')}\n\n${structuredContent}`
+      : structuredContent;
+
+    return {
+      content: extractEmbeddedImages(contentWithPageImages).content,
+      format: 'markdown',
+      images: pageImages,
+      metadata: {
+        pages: result.numpages,
+        wordCount: result.text.split(/\s+/).length,
+      },
+    };
+  } catch (error) {
+    // Fallback: try to extract text as-is
+    const text = buffer.toString('utf-8');
+    if (text.length > 100) {
+      return {
+        content: extractEmbeddedImages(normalizeMarkdown(text)).content,
+        format: 'markdown',
+        images: [],
+        metadata: {},
+      };
+    }
+    throw new Error(`Failed to parse PDF: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+/**
+ * Structure PDF content into Markdown format
+ * PDFs often have layout artifacts that need intelligent processing
+ */
+function structurePdfContent(text: string): string {
+  let md = text;
+  
+  // Remove page number lines (standalone numbers)
+  md = md.replace(/\n\s*\d+\s*\n/g, '\n\n');
+  
+  // Detect and convert potential headers
+  // Short lines at the start of paragraphs that are all caps or title case
+  const lines = md.split('\n');
+  const processedLines: string[] = [];
+  let prevLineEmpty = true;
+  
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i].trim();
+    const nextLine = lines[i + 1]?.trim() || '';
+    
+    // Skip empty lines but track them
+    if (!line) {
+      processedLines.push('');
+      prevLineEmpty = true;
+      continue;
+    }
+    
+    // Detect headers based on various heuristics
+    const isShortLine = line.length < 100;
+    const isAllCaps = line === line.toUpperCase() && line.length > 3 && /[A-Z]/.test(line);
+    const isTitleCase = /^[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*$/.test(line) && line.length > 3;
+    const looksLikeNumberedHeader = /^\d+(?:\.\d+)*\.?\s+\S/.test(line);
+    const looksLikeChapter = /^(Chapter|CHAPTER|第[一二三四五六七八九十\d]+章)/.test(line);
+    
+    if (prevLineEmpty && isShortLine) {
+      if (looksLikeChapter) {
+        // Chapter header - H1
+        processedLines.push(`# ${line}`);
+      } else if (looksLikeNumberedHeader) {
+        // Numbered section - could be H2 or H3
+        const level = (line.match(/\./g) || []).length + 2;
+        processedLines.push(`${'#'.repeat(Math.min(level, 6))} ${line}`);
+      } else if (isAllCaps && line.length < 50) {
+        // ALL CAPS short line - likely a section header
+        processedLines.push(`## ${line}`);
+      } else if (isTitleCase && !nextLine.startsWith(line.substring(0, 10))) {
+        // Title case that doesn't continue - likely a header
+        processedLines.push(`### ${line}`);
+      } else {
+        processedLines.push(line);
+      }
+    } else {
+      processedLines.push(line);
+    }
+    
+    prevLineEmpty = false;
+  }
+  
+  md = processedLines.join('\n');
+  
+  // Clean up excessive whitespace
+  md = md.replace(/\n{4,}/g, '\n\n\n');
+  
+  return normalizeMarkdown(md);
+}
+
+async function parseDocxToMarkdown(buffer: Buffer): Promise<ParsedDocument> {
+  try {
+    // Dynamic import to avoid loading if not needed
+    const mammoth = await import('mammoth');
+    
+    // Use mammoth's HTML conversion to preserve structure, then convert to Markdown
+    const htmlResult = await mammoth.convertToHtml({ buffer }, {
+      styleMap: [
+        "p[style-name='Heading 1'] => h1:fresh",
+        "p[style-name='Heading 2'] => h2:fresh",
+        "p[style-name='Heading 3'] => h3:fresh",
+        "p[style-name='Heading 4'] => h4:fresh",
+        "p[style-name='Heading 5'] => h5:fresh",
+        "p[style-name='Heading 6'] => h6:fresh",
+        "p[style-name='Title'] => h1.title:fresh",
+        "p[style-name='Subtitle'] => h2.subtitle:fresh",
+      ]
+    });
+    
+    // Convert HTML to Markdown
+    const markdown = convertHtmlToMarkdown(htmlResult.value);
+    const extracted = extractEmbeddedImages(markdown);
+    
+    // Extract metadata from document
+    const metadata: ParsedDocument['metadata'] = {
+      wordCount: markdown.split(/\s+/).length,
+    };
+    
+    // Try to extract title from the first heading
+    const titleMatch = markdown.match(/^#\s+(.+)$/m);
+    if (titleMatch) {
+      metadata.title = titleMatch[1].trim();
+    }
+    
+    return {
+      content: extracted.content,
+      format: 'markdown',
+      images: extracted.images,
+      metadata,
+    };
+  } catch (error) {
+    // Fallback: try raw text extraction
+    try {
+      const mammoth = await import('mammoth');
+      const result = await mammoth.extractRawText({ buffer });
+      return {
+        content: extractEmbeddedImages(normalizeMarkdown(result.value)).content,
+        format: 'markdown',
+        images: [],
+        metadata: {
+          wordCount: result.value.split(/\s+/).length,
+        },
+      };
+    } catch {
+      // Last resort: read as plain text
+      const text = buffer.toString('utf-8');
+      if (text.length > 100) {
+        return {
+          content: extractEmbeddedImages(normalizeMarkdown(text)).content,
+          format: 'markdown',
+          images: [],
+          metadata: {},
+        };
+      }
+    }
+    throw new Error(`Failed to parse DOCX: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+
+/**
+ * Read and concatenate multiple document files
+ * All content is normalized to Markdown format
+ */
+export async function readChunkContent(filePaths: string[]): Promise<string> {
+  const contents: string[] = [];
+  
+  for (const filePath of filePaths) {
+    try {
+      const parsed = await parseDocument(filePath);
+      contents.push(`=== ${path.basename(filePath)} ===\n${parsed.content}`);
+    } catch (error) {
+      // If parsing fails, try to read as plain text
+      try {
+        const text = await fs.readFile(filePath, 'utf-8');
+        contents.push(`=== ${path.basename(filePath)} ===\n${normalizeMarkdown(text)}`);
+      } catch {
+        contents.push(`=== ${path.basename(filePath)} ===\n[Error reading file: ${error instanceof Error ? error.message : String(error)}]`);
+      }
+    }
+  }
+  
+  return contents.join('\n\n---\n\n');
+}
+
+/**
+ * Check if a file format is supported
+ */
+export function isSupportedFormat(filePath: string): boolean {
+  const ext = path.extname(filePath).toLowerCase();
+  return ['.md', '.txt', '.html', '.htm', '.pdf', '.docx'].includes(ext);
+}