skill-seekers 2.7.3__py3-none-any.whl

skill_seekers/cli/codebase_scraper.py

@@ -0,0 +1,1225 @@
+#!/usr/bin/env python3
+"""
+Codebase Scraper CLI Tool
+
+Standalone tool for analyzing local codebases without GitHub API.
+Extracts code signatures, comments, and optionally generates API documentation.
+
+Usage:
+    codebase-scraper --directory /path/to/repo --output output/codebase/
+    codebase-scraper --directory . --depth deep --languages Python,JavaScript
+    codebase-scraper --directory /path/to/repo --build-api-reference
+
+Features:
+    - File tree walking with .gitignore support
+    - Multi-language code analysis (9 languages: Python, JavaScript/TypeScript, C/C++, C#, Go, Rust, Java, Ruby, PHP)
+    - API reference generation
+    - Comment extraction
+    - Dependency graph analysis
+    - Configurable depth levels
+
+Credits:
+    - Language parsing patterns inspired by official language specifications
+    - NetworkX for dependency graph analysis: https://networkx.org/
+    - pathspec for .gitignore support: https://pypi.org/project/pathspec/
+"""
+
+import argparse
+import json
+import logging
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+# Add parent directory to path for imports
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from skill_seekers.cli.api_reference_builder import APIReferenceBuilder
+from skill_seekers.cli.code_analyzer import CodeAnalyzer
+from skill_seekers.cli.config_extractor import ConfigExtractor
+from skill_seekers.cli.dependency_analyzer import DependencyAnalyzer
+
+# Try to import pathspec for .gitignore support
+try:
+    import pathspec
+
+    PATHSPEC_AVAILABLE = True
+except ImportError:
+    PATHSPEC_AVAILABLE = False
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+# Language extension mapping
+LANGUAGE_EXTENSIONS = {
+    ".py": "Python",
+    ".js": "JavaScript",
+    ".jsx": "JavaScript",
+    ".ts": "TypeScript",
+    ".tsx": "TypeScript",
+    ".cpp": "C++",
+    ".cc": "C++",
+    ".cxx": "C++",
+    ".h": "C++",
+    ".hpp": "C++",
+    ".hxx": "C++",
+    ".c": "C",
+    ".cs": "C#",
+    ".go": "Go",
+    ".rs": "Rust",
+    ".java": "Java",
+    ".rb": "Ruby",
+    ".php": "PHP",
+}
+
+# Default directories to exclude
+DEFAULT_EXCLUDED_DIRS = {
+    "node_modules",
+    "venv",
+    "__pycache__",
+    ".git",
+    ".svn",
+    ".hg",
+    "build",
+    "dist",
+    "target",
+    ".pytest_cache",
+    ".tox",
+    ".mypy_cache",
+    "htmlcov",
+    "coverage",
+    ".coverage",
+    ".eggs",
+    "*.egg-info",
+    ".idea",
+    ".vscode",
+    ".vs",
+    "__pypackages__",
+}
+
+
+def detect_language(file_path: Path) -> str:
+    """
+    Detect programming language from file extension.
+
+    Args:
+        file_path: Path to source file
+
+    Returns:
+        Language name or 'Unknown'
+    """
+    extension = file_path.suffix.lower()
+    return LANGUAGE_EXTENSIONS.get(extension, "Unknown")
+
+
+def load_gitignore(directory: Path) -> pathspec.PathSpec | None:
+    """
+    Load .gitignore file and create pathspec matcher.
+
+    Args:
+        directory: Root directory to search for .gitignore
+
+    Returns:
+        PathSpec object if .gitignore found, None otherwise
+    """
+    if not PATHSPEC_AVAILABLE:
+        logger.warning("pathspec not installed - .gitignore support disabled")
+        logger.warning("Install with: pip install pathspec")
+        return None
+
+    gitignore_path = directory / ".gitignore"
+    if not gitignore_path.exists():
+        logger.debug(f"No .gitignore found in {directory}")
+        return None
+
+    try:
+        with open(gitignore_path, encoding="utf-8") as f:
+            spec = pathspec.PathSpec.from_lines("gitwildmatch", f)
+        logger.info(f"Loaded .gitignore from {gitignore_path}")
+        return spec
+    except Exception as e:
+        logger.warning(f"Failed to load .gitignore: {e}")
+        return None
+
+
+def should_exclude_dir(dir_name: str, excluded_dirs: set) -> bool:
+    """
+    Check if directory should be excluded from analysis.
+
+    Args:
+        dir_name: Directory name
+        excluded_dirs: Set of directory names to exclude
+
+    Returns:
+        True if directory should be excluded
+    """
+    return dir_name in excluded_dirs
+
+
+def walk_directory(
+    root: Path,
+    patterns: list[str] | None = None,
+    gitignore_spec: pathspec.PathSpec | None = None,
+    excluded_dirs: set | None = None,
+) -> list[Path]:
+    """
+    Walk directory tree and collect source files.
+
+    Args:
+        root: Root directory to walk
+        patterns: Optional file patterns to include (e.g., ['*.py', '*.js'])
+        gitignore_spec: Optional PathSpec object for .gitignore rules
+        excluded_dirs: Set of directory names to exclude
+
+    Returns:
+        List of source file paths
+    """
+    if excluded_dirs is None:
+        excluded_dirs = DEFAULT_EXCLUDED_DIRS
+
+    files = []
+    root = Path(root).resolve()
+
+    for dirpath, dirnames, filenames in os.walk(root):
+        current_dir = Path(dirpath)
+
+        # Filter out excluded directories (in-place modification)
+        dirnames[:] = [d for d in dirnames if not should_exclude_dir(d, excluded_dirs)]
+
+        for filename in filenames:
+            file_path = current_dir / filename
+
+            # Check .gitignore rules
+            if gitignore_spec:
+                try:
+                    rel_path = file_path.relative_to(root)
+                    if gitignore_spec.match_file(str(rel_path)):
+                        logger.debug(f"Skipping (gitignore): {rel_path}")
+                        continue
+                except ValueError:
+                    # File is outside root, skip it
+                    continue
+
+            # Check file extension
+            if file_path.suffix.lower() not in LANGUAGE_EXTENSIONS:
+                continue
+
+            # Check file patterns if provided
+            if patterns and not any(file_path.match(pattern) for pattern in patterns):
+                continue
+
+            files.append(file_path)
+
+    return sorted(files)
+
+
+def analyze_codebase(
+    directory: Path,
+    output_dir: Path,
+    depth: str = "deep",
+    languages: list[str] | None = None,
+    file_patterns: list[str] | None = None,
+    build_api_reference: bool = True,
+    extract_comments: bool = True,
+    build_dependency_graph: bool = True,
+    detect_patterns: bool = True,
+    extract_test_examples: bool = True,
+    build_how_to_guides: bool = True,
+    extract_config_patterns: bool = True,
+    enhance_with_ai: bool = True,
+    ai_mode: str = "auto",
+) -> dict[str, Any]:
+    """
+    Analyze local codebase and extract code knowledge.
+
+    Args:
+        directory: Directory to analyze
+        output_dir: Output directory for results
+        depth: Analysis depth (surface, deep, full)
+        languages: Optional list of languages to analyze
+        file_patterns: Optional file patterns to include
+        build_api_reference: Generate API reference markdown
+        extract_comments: Extract inline comments
+        build_dependency_graph: Generate dependency graph and detect circular dependencies
+        detect_patterns: Detect design patterns (Singleton, Factory, Observer, etc.)
+        extract_test_examples: Extract usage examples from test files
+        build_how_to_guides: Build how-to guides from workflow examples (C3.3)
+        extract_config_patterns: Extract configuration patterns from config files (C3.4)
+        enhance_with_ai: Enhance patterns and examples with AI analysis (C3.6)
+        ai_mode: AI enhancement mode for how-to guides (auto, api, local, none)
+
+    Returns:
+        Analysis results dictionary
+    """
+    # Resolve directory to absolute path to avoid relative_to() errors
+    directory = Path(directory).resolve()
+
+    logger.info(f"Analyzing codebase: {directory}")
+    logger.info(f"Depth: {depth}")
+
+    # Create output directory
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    # Load .gitignore
+    gitignore_spec = load_gitignore(directory)
+
+    # Walk directory tree
+    logger.info("Scanning directory tree...")
+    files = walk_directory(directory, patterns=file_patterns, gitignore_spec=gitignore_spec)
+
+    logger.info(f"Found {len(files)} source files")
+
+    # Filter by language if specified
+    if languages:
+        language_set = set(languages)
+        files = [f for f in files if detect_language(f) in language_set]
+        logger.info(f"Filtered to {len(files)} files for languages: {', '.join(languages)}")
+
+    # Initialize code analyzer
+    analyzer = CodeAnalyzer(depth=depth)
+
+    # Analyze each file
+    results = {"files": []}
+    analyzed_count = 0
+
+    for file_path in files:
+        try:
+            content = file_path.read_text(encoding="utf-8", errors="ignore")
+            language = detect_language(file_path)
+
+            if language == "Unknown":
+                continue
+
+            # Analyze file
+            analysis = analyzer.analyze_file(str(file_path), content, language)
+
+            # Only include files with actual analysis results
+            if analysis and (analysis.get("classes") or analysis.get("functions")):
+                results["files"].append(
+                    {
+                        "file": str(file_path.relative_to(directory)),
+                        "language": language,
+                        **analysis,
+                    }
+                )
+                analyzed_count += 1
+
+                if analyzed_count % 10 == 0:
+                    logger.info(f"Analyzed {analyzed_count}/{len(files)} files...")
+
+        except Exception as e:
+            logger.warning(f"Error analyzing {file_path}: {e}")
+            continue
+
+    logger.info(f"✅ Successfully analyzed {analyzed_count} files")
+
+    # Save results
+    output_json = output_dir / "code_analysis.json"
+    with open(output_json, "w", encoding="utf-8") as f:
+        json.dump(results, f, indent=2)
+
+    logger.info(f"📁 Saved analysis to: {output_json}")
+
+    # Build API reference if requested
+    if build_api_reference and results["files"]:
+        logger.info("Building API reference documentation...")
+        builder = APIReferenceBuilder(results)
+        api_output_dir = output_dir / "api_reference"
+        generated_files = builder.build_reference(api_output_dir)
+        logger.info(f"✅ Generated {len(generated_files)} API reference files")
+        logger.info(f"📁 API reference: {api_output_dir}")
+
+    # Build dependency graph if requested (C2.6)
+    if build_dependency_graph:
+        logger.info("Building dependency graph...")
+        dep_analyzer = DependencyAnalyzer()
+
+        # Analyze dependencies for all files
+        for file_path in files:
+            try:
+                content = file_path.read_text(encoding="utf-8", errors="ignore")
+                language = detect_language(file_path)
+
+                if language != "Unknown":
+                    # Use relative path from directory for better graph readability
+                    rel_path = str(file_path.relative_to(directory))
+                    dep_analyzer.analyze_file(rel_path, content, language)
+            except Exception as e:
+                logger.warning(f"Error analyzing dependencies for {file_path}: {e}")
+                continue
+
+        # Build the graph
+        graph = dep_analyzer.build_graph()
+
+        # Detect circular dependencies
+        cycles = dep_analyzer.detect_cycles()
+        if cycles:
+            logger.warning(f"⚠️  Found {len(cycles)} circular dependencies:")
+            for i, cycle in enumerate(cycles[:5], 1):  # Show first 5
+                cycle_str = " → ".join(cycle) + f" → {cycle[0]}"
+                logger.warning(f"  {i}. {cycle_str}")
+            if len(cycles) > 5:
+                logger.warning(f"  ... and {len(cycles) - 5} more")
+        else:
+            logger.info("✅ No circular dependencies found")
+
+        # Save dependency graph data
+        dep_output_dir = output_dir / "dependencies"
+        dep_output_dir.mkdir(parents=True, exist_ok=True)
+
+        # Export as JSON
+        dep_json = dep_output_dir / "dependency_graph.json"
+        with open(dep_json, "w", encoding="utf-8") as f:
+            json.dump(dep_analyzer.export_json(), f, indent=2)
+        logger.info(f"📁 Saved dependency graph: {dep_json}")
+
+        # Export as Mermaid diagram
+        mermaid_file = dep_output_dir / "dependency_graph.mmd"
+        mermaid_file.write_text(dep_analyzer.export_mermaid())
+        logger.info(f"📁 Saved Mermaid diagram: {mermaid_file}")
+
+        # Save statistics
+        stats = dep_analyzer.get_statistics()
+        stats_file = dep_output_dir / "statistics.json"
+        with open(stats_file, "w", encoding="utf-8") as f:
+            json.dump(stats, f, indent=2)
+        logger.info(
+            f"📊 Statistics: {stats['total_files']} files, "
+            f"{stats['total_dependencies']} dependencies, "
+            f"{stats['circular_dependencies']} cycles"
+        )
+
+        # Try to export as DOT (requires pydot)
+        try:
+            dot_file = dep_output_dir / "dependency_graph.dot"
+            dep_analyzer.export_dot(str(dot_file))
+        except Exception:
+            pass  # pydot not installed, skip DOT export
+
+    # Detect design patterns if requested (C3.1)
+    if detect_patterns:
+        logger.info("Detecting design patterns...")
+        from skill_seekers.cli.pattern_recognizer import PatternRecognizer
+
+        pattern_recognizer = PatternRecognizer(depth=depth, enhance_with_ai=enhance_with_ai)
+        pattern_results = []
+
+        for file_path in files:
+            try:
+                content = file_path.read_text(encoding="utf-8", errors="ignore")
+                language = detect_language(file_path)
+
+                if language != "Unknown":
+                    report = pattern_recognizer.analyze_file(str(file_path), content, language)
+
+                    if report.patterns:
+                        pattern_results.append(report.to_dict())
+            except Exception as e:
+                logger.warning(f"Pattern detection failed for {file_path}: {e}")
+                continue
+
+        # Save pattern results
+        if pattern_results:
+            pattern_output = output_dir / "patterns"
+            pattern_output.mkdir(parents=True, exist_ok=True)
+
+            pattern_json = pattern_output / "detected_patterns.json"
+            with open(pattern_json, "w", encoding="utf-8") as f:
+                json.dump(pattern_results, f, indent=2)
+
+            total_patterns = sum(len(r["patterns"]) for r in pattern_results)
+            logger.info(f"✅ Detected {total_patterns} patterns in {len(pattern_results)} files")
+            logger.info(f"📁 Saved to: {pattern_json}")
+        else:
+            logger.info("No design patterns detected")
+
+    # Extract test examples if requested (C3.2)
+    if extract_test_examples:
+        logger.info("Extracting usage examples from test files...")
+        from skill_seekers.cli.test_example_extractor import TestExampleExtractor
+
+        # Create extractor
+        test_extractor = TestExampleExtractor(
+            min_confidence=0.5,
+            max_per_file=10,
+            languages=languages,
+            enhance_with_ai=enhance_with_ai,
+        )
+
+        # Extract examples from directory
+        try:
+            example_report = test_extractor.extract_from_directory(directory, recursive=True)
+
+            if example_report.total_examples > 0:
+                # Save results
+                examples_output = output_dir / "test_examples"
+                examples_output.mkdir(parents=True, exist_ok=True)
+
+                # Save as JSON
+                examples_json = examples_output / "test_examples.json"
+                with open(examples_json, "w", encoding="utf-8") as f:
+                    json.dump(example_report.to_dict(), f, indent=2)
+
+                # Save as Markdown
+                examples_md = examples_output / "test_examples.md"
+                examples_md.write_text(example_report.to_markdown(), encoding="utf-8")
+
+                logger.info(
+                    f"✅ Extracted {example_report.total_examples} test examples "
+                    f"({example_report.high_value_count} high-value)"
+                )
+                logger.info(f"📁 Saved to: {examples_output}")
+            else:
+                logger.info("No test examples extracted")
+
+        except Exception as e:
+            logger.warning(f"Test example extraction failed: {e}")
+            example_report = None
+
+    # Build how-to guides from workflow examples (C3.3)
+    if build_how_to_guides and extract_test_examples:
+        logger.info("Building how-to guides from workflow examples...")
+        try:
+            from skill_seekers.cli.how_to_guide_builder import HowToGuideBuilder
+
+            # Create guide builder
+            guide_builder = HowToGuideBuilder(enhance_with_ai=enhance_with_ai)
+
+            # Build guides from workflow examples
+            tutorials_dir = output_dir / "tutorials"
+
+            # Get workflow examples from the example_report if available
+            if (
+                "example_report" in locals()
+                and example_report
+                and example_report.total_examples > 0
+            ):
+                # Convert example_report to list of dicts for processing
+                examples_list = example_report.to_dict().get("examples", [])
+
+                guide_collection = guide_builder.build_guides_from_examples(
+                    examples_list,
+                    grouping_strategy="ai-tutorial-group",
+                    output_dir=tutorials_dir,
+                    enhance_with_ai=enhance_with_ai,
+                    ai_mode=ai_mode,
+                )
+
+                if guide_collection and guide_collection.total_guides > 0:
+                    # Save collection summary
+                    collection_json = tutorials_dir / "guide_collection.json"
+                    with open(collection_json, "w", encoding="utf-8") as f:
+                        json.dump(guide_collection.to_dict(), f, indent=2)
+
+                    logger.info(f"✅ Built {guide_collection.total_guides} how-to guides")
+                    logger.info(f"📁 Saved to: {tutorials_dir}")
+                else:
+                    logger.info("No how-to guides generated (insufficient workflow examples)")
+            else:
+                logger.info("No workflow examples available for guide generation")
+
+        except Exception as e:
+            logger.warning(f"How-to guide building failed: {e}")
+
+    # Extract configuration patterns (C3.4)
+    if extract_config_patterns:
+        logger.info("Extracting configuration patterns...")
+        try:
+            config_extractor = ConfigExtractor()
+
+            # Extract config patterns from directory
+            extraction_result = config_extractor.extract_from_directory(directory)
+
+            if extraction_result.config_files:
+                # Convert to dict for enhancement
+                result_dict = config_extractor.to_dict(extraction_result)
+
+                # AI Enhancement (if enabled)
+                if enhance_with_ai and ai_mode != "none":
+                    try:
+                        from skill_seekers.cli.config_enhancer import ConfigEnhancer
+
+                        logger.info(f"🤖 Enhancing config analysis with AI (mode: {ai_mode})...")
+                        enhancer = ConfigEnhancer(mode=ai_mode)
+                        result_dict = enhancer.enhance_config_result(result_dict)
+                        logger.info("✅ AI enhancement complete")
+                    except Exception as e:
+                        logger.warning(f"⚠️  Config AI enhancement failed: {e}")
+
+                # Save results
+                config_output = output_dir / "config_patterns"
+                config_output.mkdir(parents=True, exist_ok=True)
+
+                # Save as JSON
+                config_json = config_output / "config_patterns.json"
+                with open(config_json, "w", encoding="utf-8") as f:
+                    json.dump(result_dict, f, indent=2)
+
+                # Save as Markdown (basic - AI enhancements in JSON only for now)
+                config_md = config_output / "config_patterns.md"
+                config_md.write_text(extraction_result.to_markdown(), encoding="utf-8")
+
+                # Count total settings across all files
+                total_settings = sum(len(cf.settings) for cf in extraction_result.config_files)
+                total_patterns = sum(len(cf.patterns) for cf in extraction_result.config_files)
+
+                logger.info(
+                    f"✅ Extracted {len(extraction_result.config_files)} config files "
+                    f"with {total_settings} settings and {total_patterns} detected patterns"
+                )
+
+                if "ai_enhancements" in result_dict:
+                    insights = result_dict["ai_enhancements"].get("overall_insights", {})
+                    if insights.get("security_issues_found"):
+                        logger.info(
+                            f"🔐 Security issues found: {insights['security_issues_found']}"
+                        )
+
+                logger.info(f"📁 Saved to: {config_output}")
+            else:
+                logger.info("No configuration files found")
+
+        except Exception as e:
+            logger.warning(f"Config pattern extraction failed: {e}")
+
+    # Detect architectural patterns (C3.7)
+    # Always run this - it provides high-level overview
+    logger.info("Analyzing architectural patterns...")
+    from skill_seekers.cli.architectural_pattern_detector import ArchitecturalPatternDetector
+
+    arch_detector = ArchitecturalPatternDetector(enhance_with_ai=enhance_with_ai)
+    arch_report = arch_detector.analyze(directory, results["files"])
+
+    if arch_report.patterns:
+        arch_output = output_dir / "architecture"
+        arch_output.mkdir(parents=True, exist_ok=True)
+
+        # Save as JSON
+        arch_json = arch_output / "architectural_patterns.json"
+        with open(arch_json, "w", encoding="utf-8") as f:
+            json.dump(arch_report.to_dict(), f, indent=2)
+
+        logger.info(f"🏗️  Detected {len(arch_report.patterns)} architectural patterns")
+        for pattern in arch_report.patterns:
+            logger.info(f"   - {pattern.pattern_name} (confidence: {pattern.confidence:.2f})")
+        logger.info(f"📁 Saved to: {arch_json}")
+    else:
+        logger.info("No clear architectural patterns detected")
+
+    # Generate SKILL.md and references/ directory
+    logger.info("Generating SKILL.md and references...")
+    _generate_skill_md(
+        output_dir=output_dir,
+        directory=directory,
+        results=results,
+        depth=depth,
+        build_api_reference=build_api_reference,
+        build_dependency_graph=build_dependency_graph,
+        detect_patterns=detect_patterns,
+        extract_test_examples=extract_test_examples,
+        extract_config_patterns=extract_config_patterns,
+    )
+
+    return results
+
+
+def _generate_skill_md(
+    output_dir: Path,
+    directory: Path,
+    results: dict[str, Any],
+    depth: str,
+    build_api_reference: bool,
+    build_dependency_graph: bool,
+    detect_patterns: bool,
+    extract_test_examples: bool,
+    extract_config_patterns: bool,
+):
+    """
+    Generate rich SKILL.md from codebase analysis results.
+
+    Creates a 300+ line skill file with:
+    - Front matter (name, description)
+    - Repository info (path, languages, file count)
+    - When to Use section
+    - Quick Reference (patterns, languages, stats)
+    - Code Examples (from test files)
+    - API Reference (from code analysis)
+    - Architecture Overview
+    - Configuration Patterns
+    - Available References
+    """
+    repo_name = directory.name
+
+    # Generate skill name (lowercase, hyphens only, max 64 chars)
+    skill_name = repo_name.lower().replace("_", "-").replace(" ", "-")[:64]
+
+    # Generate description
+    description = f"Local codebase analysis for {repo_name}"
+
+    # Count files by language
+    language_stats = _get_language_stats(results.get("files", []))
+    total_files = len(results.get("files", []))
+
+    # Start building content
+    skill_content = f"""---
+name: {skill_name}
+description: {description}
+---
+
+# {repo_name} Codebase
+
+## Description
+
+Local codebase analysis and documentation generated from code analysis.
+
+**Path:** `{directory}`
+**Files Analyzed:** {total_files}
+**Languages:** {", ".join(language_stats.keys())}
+**Analysis Depth:** {depth}
+
+## When to Use This Skill
+
+Use this skill when you need to:
+- Understand the codebase architecture and design patterns
+- Find implementation examples and usage patterns
+- Review API documentation extracted from code
+- Check configuration patterns and best practices
+- Explore test examples and real-world usage
+- Navigate the codebase structure efficiently
+
+## ⚡ Quick Reference
+
+### Codebase Statistics
+
+"""
+
+    # Language breakdown
+    skill_content += "**Languages:**\n"
+    for lang, count in sorted(language_stats.items(), key=lambda x: x[1], reverse=True):
+        percentage = (count / total_files * 100) if total_files > 0 else 0
+        skill_content += f"- **{lang}**: {count} files ({percentage:.1f}%)\n"
+    skill_content += "\n"
+
+    # Analysis features performed
+    skill_content += "**Analysis Performed:**\n"
+    if build_api_reference:
+        skill_content += "- ✅ API Reference (C2.5)\n"
+    if build_dependency_graph:
+        skill_content += "- ✅ Dependency Graph (C2.6)\n"
+    if detect_patterns:
+        skill_content += "- ✅ Design Patterns (C3.1)\n"
+    if extract_test_examples:
+        skill_content += "- ✅ Test Examples (C3.2)\n"
+    if extract_config_patterns:
+        skill_content += "- ✅ Configuration Patterns (C3.4)\n"
+    skill_content += "- ✅ Architectural Analysis (C3.7)\n\n"
+
+    # Add design patterns if available
+    if detect_patterns:
+        patterns_content = _format_patterns_section(output_dir)
+        if patterns_content:
+            skill_content += patterns_content
+
+    # Add code examples if available
+    if extract_test_examples:
+        examples_content = _format_examples_section(output_dir)
+        if examples_content:
+            skill_content += examples_content
+
+    # Add API reference if available
+    if build_api_reference:
+        api_content = _format_api_section(output_dir)
+        if api_content:
+            skill_content += api_content
+
+    # Add architecture if available
+    arch_content = _format_architecture_section(output_dir)
+    if arch_content:
+        skill_content += arch_content
+
+    # Add configuration patterns if available
+    if extract_config_patterns:
+        config_content = _format_config_section(output_dir)
+        if config_content:
+            skill_content += config_content
+
+    # Available references
+    skill_content += "## 📚 Available References\n\n"
+    skill_content += "This skill includes detailed reference documentation:\n\n"
+
+    refs_added = False
+    if build_api_reference and (output_dir / "api_reference").exists():
+        skill_content += (
+            "- **API Reference**: `references/api_reference/` - Complete API documentation\n"
+        )
+        refs_added = True
+    if build_dependency_graph and (output_dir / "dependencies").exists():
+        skill_content += (
+            "- **Dependencies**: `references/dependencies/` - Dependency graph and analysis\n"
+        )
+        refs_added = True
+    if detect_patterns and (output_dir / "patterns").exists():
+        skill_content += "- **Patterns**: `references/patterns/` - Detected design patterns\n"
+        refs_added = True
+    if extract_test_examples and (output_dir / "test_examples").exists():
+        skill_content += "- **Examples**: `references/test_examples/` - Usage examples from tests\n"
+        refs_added = True
+    if extract_config_patterns and (output_dir / "config_patterns").exists():
+        skill_content += (
+            "- **Configuration**: `references/config_patterns/` - Configuration patterns\n"
+        )
+        refs_added = True
+    if (output_dir / "architecture").exists():
+        skill_content += "- **Architecture**: `references/architecture/` - Architectural patterns\n"
+        refs_added = True
+
+    if not refs_added:
+        skill_content += "No additional references generated (analysis features disabled).\n"
+
+    skill_content += "\n"
+
+    # Footer
+    skill_content += "---\n\n"
+    skill_content += "**Generated by Skill Seeker** | Codebase Analyzer with C3.x Analysis\n"
+
+    # Write SKILL.md
+    skill_path = output_dir / "SKILL.md"
+    skill_path.write_text(skill_content, encoding="utf-8")
+
+    line_count = len(skill_content.split("\n"))
+    logger.info(f"✅ Generated SKILL.md: {skill_path} ({line_count} lines)")
+
+    # Generate references/ directory structure
+    _generate_references(output_dir)
+
+
+def _get_language_stats(files: list[dict]) -> dict[str, int]:
+    """Count files by language from analysis results."""
+    stats = {}
+    for file_data in files:
+        # files is a list of dicts with 'language' key
+        lang = file_data.get("language", "Unknown")
+        if lang != "Unknown":
+            stats[lang] = stats.get(lang, 0) + 1
+    return stats
+
+
+def _format_patterns_section(output_dir: Path) -> str:
+    """Format design patterns section from patterns/detected_patterns.json."""
+    patterns_file = output_dir / "patterns" / "detected_patterns.json"
+    if not patterns_file.exists():
+        return ""
+
+    try:
+        with open(patterns_file, encoding="utf-8") as f:
+            patterns_data = json.load(f)
+    except Exception:
+        return ""
+
+    if not patterns_data:
+        return ""
+
+    # Count patterns by type (deduplicate by class, keep highest confidence)
+    pattern_counts = {}
+    by_class = {}
+
+    for pattern_file in patterns_data:
+        for pattern in pattern_file.get("patterns", []):
+            ptype = pattern.get("pattern_type", "Unknown")
+            cls = pattern.get("class_name", "")
+            confidence = pattern.get("confidence", 0)
+
+            # Skip low confidence
+            if confidence < 0.7:
+                continue
+
+            # Deduplicate by class
+            key = f"{cls}:{ptype}"
+            if key not in by_class or by_class[key]["confidence"] < confidence:
+                by_class[key] = pattern
+
+            # Count by type
+            pattern_counts[ptype] = pattern_counts.get(ptype, 0) + 1
+
+    if not pattern_counts:
+        return ""
+
+    content = "### 🎨 Design Patterns Detected\n\n"
+    content += "*From C3.1 codebase analysis (confidence > 0.7)*\n\n"
+
+    # Top 5 pattern types
+    for ptype, count in sorted(pattern_counts.items(), key=lambda x: x[1], reverse=True)[:5]:
+        content += f"- **{ptype}**: {count} instances\n"
+
+    content += f"\n*Total: {len(by_class)} high-confidence patterns*\n\n"
+    content += "*See `references/patterns/` for complete pattern analysis*\n\n"
+    return content
+
+
+def _format_examples_section(output_dir: Path) -> str:
+    """Format code examples section from test_examples/test_examples.json."""
+    examples_file = output_dir / "test_examples" / "test_examples.json"
+    if not examples_file.exists():
+        return ""
+
+    try:
+        with open(examples_file, encoding="utf-8") as f:
+            examples_data = json.load(f)
+    except Exception:
+        return ""
+
+    examples = examples_data.get("examples", [])
+    if not examples:
+        return ""
+
+    # Filter high-value examples (complexity > 0.7)
+    high_value = [ex for ex in examples if ex.get("complexity_score", 0) > 0.7]
+
+    if not high_value:
+        # If no high complexity, take any examples
+        high_value = examples[:10]
+
+    if not high_value:
+        return ""
+
+    content = "## 📝 Code Examples\n\n"
+    content += "*High-quality examples extracted from test files (C3.2)*\n\n"
+
+    # Top 10 examples
+    for ex in sorted(high_value, key=lambda x: x.get("complexity_score", 0), reverse=True)[:10]:
+        desc = ex.get("description", "Example")
+        lang = ex.get("language", "python").lower()
+        code = ex.get("code", "")
+        complexity = ex.get("complexity_score", 0)
+
+        content += f"**{desc}** (complexity: {complexity:.2f})\n\n"
+        content += f"```{lang}\n{code}\n```\n\n"
+
+    content += "*See `references/test_examples/` for all extracted examples*\n\n"
+    return content
+
+
+def _format_api_section(output_dir: Path) -> str:
+    """Format API reference section."""
+    api_dir = output_dir / "api_reference"
+    if not api_dir.exists():
+        return ""
+
+    api_md = api_dir / "api_reference.md"
+    if not api_md.exists():
+        return ""
+
+    try:
+        api_content = api_md.read_text(encoding="utf-8")
+    except Exception:
+        return ""
+
+    # Extract first section (up to 500 chars)
+    preview = api_content[:500]
+    if len(api_content) > 500:
+        preview += "..."
+
+    content = "## 🔧 API Reference\n\n"
+    content += "*Extracted from codebase analysis (C2.5)*\n\n"
+    content += preview + "\n\n"
+    content += "*See `references/api_reference/` for complete API documentation*\n\n"
+    return content
+
+
+def _format_architecture_section(output_dir: Path) -> str:
+    """Format architecture section from architecture/architectural_patterns.json."""
+    arch_file = output_dir / "architecture" / "architectural_patterns.json"
+    if not arch_file.exists():
+        return ""
+
+    try:
+        with open(arch_file, encoding="utf-8") as f:
+            arch_data = json.load(f)
+    except Exception:
+        return ""
+
+    patterns = arch_data.get("patterns", [])
+    if not patterns:
+        return ""
+
+    content = "## 🏗️ Architecture Overview\n\n"
+    content += "*From C3.7 architectural analysis*\n\n"
+
+    content += "**Detected Architectural Patterns:**\n\n"
+    for pattern in patterns[:5]:
+        name = pattern.get("pattern_name", "Unknown")
+        confidence = pattern.get("confidence", 0)
+        indicators = pattern.get("indicators", [])
+
+        content += f"- **{name}** (confidence: {confidence:.2f})\n"
+        if indicators:
+            content += f"  - Indicators: {', '.join(indicators[:3])}\n"
+
+    content += f"\n*Total: {len(patterns)} architectural patterns detected*\n\n"
+    content += "*See `references/architecture/` for complete architectural analysis*\n\n"
+    return content
+
+
+def _format_config_section(output_dir: Path) -> str:
+    """Format configuration patterns section."""
+    config_file = output_dir / "config_patterns" / "config_patterns.json"
+    if not config_file.exists():
+        return ""
+
+    try:
+        with open(config_file, encoding="utf-8") as f:
+            config_data = json.load(f)
+    except Exception:
+        return ""
+
+    config_files = config_data.get("config_files", [])
+    if not config_files:
+        return ""
+
+    total_settings = sum(len(cf.get("settings", [])) for cf in config_files)
+    total_patterns = sum(len(cf.get("patterns", [])) for cf in config_files)
+
+    content = "## ⚙️ Configuration Patterns\n\n"
+    content += "*From C3.4 configuration analysis*\n\n"
+    content += f"**Configuration Files Analyzed:** {len(config_files)}\n"
+    content += f"**Total Settings:** {total_settings}\n"
+    content += f"**Patterns Detected:** {total_patterns}\n\n"
+
+    # List config file types found
+    file_types = {}
+    for cf in config_files:
+        ctype = cf.get("config_type", "unknown")
+        file_types[ctype] = file_types.get(ctype, 0) + 1
+
+    if file_types:
+        content += "**Configuration Types:**\n"
+        for ctype, count in sorted(file_types.items(), key=lambda x: x[1], reverse=True):
+            content += f"- {ctype}: {count} files\n"
+        content += "\n"
+
+    content += "*See `references/config_patterns/` for detailed configuration analysis*\n\n"
+    return content
+
+
+def _generate_references(output_dir: Path):
+    """
+    Generate references/ directory structure by symlinking analysis output.
+
+    Creates a clean references/ directory that links to all analysis outputs.
+    """
+    references_dir = output_dir / "references"
+    references_dir.mkdir(exist_ok=True)
+
+    # Map analysis directories to reference names
+    mappings = {
+        "api_reference": "api_reference",
+        "dependencies": "dependencies",
+        "patterns": "patterns",
+        "test_examples": "test_examples",
+        "tutorials": "tutorials",
+        "config_patterns": "config_patterns",
+        "architecture": "architecture",
+    }
+
+    for source, target in mappings.items():
+        source_dir = output_dir / source
+        target_dir = references_dir / target
+
+        if source_dir.exists() and source_dir.is_dir():
+            # Copy directory to references/ (not symlink, for portability)
+            if target_dir.exists():
+                import shutil
+
+                shutil.rmtree(target_dir)
+
+            import shutil
+
+            shutil.copytree(source_dir, target_dir)
+            logger.debug(f"Copied {source} → references/{target}")
+
+    logger.info(f"✅ Generated references directory: {references_dir}")
+
+
+def main():
+    """Command-line interface for codebase analysis."""
+    parser = argparse.ArgumentParser(
+        description="Analyze local codebases and extract code knowledge",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Analyze current directory
+  codebase-scraper --directory . --output output/codebase/
+
+  # Deep analysis with API reference and dependency graph
+  codebase-scraper --directory /path/to/repo --depth deep --build-api-reference --build-dependency-graph
+
+  # Analyze only Python and JavaScript
+  codebase-scraper --directory . --languages Python,JavaScript
+
+  # Use file patterns
+  codebase-scraper --directory . --file-patterns "*.py,src/**/*.js"
+
+  # Full analysis with all features (default)
+  codebase-scraper --directory . --depth deep
+
+  # Surface analysis (fast, skip all analysis features)
+  codebase-scraper --directory . --depth surface --skip-api-reference --skip-dependency-graph --skip-patterns --skip-test-examples
+
+  # Skip specific features
+  codebase-scraper --directory . --skip-patterns --skip-test-examples
+""",
+    )
+
+    parser.add_argument("--directory", required=True, help="Directory to analyze")
+    parser.add_argument(
+        "--output", default="output/codebase/", help="Output directory (default: output/codebase/)"
+    )
+    parser.add_argument(
+        "--depth",
+        choices=["surface", "deep", "full"],
+        default="deep",
+        help="Analysis depth (default: deep)",
+    )
+    parser.add_argument(
+        "--languages", help="Comma-separated languages to analyze (e.g., Python,JavaScript,C++)"
+    )
+    parser.add_argument(
+        "--file-patterns", help="Comma-separated file patterns (e.g., *.py,src/**/*.js)"
+    )
+    parser.add_argument(
+        "--skip-api-reference",
+        action="store_true",
+        default=False,
+        help="Skip API reference markdown documentation generation (default: enabled)",
+    )
+    parser.add_argument(
+        "--skip-dependency-graph",
+        action="store_true",
+        default=False,
+        help="Skip dependency graph and circular dependency detection (default: enabled)",
+    )
+    parser.add_argument(
+        "--skip-patterns",
+        action="store_true",
+        default=False,
+        help="Skip design pattern detection (Singleton, Factory, Observer, etc.) (default: enabled)",
+    )
+    parser.add_argument(
+        "--skip-test-examples",
+        action="store_true",
+        default=False,
+        help="Skip test example extraction (instantiation, method calls, configs, etc.) (default: enabled)",
+    )
+    parser.add_argument(
+        "--skip-how-to-guides",
+        action="store_true",
+        default=False,
+        help="Skip how-to guide generation from workflow examples (default: enabled)",
+    )
+    parser.add_argument(
+        "--skip-config-patterns",
+        action="store_true",
+        default=False,
+        help="Skip configuration pattern extraction from config files (JSON, YAML, TOML, ENV, etc.) (default: enabled)",
+    )
+    parser.add_argument(
+        "--ai-mode",
+        choices=["auto", "api", "local", "none"],
+        default="auto",
+        help="AI enhancement mode for how-to guides: auto (detect best), api (Claude API), local (Claude Code CLI), none (disable) (default: auto)",
+    )
+    parser.add_argument("--no-comments", action="store_true", help="Skip comment extraction")
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose logging")
+
+    # Check for deprecated flags
+    deprecated_flags = {
+        "--build-api-reference": "--skip-api-reference",
+        "--build-dependency-graph": "--skip-dependency-graph",
+        "--detect-patterns": "--skip-patterns",
+        "--extract-test-examples": "--skip-test-examples",
+        "--build-how-to-guides": "--skip-how-to-guides",
+        "--extract-config-patterns": "--skip-config-patterns",
+    }
+
+    for old_flag, new_flag in deprecated_flags.items():
+        if old_flag in sys.argv:
+            logger.warning(
+                f"⚠️  DEPRECATED: {old_flag} is deprecated. "
+                f"All features are now enabled by default. "
+                f"Use {new_flag} to disable this feature."
+            )
+
+    args = parser.parse_args()
+
+    # Set logging level
+    if args.verbose:
+        logging.getLogger().setLevel(logging.DEBUG)
+
+    # Validate directory
+    directory = Path(args.directory)
+    if not directory.exists():
+        logger.error(f"Directory not found: {directory}")
+        return 1
+
+    if not directory.is_dir():
+        logger.error(f"Not a directory: {directory}")
+        return 1
+
+    # Parse languages
+    languages = None
+    if args.languages:
+        languages = [lang.strip() for lang in args.languages.split(",")]
+
+    # Parse file patterns
+    file_patterns = None
+    if args.file_patterns:
+        file_patterns = [p.strip() for p in args.file_patterns.split(",")]
+
+    # Analyze codebase
+    try:
+        results = analyze_codebase(
+            directory=directory,
+            output_dir=Path(args.output),
+            depth=args.depth,
+            languages=languages,
+            file_patterns=file_patterns,
+            build_api_reference=not args.skip_api_reference,
+            extract_comments=not args.no_comments,
+            build_dependency_graph=not args.skip_dependency_graph,
+            detect_patterns=not args.skip_patterns,
+            extract_test_examples=not args.skip_test_examples,
+            build_how_to_guides=not args.skip_how_to_guides,
+            extract_config_patterns=not args.skip_config_patterns,
+            enhance_with_ai=True,  # Auto-disables if no API key present
+            ai_mode=args.ai_mode,  # NEW: AI enhancement mode for how-to guides
+        )
+
+        # Print summary
+        print(f"\n{'=' * 60}")
+        print("CODEBASE ANALYSIS COMPLETE")
+        print(f"{'=' * 60}")
+        print(f"Files analyzed: {len(results['files'])}")
+        print(f"Output directory: {args.output}")
+        if not args.skip_api_reference:
+            print(f"API reference: {Path(args.output) / 'api_reference'}")
+        print(f"{'=' * 60}\n")
+
+        return 0
+
+    except KeyboardInterrupt:
+        logger.error("\nAnalysis interrupted by user")
+        return 130
+    except Exception as e:
+        logger.error(f"Analysis failed: {e}")
+        import traceback
+
+        traceback.print_exc()
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())