code-finder 0.1.0__py3-none-any.whl

claude_context/readme_extractor.py

@@ -0,0 +1,485 @@
+"""
+README Extraction Module
+
+Extracts essential user-facing content from README files:
+- Installation commands
+- Quickstart code examples
+- Authentication setup
+- Configuration examples
+
+This is the CRITICAL missing piece for evidence-first documentation.
+"""
+
+import re
+import logging
+from typing import Dict, List, Optional, Tuple
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class InstallationInfo:
+    """Structured installation information"""
+    command: str
+    requirements: List[str]
+    source: str = "README"
+
+
+@dataclass
+class CodeExample:
+    """Structured code example"""
+    code: str
+    language: str
+    context: str  # What this example demonstrates
+    source: str = "README"
+
+
+class ReadmeExtractor:
+    """
+    Extracts essential content from README files.
+
+    Uses markdown parsing + heuristics to find:
+    - Installation commands (pip, npm, cargo, etc.)
+    - Quickstart code examples
+    - Authentication setup
+    """
+
+    # Section headers that typically contain installation info
+    INSTALL_SECTION_PATTERNS = [
+        r"^#{1,3}\s*installation",
+        r"^#{1,3}\s*install",
+        r"^#{1,3}\s*getting\s+started",
+        r"^#{1,3}\s*quick\s+start",
+        r"^#{1,3}\s*setup",
+    ]
+
+    # Installation command patterns
+    INSTALL_COMMAND_PATTERNS = [
+        r"pip\s+install\s+[\w\-\[\]]+",
+        r"npm\s+install\s+[\w\-@/]+",
+        r"yarn\s+add\s+[\w\-@/]+",
+        r"cargo\s+install\s+[\w\-]+",
+        r"gem\s+install\s+[\w\-]+",
+        r"go\s+get\s+[\w\-./]+",
+        r"conda\s+install\s+[\w\-]+",
+    ]
+
+    # Quickstart section patterns
+    QUICKSTART_SECTION_PATTERNS = [
+        r"^#{1,3}\s*quick\s+start",
+        r"^#{1,3}\s*quickstart",
+        r"^#{1,3}\s*getting\s+started",
+        r"^#{1,3}\s*usage",
+        r"^#{1,3}\s*example",
+    ]
+
+    # Auth-related section patterns
+    AUTH_SECTION_PATTERNS = [
+        r"^#{1,3}\s*authentication",
+        r"^#{1,3}\s*auth",
+        r"^#{1,3}\s*api\s+key",
+        r"^#{1,3}\s*credentials",
+    ]
+
+    def __init__(self, readme_content: str):
+        """Initialize with README content"""
+        self.content = readme_content
+        self.lines = readme_content.split('\n')
+        logger.debug(f"ReadmeExtractor initialized with {len(self.lines)} lines")
+
+    def extract_installation_info(self) -> Optional[InstallationInfo]:
+        """
+        Extract installation command from README.
+
+        Returns:
+            InstallationInfo with command and requirements, or None if not found
+        """
+        # Find installation section
+        install_section = self._find_section(self.INSTALL_SECTION_PATTERNS)
+
+        if not install_section:
+            logger.warning("No installation section found in README")
+            return None
+
+        # Look for installation commands in this section
+        for line in install_section:
+            # Check code blocks first
+            if line.strip().startswith('```'):
+                continue
+
+            for pattern in self.INSTALL_COMMAND_PATTERNS:
+                match = re.search(pattern, line, re.IGNORECASE)
+                if match:
+                    command = match.group(0)
+                    logger.info(f"Found installation command: {command}")
+
+                    # Extract requirements if mentioned
+                    requirements = self._extract_requirements(install_section)
+
+                    return InstallationInfo(
+                        command=command,
+                        requirements=requirements,
+                        source="README"
+                    )
+
+        logger.warning("Installation section found but no install command extracted")
+        return None
+
+    def extract_quickstart_example(self) -> Optional[CodeExample]:
+        """
+        Extract the first substantial code example from README.
+
+        Prioritizes:
+        1. Code in "Quick Start" section
+        2. Code in "Usage" section
+        3. First Python/JS/etc code block (not bash/shell)
+
+        Returns:
+            CodeExample with code, language, and context
+        """
+        # Find quickstart/usage section
+        quickstart_section = self._find_section(self.QUICKSTART_SECTION_PATTERNS)
+
+        if quickstart_section:
+            code_example = self._extract_first_substantial_code(quickstart_section)
+            if code_example:
+                code_example.context = "Quickstart example from README"
+                logger.info(f"Found quickstart example: {len(code_example.code)} chars, language: {code_example.language}")
+                return code_example
+
+        # Fallback: Find first substantial code block (skip install commands)
+        code_example = self._extract_first_substantial_code(self.lines, skip_install=True)
+        if code_example:
+            code_example.context = "First code example from README"
+            logger.info(f"Found code example: {len(code_example.code)} chars")
+            return code_example
+
+        logger.warning("No code examples found in README")
+        return None
+
+    def extract_authentication_example(self) -> Optional[CodeExample]:
+        """
+        Extract authentication setup code from README.
+
+        Returns:
+            CodeExample showing how to authenticate, or None
+        """
+        # Find auth section
+        auth_section = self._find_section(self.AUTH_SECTION_PATTERNS)
+
+        if not auth_section:
+            # Look for auth keywords in any code block
+            for i, line in enumerate(self.lines):
+                if any(keyword in line.lower() for keyword in ['api_key', 'apikey', 'token', 'auth', 'credential']):
+                    # Check surrounding lines for code block
+                    context_lines = self.lines[max(0, i-10):min(len(self.lines), i+10)]
+                    code = self._extract_first_code_block(context_lines)
+                    if code:
+                        code.context = "Authentication setup"
+                        logger.info("Found authentication example via keyword search")
+                        return code
+            return None
+
+        # Extract code from auth section
+        code_example = self._extract_first_code_block(auth_section)
+        if code_example:
+            code_example.context = "Authentication setup from README"
+            logger.info("Found authentication example in auth section")
+            return code_example
+
+        return None
+
+    def extract_all_code_examples(self, limit: int = 5) -> List[CodeExample]:
+        """
+        Extract all code examples from README.
+
+        Args:
+            limit: Maximum number of examples to extract
+
+        Returns:
+            List of CodeExample objects
+        """
+        examples = []
+        in_code_block = False
+        current_code = []
+        current_language = ""
+        block_start_line = 0
+
+        for i, line in enumerate(self.lines):
+            # Detect code block start
+            if line.strip().startswith('```'):
+                if not in_code_block:
+                    # Starting a code block
+                    in_code_block = True
+                    current_code = []
+                    # Extract language
+                    lang_match = line.strip()[3:].strip()
+                    current_language = lang_match if lang_match else "text"
+                    block_start_line = i
+                else:
+                    # Ending a code block
+                    in_code_block = False
+                    if current_code and len('\n'.join(current_code)) > 20:  # Substantial code
+                        # Get context from preceding lines
+                        context = self._get_context_before_line(block_start_line)
+
+                        examples.append(CodeExample(
+                            code='\n'.join(current_code),
+                            language=current_language,
+                            context=context,
+                            source="README"
+                        ))
+
+                        if len(examples) >= limit:
+                            break
+            elif in_code_block:
+                current_code.append(line)
+
+        logger.info(f"Extracted {len(examples)} code examples from README")
+        return examples
+
+    def _find_section(self, header_patterns: List[str]) -> Optional[List[str]]:
+        """
+        Find a section by header patterns.
+
+        Returns:
+            List of lines in the section, or None if not found
+        """
+        best_match = None
+        best_match_level = 0
+        best_match_index = -1
+
+        # First pass: find all matching sections and pick the most specific one
+        for i, line in enumerate(self.lines):
+            stripped = line.strip()
+            for pattern in header_patterns:
+                if re.match(pattern, stripped, re.IGNORECASE):
+                    # Count the header level (number of #)
+                    level_match = re.match(r'^(#{1,6})\s+', stripped)
+                    level = len(level_match.group(1)) if level_match else 0
+
+                    # Prefer more specific (deeper) sections for install commands
+                    # e.g., "### Installation" (level 3) over "## Getting Started" (level 2)
+                    if level > best_match_level:
+                        best_match = (i, pattern, level)
+                        best_match_level = level
+                        best_match_index = i
+
+        if best_match is None:
+            return None
+
+        i, pattern, matched_level = best_match
+
+        # Extract section content - stop at same or higher level header
+        section_lines = []
+        for j in range(i + 1, len(self.lines)):
+            next_line = self.lines[j]
+            # Check if this is a header
+            header_match = re.match(r'^(#{1,6})\s+', next_line)
+            if header_match:
+                next_level = len(header_match.group(1))
+                # Stop at same or higher level (fewer or equal #)
+                if next_level <= matched_level:
+                    break
+            # Include subsections (more # than matched level)
+            section_lines.append(next_line)
+
+        logger.debug(f"Found section matching '{pattern}' at level {matched_level}: {len(section_lines)} lines")
+        return section_lines
+
+    def _extract_first_code_block(self, lines: List[str]) -> Optional[CodeExample]:
+        """Extract the first code block from lines (any language)"""
+        in_code_block = False
+        current_code = []
+        current_language = ""
+
+        for line in lines:
+            if line.strip().startswith('```'):
+                if not in_code_block:
+                    in_code_block = True
+                    current_code = []
+                    lang_match = line.strip()[3:].strip()
+                    current_language = lang_match if lang_match else "text"
+                else:
+                    # End of code block
+                    if current_code and len('\n'.join(current_code)) > 20:
+                        return CodeExample(
+                            code='\n'.join(current_code),
+                            language=current_language,
+                            context="",
+                            source="README"
+                        )
+                    in_code_block = False
+            elif in_code_block:
+                current_code.append(line)
+
+        return None
+
+    def _extract_first_substantial_code(self, lines: List[str], skip_install: bool = False) -> Optional[CodeExample]:
+        """
+        Extract first substantial code block, preferring actual code over install commands.
+
+        Args:
+            lines: Lines to search
+            skip_install: If True, skip bash/shell blocks with install commands
+
+        Returns:
+            CodeExample with substantial code
+        """
+        in_code_block = False
+        current_code = []
+        current_language = ""
+
+        for line in lines:
+            if line.strip().startswith('```'):
+                if not in_code_block:
+                    in_code_block = True
+                    current_code = []
+                    lang_match = line.strip()[3:].strip()
+                    current_language = lang_match if lang_match else "text"
+                else:
+                    # End of code block
+                    if current_code:
+                        code_text = '\n'.join(current_code)
+
+                        # Skip install commands if requested
+                        if skip_install:
+                            if current_language in ['bash', 'sh', 'shell', 'console']:
+                                # Check if this is just an install command
+                                if any(cmd in code_text for cmd in ['pip install', 'npm install', 'yarn add']):
+                                    if len(code_text) < 100:  # Short install-only block
+                                        in_code_block = False
+                                        continue
+
+                        # Return if substantial (>30 chars for quickstart)
+                        if len(code_text) > 30:
+                            return CodeExample(
+                                code=code_text,
+                                language=current_language,
+                                context="",
+                                source="README"
+                            )
+
+                    in_code_block = False
+            elif in_code_block:
+                current_code.append(line)
+
+        return None
+
+    def _extract_requirements(self, section_lines: List[str]) -> List[str]:
+        """Extract requirements from installation section"""
+        requirements = []
+
+        for line in section_lines:
+            # Look for Python version patterns (handles markdown formatting like **Python:** 3.10)
+            # Matches: "Python 3.10", "Python: 3.10", "**Python:** 3.10", "python >= 3.8"
+            python_patterns = [
+                r'python[:\s*]+(\d+\.\d+)(?:\s*(?:or\s+)?(?:higher|\+|>=))?',
+                r'python\s*>=?\s*(\d+\.\d+)',
+                r'python\s+(\d+\.\d+)[\+\w]*',
+            ]
+            for pattern in python_patterns:
+                match = re.search(pattern, line, re.IGNORECASE)
+                if match:
+                    version = match.group(1)
+                    req = f"Python {version}+"
+                    if req not in requirements:
+                        requirements.append(req)
+                    break
+
+            # Look for Node.js version patterns
+            node_patterns = [
+                r'node[:\s*]+(\d+(?:\.\d+)?)',
+                r'node\.?js[:\s*]+(\d+(?:\.\d+)?)',
+                r'node\s*>=?\s*(\d+(?:\.\d+)?)',
+            ]
+            for pattern in node_patterns:
+                match = re.search(pattern, line, re.IGNORECASE)
+                if match:
+                    version = match.group(1)
+                    req = f"Node.js {version}+"
+                    if req not in requirements:
+                        requirements.append(req)
+                    break
+
+        return requirements
+
+    def _get_context_before_line(self, line_num: int, lookback: int = 3) -> str:
+        """Get context from lines before a code block"""
+        start = max(0, line_num - lookback)
+        context_lines = []
+
+        for i in range(start, line_num):
+            line = self.lines[i].strip()
+            # Skip empty lines and headers
+            if line and not line.startswith('#'):
+                context_lines.append(line)
+
+        return ' '.join(context_lines) if context_lines else "Code example"
+
+
+def extract_from_readme(readme_content: str) -> Dict:
+    """
+    Main entry point for README extraction.
+
+    Returns structured evidence dictionary with:
+    - installation: InstallationInfo or None
+    - quickstart: CodeExample or None
+    - authentication: CodeExample or None
+    - examples: List[CodeExample]
+    """
+    extractor = ReadmeExtractor(readme_content)
+
+    return {
+        "installation": extractor.extract_installation_info(),
+        "quickstart": extractor.extract_quickstart_example(),
+        "authentication": extractor.extract_authentication_example(),
+        "examples": extractor.extract_all_code_examples(limit=3)
+    }
+
+
+# Example usage
+if __name__ == "__main__":
+    # Test with sample README
+    sample_readme = """
+# My Project
+
+A cool project that does things.
+
+## Installation
+
+```bash
+pip install my-project
+```
+
+Requirements:
+- Python 3.8+
+- NumPy
+
+## Quick Start
+
+```python
+from my_project import Client
+
+client = Client(api_key="your-key")
+result = client.do_something()
+print(result)
+```
+
+## Authentication
+
+Set your API key:
+
+```python
+import os
+os.environ["API_KEY"] = "your-key-here"
+```
+"""
+
+    result = extract_from_readme(sample_readme)
+    print("Extraction Results:")
+    print(f"Installation: {result['installation']}")
+    print(f"Quickstart: {result['quickstart']}")
+    print(f"Auth: {result['authentication']}")
+    print(f"Examples: {len(result['examples'])} found")