contextual-engine 0.1.0__py3-none-any.whl

contextual/indexing/processor.py

@@ -0,0 +1,436 @@
+"""File discovery and chunking orchestrator.
+
+Walks project directories, respects .gitignore/.contextualignore patterns,
+excludes generated files and vendor directories, and coordinates parallel
+chunking across discovered source files.
+
+Key responsibilities:
+- Honor .gitignore + .contextualignore via pathspec
+- Skip generated files (*.min.js, *_pb2.py, *.d.ts, etc.)
+- Skip vendor directories (node_modules/, venv/, __pycache__/, etc.)
+- Detect language from file extensions
+- Coordinate chunking while respecting file size limits
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pathspec
+
+from contextual.core.errors import ErrorCode, IndexingError
+from contextual.indexing.chunker import chunk_file, detect_language
+from contextual.observability import get_logger
+from contextual.security.paths import safe_path
+
+
+if TYPE_CHECKING:
+    from contextual.core.models import Chunk
+
+logger = get_logger(__name__)
+
+# Generated file patterns (skip these)
+GENERATED_PATTERNS = {
+    # Minified/compiled
+    r"\.min\.js$",
+    r"\.min\.css$",
+    r"-min\.js$",
+    r"\.bundle\.js$",
+    # Protocol buffers
+    r"_pb2\.py$",
+    r"_pb2_grpc\.py$",
+    r"\.pb\.go$",
+    r"\.pb\.h$",
+    r"\.pb\.cc$",
+    # TypeScript declarations (generated)
+    r"\.d\.ts$",
+    # Auto-generated comments
+    r"^.*?//\s*Code generated .* DO NOT EDIT",  # Go standard
+    r"^.*?//\s*AUTO-GENERATED",
+    r"^.*?#\s*AUTO-GENERATED",
+    # Build artifacts
+    r"\.pyc$",
+    r"\.pyo$",
+    r"\.so$",
+    r"\.dylib$",
+    r"\.dll$",
+    r"\.exe$",
+    # Lock files (too large, not code)
+    r"package-lock\.json$",
+    r"yarn\.lock$",
+    r"poetry\.lock$",
+    r"Pipfile\.lock$",
+    r"Cargo\.lock$",
+    r"Gemfile\.lock$",
+    r"pnpm-lock\.yaml$",
+}
+
+# Vendor/build directory patterns (skip entire directories)
+VENDOR_DIR_PATTERNS = {
+    "node_modules",
+    "__pycache__",
+    ".git",
+    ".svn",
+    ".hg",
+    "venv",
+    "env",
+    ".venv",
+    ".env",
+    "virtualenv",
+    "build",
+    "dist",
+    "target",  # Rust/Java
+    ".tox",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    "htmlcov",
+    "coverage",
+    ".coverage",
+    ".eggs",
+    "*.egg-info",
+    ".gradle",
+    "gradle",
+    "bin",  # Common binary dir
+    "obj",  # C#/C++ object files
+    ".next",  # Next.js
+    ".nuxt",  # Nuxt.js
+    "bower_components",
+    "jspm_packages",
+}
+
+# Compiled regex for generated file detection
+_GENERATED_REGEX = [re.compile(pattern) for pattern in GENERATED_PATTERNS]
+
+
+class FileProcessor:
+    """File discovery and chunking orchestrator.
+    
+    Walks a project directory tree, applying ignore patterns and
+    filtering rules, then coordinates chunking of discovered files.
+    """
+
+    def __init__(self, project_root: Path) -> None:
+        """Initialize processor.
+        
+        Args:
+            project_root: Absolute path to project root
+            
+        Raises:
+            IndexingError: If project root invalid
+        """
+        # Validate project root
+        if not project_root.is_dir():
+            raise IndexingError(
+                ErrorCode.SECURITY_PATH_INVALID,
+                f"Project root is not a directory: {project_root}",
+            )
+
+        self.project_root = project_root.resolve()
+
+        # Load ignore patterns
+        self.gitignore_spec = self._load_gitignore()
+        self.contextualignore_spec = self._load_contextualignore()
+
+    def _load_gitignore(self) -> pathspec.PathSpec | None:
+        """Load .gitignore patterns from project root."""
+        gitignore_path = self.project_root / ".gitignore"
+        if not gitignore_path.exists():
+            return None
+
+        try:
+            with open(gitignore_path, encoding="utf-8") as f:
+                patterns = f.read().splitlines()
+            return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+        except Exception as e:
+            logger.warning(
+                "gitignore_load_failed",
+                path=str(gitignore_path),
+                error=str(e),
+            )
+            return None
+
+    def _load_contextualignore(self) -> pathspec.PathSpec | None:
+        """Load .contextualignore patterns from project root."""
+        ignore_path = self.project_root / ".contextualignore"
+        if not ignore_path.exists():
+            return None
+
+        try:
+            with open(ignore_path, encoding="utf-8") as f:
+                patterns = f.read().splitlines()
+            return pathspec.PathSpec.from_lines("gitwildmatch", patterns)
+        except Exception as e:
+            logger.warning(
+                "contextualignore_load_failed",
+                path=str(ignore_path),
+                error=str(e),
+            )
+            return None
+
+    def discover_files(self) -> list[Path]:
+        """Discover all indexable files in project.
+        
+        Returns:
+            List of absolute paths to files that should be chunked
+        """
+        discovered = []
+
+        for path in self._walk_directory(self.project_root):
+            # Get relative path for pattern matching
+            try:
+                rel_path = path.relative_to(self.project_root)
+            except ValueError:
+                continue
+
+            # Apply filters
+            if self._should_skip(path, rel_path):
+                continue
+
+            # Check if language supported
+            language = detect_language(path)
+            if not language:
+                continue
+
+            discovered.append(path)
+
+        logger.info(
+            "file_discovery_complete",
+            total_files=len(discovered),
+            project_root=str(self.project_root),
+        )
+
+        return discovered
+
+    def _walk_directory(self, root: Path) -> list[Path]:
+        """Recursively walk directory tree.
+        
+        Args:
+            root: Directory to walk
+            
+        Returns:
+            List of file paths (not directories)
+        """
+        files = []
+
+        try:
+            for item in root.iterdir():
+                # Skip vendor directories
+                if item.is_dir():
+                    if self._is_vendor_dir(item):
+                        continue
+                    # Recurse
+                    files.extend(self._walk_directory(item))
+                else:
+                    files.append(item)
+        except PermissionError:
+            logger.warning("permission_denied", path=str(root))
+        except Exception as e:
+            logger.error("walk_error", path=str(root), error=str(e))
+
+        return files
+
+    def _is_vendor_dir(self, path: Path) -> bool:
+        """Check if directory is a vendor/build directory to skip."""
+        dir_name = path.name
+
+        # Exact match
+        if dir_name in VENDOR_DIR_PATTERNS:
+            return True
+
+        # Glob patterns (e.g., *.egg-info)
+        for pattern in VENDOR_DIR_PATTERNS:
+            if "*" in pattern:
+                if path.match(pattern):
+                    return True
+
+        return False
+
+    def _should_skip(self, path: Path, rel_path: Path) -> bool:
+        """Check if file should be skipped.
+        
+        Args:
+            path: Absolute path
+            rel_path: Path relative to project root
+            
+        Returns:
+            True if file should be skipped
+        """
+        rel_path_str = str(rel_path)
+
+        # Check .gitignore
+        if self.gitignore_spec and self.gitignore_spec.match_file(rel_path_str):
+            return True
+
+        # Check .contextualignore
+        if self.contextualignore_spec and self.contextualignore_spec.match_file(
+            rel_path_str
+        ):
+            return True
+
+        # Check generated file patterns
+        if self._is_generated_file(path):
+            return True
+
+        return False
+
+    def _is_generated_file(self, path: Path) -> bool:
+        """Check if file matches generated file patterns."""
+        path_str = str(path)
+
+        # Check regex patterns
+        for regex in _GENERATED_REGEX:
+            if regex.search(path_str):
+                return True
+
+        # Check file content for generation markers
+        if self._has_generation_marker(path):
+            return True
+
+        return False
+
+    def _has_generation_marker(self, path: Path) -> bool:
+        """Check if file has auto-generation markers in first few lines.
+        
+        Reads first 10 lines to detect comments like:
+        - // Code generated ... DO NOT EDIT
+        - # AUTO-GENERATED
+        """
+        try:
+            with open(path, "rb") as f:
+                # Read first 1KB (usually enough for headers)
+                header = f.read(1024)
+
+            # Decode and check first few lines
+            text = header.decode("utf-8", errors="ignore")
+            first_lines = text.split("\n")[:10]
+
+            generation_markers = [
+                "CODE GENERATED",
+                "AUTO-GENERATED",
+                "AUTOGENERATED",
+                "DO NOT EDIT",
+                "Generated by",
+                "This file is automatically generated",
+            ]
+
+            for line in first_lines:
+                line_upper = line.upper()
+                for marker in generation_markers:
+                    if marker in line_upper:
+                        return True
+
+            return False
+
+        except Exception:
+            # If we can't read the file, don't skip it based on this check
+            return False
+
+    def process_files(
+        self,
+        file_paths: list[Path] | None = None,
+        max_file_size: int = 2 * 1024 * 1024,
+    ) -> list[Chunk]:
+        """Process files and generate chunks.
+        
+        Args:
+            file_paths: Specific files to process (None = discover all)
+            max_file_size: Maximum file size in bytes (default 2MB)
+            
+        Returns:
+            List of all generated chunks
+            
+        Raises:
+            IndexingError: If processing fails
+        """
+        # Discover files if not provided
+        if file_paths is None:
+            file_paths = self.discover_files()
+
+        all_chunks = []
+        processed_count = 0
+        skipped_count = 0
+
+        for file_path in file_paths:
+            try:
+                # Validate path security
+                file_path = safe_path(file_path, self.project_root)
+
+                # Check file size
+                file_size = file_path.stat().st_size
+                if file_size > max_file_size:
+                    logger.warning(
+                        "file_too_large",
+                        path=str(file_path),
+                        size=file_size,
+                        max_size=max_file_size,
+                    )
+                    skipped_count += 1
+                    continue
+
+                # Read file content
+                with open(file_path, "rb") as f:
+                    content = f.read()
+
+                # Detect language
+                language = detect_language(file_path)
+                if not language:
+                    skipped_count += 1
+                    continue
+
+                # Get relative path
+                rel_path = file_path.relative_to(self.project_root)
+
+                # Chunk the file
+                chunks = chunk_file(rel_path, content, language)
+                all_chunks.extend(chunks)
+                processed_count += 1
+
+                logger.debug(
+                    "file_processed",
+                    path=str(rel_path),
+                    language=language,
+                    chunks=len(chunks),
+                )
+
+            except Exception as e:
+                logger.error(
+                    "file_processing_failed",
+                    path=str(file_path),
+                    error=str(e),
+                )
+                skipped_count += 1
+                continue
+
+        logger.info(
+            "processing_complete",
+            processed=processed_count,
+            skipped=skipped_count,
+            total_chunks=len(all_chunks),
+        )
+
+        return all_chunks
+
+
+def index_project(
+    project_root: Path,
+    file_paths: list[Path] | None = None,
+    max_file_size: int = 2 * 1024 * 1024,
+) -> list[Chunk]:
+    """Index a project directory.
+    
+    Args:
+        project_root: Absolute path to project root
+        file_paths: Specific files to index (None = discover all)
+        max_file_size: Maximum file size in bytes
+        
+    Returns:
+        List of all generated chunks
+        
+    Raises:
+        IndexingError: If indexing fails
+    """
+    processor = FileProcessor(project_root)
+    return processor.process_files(file_paths, max_file_size)

contextual/indexing/queries/readme.md

@@ -0,0 +1,22 @@
+# Tree-sitter Query Files
+ 
+This directory is reserved for Tree-sitter .scm query files for symbol extraction.
+ 
+## Current Status
+Symbol extraction is currently implemented via direct AST traversal in `symbol_extractor.py`.
+Tree-sitter query files (.scm) are an optional enhancement for more precise extraction.
+ 
+## Planned Files
+- python.scm - Python symbol queries
+- typescript.scm - TypeScript symbol queries  
+- javascript.scm - JavaScript symbol queries
+- rust.scm - Rust symbol queries
+- go.scm - Go symbol queries
+ 
+## When to Use
+.scm query files will be used when:
+1. More precise extraction is needed
+2. Language-specific edge cases require custom queries
+3. Performance optimization is needed
+ 
+For Phase 1, direct AST traversal is sufficient.