groknroll 2.0.0__py3-none-any.whl

groknroll/core/large_codebase.py

@@ -0,0 +1,564 @@
+"""
+Large Codebase Handler
+
+Specialized tools for working with massive codebases without context rot.
+Uses RLM's unlimited context + intelligent chunking + hierarchical indexing.
+"""
+
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Set
+from dataclasses import dataclass
+import hashlib
+import json
+
+
+@dataclass
+class CodeChunk:
+    """Represents a chunk of the codebase"""
+    id: str
+    path: Path
+    language: str
+    size: int  # lines of code
+    dependencies: List[str]  # Other chunk IDs this depends on
+    summary: Optional[str] = None
+    last_analyzed: Optional[float] = None
+    hash: Optional[str] = None
+
+
+@dataclass
+class CodebaseMap:
+    """Hierarchical map of the codebase"""
+    root: Path
+    chunks: Dict[str, CodeChunk]
+    modules: Dict[str, List[str]]  # module name -> chunk IDs
+    dependency_graph: Dict[str, Set[str]]  # chunk ID -> dependencies
+    total_files: int
+    total_lines: int
+
+
+class LargeCodebaseHandler:
+    """
+    Handle massive codebases without context rot
+
+    Strategy:
+    1. Hierarchical chunking (directory-based)
+    2. Dependency tracking
+    3. Incremental analysis (only changed files)
+    4. Smart context selection (only relevant chunks)
+    5. RLM handles unlimited context when needed
+    """
+
+    def __init__(self, project_path: Path, db):
+        """
+        Initialize large codebase handler
+
+        Args:
+            project_path: Project root
+            db: Database instance
+        """
+        self.project_path = project_path.resolve()
+        self.db = db
+        self.cache_dir = project_path / ".groknroll" / "cache"
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load or create codebase map
+        self.map_file = self.cache_dir / "codebase_map.json"
+        self.codebase_map = self._load_or_create_map()
+
+    def chunk_codebase(self, max_chunk_size: int = 1000) -> CodebaseMap:
+        """
+        Chunk codebase into manageable pieces
+
+        Args:
+            max_chunk_size: Max lines per chunk
+
+        Returns:
+            CodebaseMap
+        """
+        chunks = {}
+        modules = {}
+        total_files = 0
+        total_lines = 0
+
+        # Walk directory tree
+        for root, dirs, files in os.walk(self.project_path):
+            # Skip common ignore directories
+            dirs[:] = [d for d in dirs if d not in {
+                '.git', 'node_modules', '.venv', 'venv', '__pycache__',
+                'build', 'dist', '.next', 'target', '.groknroll'
+            }]
+
+            root_path = Path(root)
+            module_name = root_path.relative_to(self.project_path).as_posix()
+
+            for file in files:
+                if self._should_index(file):
+                    file_path = root_path / file
+
+                    try:
+                        lines = len(file_path.read_text().splitlines())
+                        total_lines += lines
+                        total_files += 1
+
+                        # Create chunk
+                        chunk_id = self._generate_chunk_id(file_path)
+                        chunk = CodeChunk(
+                            id=chunk_id,
+                            path=file_path,
+                            language=self._detect_language(file_path.suffix),
+                            size=lines,
+                            dependencies=[],  # Will be populated later
+                            hash=self._file_hash(file_path)
+                        )
+
+                        chunks[chunk_id] = chunk
+
+                        # Add to module
+                        if module_name not in modules:
+                            modules[module_name] = []
+                        modules[module_name].append(chunk_id)
+
+                    except Exception as e:
+                        # Skip files we can't read
+                        continue
+
+        # Build dependency graph
+        dependency_graph = self._build_dependency_graph(chunks)
+
+        codebase_map = CodebaseMap(
+            root=self.project_path,
+            chunks=chunks,
+            modules=modules,
+            dependency_graph=dependency_graph,
+            total_files=total_files,
+            total_lines=total_lines
+        )
+
+        # Save map
+        self._save_map(codebase_map)
+
+        return codebase_map
+
+    def get_relevant_context(
+        self,
+        query: str,
+        max_chunks: int = 50,
+        include_dependencies: bool = True
+    ) -> List[CodeChunk]:
+        """
+        Get relevant code chunks for a query (smart context selection)
+
+        Args:
+            query: Search query or task description
+            max_chunks: Maximum chunks to return
+            include_dependencies: Include dependent chunks
+
+        Returns:
+            List of relevant CodeChunk objects
+        """
+        # Search for relevant chunks
+        relevant_chunks = []
+
+        # 1. Keyword-based search
+        keywords = self._extract_keywords(query)
+        scored_chunks = []
+
+        for chunk_id, chunk in self.codebase_map.chunks.items():
+            score = 0
+
+            # Score based on path matching
+            path_str = str(chunk.path).lower()
+            for keyword in keywords:
+                if keyword in path_str:
+                    score += 10
+
+            # Score based on module relevance
+            module = str(chunk.path.parent.relative_to(self.project_path))
+            for keyword in keywords:
+                if keyword in module.lower():
+                    score += 5
+
+            if score > 0:
+                scored_chunks.append((score, chunk))
+
+        # Sort by score
+        scored_chunks.sort(reverse=True, key=lambda x: x[0])
+
+        # Take top chunks
+        relevant_chunks = [chunk for _, chunk in scored_chunks[:max_chunks]]
+
+        # 2. Include dependencies if requested
+        if include_dependencies:
+            additional_chunks = set()
+            for chunk in relevant_chunks:
+                if chunk.id in self.codebase_map.dependency_graph:
+                    deps = self.codebase_map.dependency_graph[chunk.id]
+                    for dep_id in deps:
+                        if dep_id in self.codebase_map.chunks:
+                            additional_chunks.add(self.codebase_map.chunks[dep_id])
+
+            relevant_chunks.extend(list(additional_chunks)[:max_chunks // 2])
+
+        return relevant_chunks[:max_chunks]
+
+    def get_module_summary(self, module_name: str) -> Dict[str, Any]:
+        """
+        Get summary of a module (directory)
+
+        Args:
+            module_name: Module path (e.g., "src/core")
+
+        Returns:
+            Module summary
+        """
+        if module_name not in self.codebase_map.modules:
+            return {
+                "error": f"Module not found: {module_name}",
+                "available_modules": list(self.codebase_map.modules.keys())[:20]
+            }
+
+        chunk_ids = self.codebase_map.modules[module_name]
+        chunks = [self.codebase_map.chunks[cid] for cid in chunk_ids]
+
+        # Aggregate statistics
+        total_lines = sum(c.size for c in chunks)
+        languages = {}
+
+        for chunk in chunks:
+            lang = chunk.language
+            if lang not in languages:
+                languages[lang] = {"files": 0, "lines": 0}
+            languages[lang]["files"] += 1
+            languages[lang]["lines"] += chunk.size
+
+        return {
+            "module": module_name,
+            "files": len(chunks),
+            "total_lines": total_lines,
+            "languages": languages,
+            "chunks": [
+                {
+                    "file": chunk.path.name,
+                    "language": chunk.language,
+                    "lines": chunk.size
+                }
+                for chunk in chunks
+            ]
+        }
+
+    def get_changed_chunks(self, since_last_analysis: bool = True) -> List[CodeChunk]:
+        """
+        Get chunks that have changed (incremental analysis)
+
+        Args:
+            since_last_analysis: Only get chunks changed since last analysis
+
+        Returns:
+            List of changed chunks
+        """
+        changed = []
+
+        for chunk_id, chunk in self.codebase_map.chunks.items():
+            current_hash = self._file_hash(chunk.path)
+
+            if chunk.hash != current_hash:
+                # File has changed
+                changed.append(chunk)
+
+                # Update hash
+                chunk.hash = current_hash
+
+        # Save updated map
+        if changed:
+            self._save_map(self.codebase_map)
+
+        return changed
+
+    def get_codebase_overview(self) -> Dict[str, Any]:
+        """
+        Get high-level overview of entire codebase
+
+        Returns:
+            Codebase overview
+        """
+        # Language statistics
+        languages = {}
+        for chunk in self.codebase_map.chunks.values():
+            lang = chunk.language
+            if lang not in languages:
+                languages[lang] = {"files": 0, "lines": 0}
+            languages[lang]["files"] += 1
+            languages[lang]["lines"] += chunk.size
+
+        # Top modules by size
+        module_sizes = {}
+        for module_name, chunk_ids in self.codebase_map.modules.items():
+            total_lines = sum(
+                self.codebase_map.chunks[cid].size
+                for cid in chunk_ids
+                if cid in self.codebase_map.chunks
+            )
+            module_sizes[module_name] = total_lines
+
+        top_modules = sorted(
+            module_sizes.items(),
+            key=lambda x: x[1],
+            reverse=True
+        )[:10]
+
+        return {
+            "total_files": self.codebase_map.total_files,
+            "total_lines": self.codebase_map.total_lines,
+            "total_chunks": len(self.codebase_map.chunks),
+            "total_modules": len(self.codebase_map.modules),
+            "languages": languages,
+            "top_modules": [
+                {"module": name, "lines": lines}
+                for name, lines in top_modules
+            ]
+        }
+
+    def navigate_to_definition(self, symbol: str) -> List[Dict[str, Any]]:
+        """
+        Find definition of a symbol across codebase
+
+        Args:
+            symbol: Symbol to find (function, class, variable)
+
+        Returns:
+            List of potential definitions
+        """
+        results = []
+
+        # Search patterns for different languages
+        patterns = {
+            "python": [
+                f"def {symbol}(",
+                f"class {symbol}:",
+                f"{symbol} = "
+            ],
+            "javascript": [
+                f"function {symbol}(",
+                f"const {symbol} =",
+                f"class {symbol}",
+            ],
+            "typescript": [
+                f"function {symbol}(",
+                f"const {symbol}:",
+                f"class {symbol}",
+                f"interface {symbol}",
+            ]
+        }
+
+        for chunk in self.codebase_map.chunks.values():
+            try:
+                content = chunk.path.read_text()
+
+                # Get patterns for this language
+                lang_patterns = patterns.get(chunk.language, [])
+
+                for pattern in lang_patterns:
+                    if pattern in content:
+                        # Find line number
+                        lines = content.splitlines()
+                        for i, line in enumerate(lines, 1):
+                            if pattern in line:
+                                results.append({
+                                    "file": str(chunk.path.relative_to(self.project_path)),
+                                    "line": i,
+                                    "snippet": line.strip(),
+                                    "language": chunk.language
+                                })
+            except Exception:
+                continue
+
+        return results
+
+    # =========================================================================
+    # Helper Methods
+    # =========================================================================
+
+    def _load_or_create_map(self) -> CodebaseMap:
+        """Load existing map or create new one"""
+        if self.map_file.exists():
+            try:
+                data = json.loads(self.map_file.read_text())
+
+                # Reconstruct CodebaseMap
+                chunks = {
+                    cid: CodeChunk(
+                        id=c["id"],
+                        path=Path(c["path"]),
+                        language=c["language"],
+                        size=c["size"],
+                        dependencies=c["dependencies"],
+                        summary=c.get("summary"),
+                        last_analyzed=c.get("last_analyzed"),
+                        hash=c.get("hash")
+                    )
+                    for cid, c in data["chunks"].items()
+                }
+
+                return CodebaseMap(
+                    root=Path(data["root"]),
+                    chunks=chunks,
+                    modules=data["modules"],
+                    dependency_graph={
+                        k: set(v) for k, v in data["dependency_graph"].items()
+                    },
+                    total_files=data["total_files"],
+                    total_lines=data["total_lines"]
+                )
+            except Exception as e:
+                # If loading fails, create new map
+                pass
+
+        # Create new map
+        return self.chunk_codebase()
+
+    def _save_map(self, codebase_map: CodebaseMap) -> None:
+        """Save codebase map to disk"""
+        data = {
+            "root": str(codebase_map.root),
+            "chunks": {
+                cid: {
+                    "id": c.id,
+                    "path": str(c.path),
+                    "language": c.language,
+                    "size": c.size,
+                    "dependencies": c.dependencies,
+                    "summary": c.summary,
+                    "last_analyzed": c.last_analyzed,
+                    "hash": c.hash
+                }
+                for cid, c in codebase_map.chunks.items()
+            },
+            "modules": codebase_map.modules,
+            "dependency_graph": {
+                k: list(v) for k, v in codebase_map.dependency_graph.items()
+            },
+            "total_files": codebase_map.total_files,
+            "total_lines": codebase_map.total_lines
+        }
+
+        self.map_file.write_text(json.dumps(data, indent=2))
+
+    def _generate_chunk_id(self, file_path: Path) -> str:
+        """Generate unique chunk ID"""
+        relative_path = file_path.relative_to(self.project_path)
+        return hashlib.md5(str(relative_path).encode()).hexdigest()[:16]
+
+    def _file_hash(self, file_path: Path) -> str:
+        """Calculate file hash for change detection"""
+        try:
+            content = file_path.read_bytes()
+            return hashlib.md5(content).hexdigest()
+        except Exception:
+            return ""
+
+    def _should_index(self, filename: str) -> bool:
+        """Check if file should be indexed"""
+        # Skip common non-code files
+        skip_extensions = {
+            '.pyc', '.pyo', '.so', '.dylib', '.dll',
+            '.jpg', '.jpeg', '.png', '.gif', '.svg',
+            '.pdf', '.zip', '.tar', '.gz',
+            '.lock', '.log', '.tmp'
+        }
+
+        ext = Path(filename).suffix.lower()
+        return ext not in skip_extensions
+
+    def _detect_language(self, extension: str) -> str:
+        """Detect language from file extension"""
+        lang_map = {
+            '.py': 'python',
+            '.js': 'javascript',
+            '.ts': 'typescript',
+            '.jsx': 'javascript',
+            '.tsx': 'typescript',
+            '.go': 'go',
+            '.rs': 'rust',
+            '.java': 'java',
+            '.cpp': 'cpp',
+            '.c': 'c',
+            '.rb': 'ruby',
+            '.php': 'php',
+            '.md': 'markdown',
+            '.json': 'json',
+            '.yaml': 'yaml',
+            '.yml': 'yaml',
+        }
+        return lang_map.get(extension.lower(), 'unknown')
+
+    def _extract_keywords(self, query: str) -> List[str]:
+        """Extract keywords from query"""
+        # Simple keyword extraction (can be enhanced)
+        words = query.lower().split()
+
+        # Filter out common words
+        stop_words = {'the', 'a', 'an', 'in', 'on', 'at', 'to', 'for', 'of', 'and', 'or'}
+        keywords = [w for w in words if w not in stop_words and len(w) > 2]
+
+        return keywords
+
+    def _build_dependency_graph(self, chunks: Dict[str, CodeChunk]) -> Dict[str, Set[str]]:
+        """Build dependency graph between chunks"""
+        graph = {}
+
+        # For Python files, look for imports
+        for chunk_id, chunk in chunks.items():
+            if chunk.language == 'python':
+                try:
+                    content = chunk.path.read_text()
+                    imports = self._extract_python_imports(content)
+
+                    # Map imports to chunks
+                    deps = set()
+                    for imp in imports:
+                        # Try to find corresponding chunk
+                        for other_id, other_chunk in chunks.items():
+                            if other_id != chunk_id:
+                                # Check if import matches this file
+                                if self._import_matches_file(imp, other_chunk.path):
+                                    deps.add(other_id)
+
+                    if deps:
+                        graph[chunk_id] = deps
+
+                except Exception:
+                    continue
+
+        return graph
+
+    def _extract_python_imports(self, content: str) -> List[str]:
+        """Extract Python imports from content"""
+        imports = []
+
+        for line in content.splitlines():
+            line = line.strip()
+
+            if line.startswith('import '):
+                # import foo
+                module = line[7:].split()[0].strip()
+                imports.append(module)
+            elif line.startswith('from '):
+                # from foo import bar
+                parts = line.split()
+                if len(parts) >= 2:
+                    module = parts[1].strip()
+                    imports.append(module)
+
+        return imports
+
+    def _import_matches_file(self, import_name: str, file_path: Path) -> bool:
+        """Check if import matches a file"""
+        # Convert import to path
+        import_path = import_name.replace('.', '/')
+        file_str = str(file_path)
+
+        return import_path in file_str
+
+
+# Missing import
+import os