groknroll 2.0.0__py3-none-any.whl

groknroll/oracle/codebase_indexer.py

@@ -0,0 +1,238 @@
+"""
+Codebase Indexer - Builds comprehensive index of the entire codebase
+"""
+
+import ast
+import os
+from pathlib import Path
+from typing import Dict, List, Optional, Set
+from dataclasses import dataclass, field
+
+
+@dataclass
+class FileInfo:
+    """Information about a source file"""
+    path: Path
+    relative_path: str
+    size: int
+    lines: int
+    language: str
+    content: Optional[str] = None
+    functions: List[str] = field(default_factory=list)
+    classes: List[str] = field(default_factory=list)
+    imports: List[str] = field(default_factory=list)
+
+
+@dataclass
+class CodebaseIndex:
+    """Complete index of the codebase"""
+    root_path: Path
+    files: Dict[str, FileInfo] = field(default_factory=dict)
+    functions: Dict[str, List[str]] = field(default_factory=dict)  # function_name -> [file_paths]
+    classes: Dict[str, List[str]] = field(default_factory=dict)  # class_name -> [file_paths]
+    imports: Dict[str, Set[str]] = field(default_factory=dict)  # module -> {files_that_import_it}
+    total_files: int = 0
+    total_lines: int = 0
+    total_size: int = 0
+
+
+class CodebaseIndexer:
+    """
+    Indexes the entire codebase for the Oracle Agent
+
+    Extracts:
+    - File structure and contents
+    - Function and class definitions
+    - Import relationships
+    - Documentation strings
+    """
+
+    SUPPORTED_EXTENSIONS = {
+        '.py': 'python',
+        '.js': 'javascript',
+        '.ts': 'typescript',
+        '.jsx': 'javascript',
+        '.tsx': 'typescript',
+        '.md': 'markdown',
+        '.txt': 'text',
+        '.yaml': 'yaml',
+        '.yml': 'yaml',
+        '.json': 'json',
+        '.toml': 'toml',
+    }
+
+    IGNORE_DIRS = {
+        '__pycache__', '.git', '.venv', 'venv', 'node_modules',
+        'dist', 'build', '.pytest_cache', '.egg-info', 'rlm_old_backup',
+        '.vscode', '.idea', 'logs', 'agent_logs'
+    }
+
+    def __init__(self, root_path: Path):
+        """Initialize indexer with repository root"""
+        self.root_path = Path(root_path).resolve()
+        self.index = CodebaseIndex(root_path=self.root_path)
+
+    def build_index(self, include_content: bool = True) -> CodebaseIndex:
+        """
+        Build complete codebase index
+
+        Args:
+            include_content: Whether to include full file contents in index
+
+        Returns:
+            CodebaseIndex with all extracted information
+        """
+        print(f"🔍 Indexing codebase at: {self.root_path}")
+
+        # Walk directory tree
+        for root, dirs, files in os.walk(self.root_path):
+            # Filter out ignored directories
+            dirs[:] = [d for d in dirs if d not in self.IGNORE_DIRS]
+
+            root_path = Path(root)
+
+            for file in files:
+                file_path = root_path / file
+                ext = file_path.suffix.lower()
+
+                if ext in self.SUPPORTED_EXTENSIONS:
+                    self._index_file(file_path, include_content)
+
+        print(f"✅ Indexed {self.index.total_files} files ({self.index.total_lines:,} lines)")
+        return self.index
+
+    def _index_file(self, file_path: Path, include_content: bool):
+        """Index a single file"""
+        try:
+            relative_path = str(file_path.relative_to(self.root_path))
+
+            # Read file content
+            content = file_path.read_text(encoding='utf-8', errors='ignore')
+            lines = content.count('\n') + 1
+            size = file_path.stat().st_size
+
+            ext = file_path.suffix.lower()
+            language = self.SUPPORTED_EXTENSIONS[ext]
+
+            # Create file info
+            file_info = FileInfo(
+                path=file_path,
+                relative_path=relative_path,
+                size=size,
+                lines=lines,
+                language=language,
+                content=content if include_content else None
+            )
+
+            # Extract Python-specific metadata
+            if language == 'python':
+                self._extract_python_info(file_info, content)
+
+            # Store in index
+            self.index.files[relative_path] = file_info
+            self.index.total_files += 1
+            self.index.total_lines += lines
+            self.index.total_size += size
+
+        except Exception as e:
+            print(f"⚠️  Error indexing {file_path}: {e}")
+
+    def _extract_python_info(self, file_info: FileInfo, content: str):
+        """Extract Python-specific information using AST"""
+        try:
+            tree = ast.parse(content)
+
+            for node in ast.walk(tree):
+                # Extract function definitions
+                if isinstance(node, ast.FunctionDef):
+                    func_name = node.name
+                    file_info.functions.append(func_name)
+
+                    if func_name not in self.index.functions:
+                        self.index.functions[func_name] = []
+                    self.index.functions[func_name].append(file_info.relative_path)
+
+                # Extract class definitions
+                elif isinstance(node, ast.ClassDef):
+                    class_name = node.name
+                    file_info.classes.append(class_name)
+
+                    if class_name not in self.index.classes:
+                        self.index.classes[class_name] = []
+                    self.index.classes[class_name].append(file_info.relative_path)
+
+                # Extract imports
+                elif isinstance(node, (ast.Import, ast.ImportFrom)):
+                    if isinstance(node, ast.Import):
+                        for alias in node.names:
+                            module = alias.name
+                            file_info.imports.append(module)
+
+                            if module not in self.index.imports:
+                                self.index.imports[module] = set()
+                            self.index.imports[module].add(file_info.relative_path)
+
+                    elif isinstance(node, ast.ImportFrom):
+                        module = node.module or ''
+                        if module:
+                            file_info.imports.append(module)
+
+                            if module not in self.index.imports:
+                                self.index.imports[module] = set()
+                            self.index.imports[module].add(file_info.relative_path)
+
+        except SyntaxError:
+            # Skip files with syntax errors
+            pass
+
+    def get_file_info(self, relative_path: str) -> Optional[FileInfo]:
+        """Get information about a specific file"""
+        return self.index.files.get(relative_path)
+
+    def find_function(self, function_name: str) -> List[str]:
+        """Find all files containing a function"""
+        return self.index.functions.get(function_name, [])
+
+    def find_class(self, class_name: str) -> List[str]:
+        """Find all files containing a class"""
+        return self.index.classes.get(class_name, [])
+
+    def find_importers(self, module_name: str) -> Set[str]:
+        """Find all files that import a module"""
+        return self.index.imports.get(module_name, set())
+
+    def search_files(self, pattern: str) -> List[FileInfo]:
+        """Search for files matching a pattern"""
+        results = []
+        for relative_path, file_info in self.index.files.items():
+            if pattern.lower() in relative_path.lower():
+                results.append(file_info)
+        return results
+
+    def get_summary(self) -> str:
+        """Get a summary of the indexed codebase"""
+        summary = [
+            f"Codebase Summary",
+            f"=" * 80,
+            f"Root: {self.index.root_path}",
+            f"Total Files: {self.index.total_files}",
+            f"Total Lines: {self.index.total_lines:,}",
+            f"Total Size: {self.index.total_size / 1024:.1f} KB",
+            f"",
+            f"Functions: {len(self.index.functions)}",
+            f"Classes: {len(self.index.classes)}",
+            f"Modules: {len(self.index.imports)}",
+        ]
+
+        # Language breakdown
+        lang_counts = {}
+        for file_info in self.index.files.values():
+            lang = file_info.language
+            lang_counts[lang] = lang_counts.get(lang, 0) + 1
+
+        summary.append("")
+        summary.append("Files by Language:")
+        for lang, count in sorted(lang_counts.items(), key=lambda x: -x[1]):
+            summary.append(f"  {lang}: {count}")
+
+        return "\n".join(summary)

groknroll/oracle/oracle_agent.py

@@ -0,0 +1,278 @@
+"""
+Oracle Agent - RLM-powered codebase knowledge system
+
+The Oracle has unlimited context and knows everything about the codebase.
+"""
+
+from pathlib import Path
+from typing import Dict, Optional, List, Any
+from dataclasses import dataclass
+
+from groknroll.core.rlm import RLM
+from groknroll.oracle.codebase_indexer import CodebaseIndexer, CodebaseIndex
+
+
+@dataclass
+class OracleResponse:
+    """Response from the Oracle"""
+    question: str
+    answer: str
+    confidence: str  # high, medium, low
+    sources: List[str]  # Files referenced
+    execution_time: float
+
+    def __str__(self):
+        return f"""
+Oracle Response
+{'=' * 80}
+Question: {self.question}
+
+Answer:
+{self.answer}
+
+Confidence: {self.confidence}
+Sources: {', '.join(self.sources[:5])}{'...' if len(self.sources) > 5 else ''}
+Time: {self.execution_time:.2f}s
+{'=' * 80}
+"""
+
+
+class OracleAgent:
+    """
+    Oracle Agent - Knows everything about the codebase
+
+    Uses RLM's unlimited context to answer questions about:
+    - Code structure and architecture
+    - Function and class locations
+    - Import dependencies
+    - Implementation details
+    - Best practices in the codebase
+    - How to use features
+    - Where bugs might be
+    - Refactoring suggestions
+
+    Example:
+        oracle = OracleAgent("/path/to/project")
+        response = oracle.ask("Where is the RLM class defined?")
+        print(response.answer)
+    """
+
+    def __init__(
+        self,
+        project_path: Path,
+        backend: str = "openai",
+        model: str = "gpt-4o-mini",
+        verbose: bool = False
+    ):
+        """
+        Initialize Oracle Agent
+
+        Args:
+            project_path: Path to the project root
+            backend: LLM backend to use
+            model: Model name
+            verbose: Whether to show detailed output
+        """
+        self.project_path = Path(project_path).resolve()
+        self.verbose = verbose
+
+        print(f"🔮 Initializing Oracle Agent for: {self.project_path}")
+
+        # Build codebase index
+        self.indexer = CodebaseIndexer(self.project_path)
+        self.index = self.indexer.build_index(include_content=True)
+
+        # Initialize RLM
+        self.rlm = RLM(
+            backend=backend,
+            backend_kwargs={"model_name": model},
+            environment="local",
+            max_iterations=10,
+            verbose=verbose
+        )
+
+        print(f"✅ Oracle ready - {self.index.total_files} files indexed")
+
+    def ask(self, question: str, context: Optional[Dict[str, Any]] = None) -> OracleResponse:
+        """
+        Ask the Oracle a question about the codebase
+
+        Args:
+            question: Question about the codebase
+            context: Additional context to provide
+
+        Returns:
+            OracleResponse with answer and metadata
+        """
+        print(f"\n🔮 Oracle processing: {question[:80]}...")
+
+        # Build comprehensive context for RLM
+        rlm_context = self._build_context(question, context)
+
+        # Ask RLM
+        import time
+        start = time.time()
+
+        result = self.rlm.completion(rlm_context)
+
+        execution_time = time.time() - start
+
+        # Extract sources from response
+        sources = self._extract_sources(result.response)
+
+        return OracleResponse(
+            question=question,
+            answer=result.response,
+            confidence="high",  # Could be inferred from response
+            sources=sources,
+            execution_time=execution_time
+        )
+
+    def _build_context(self, question: str, extra_context: Optional[Dict] = None) -> str:
+        """Build comprehensive context for RLM"""
+
+        parts = [
+            "You are the Oracle - an all-knowing agent with complete knowledge of this codebase.",
+            "You have unlimited context and can examine any file in detail.",
+            "",
+            "# Codebase Overview",
+            self.indexer.get_summary(),
+            "",
+            "# Available Files",
+        ]
+
+        # Add file listing
+        for relative_path in sorted(self.index.files.keys())[:100]:  # Top 100 files
+            file_info = self.index.files[relative_path]
+            parts.append(f"- {relative_path} ({file_info.language}, {file_info.lines} lines)")
+
+        parts.append("")
+        parts.append("# Question")
+        parts.append(question)
+        parts.append("")
+
+        # Add relevant context based on question
+        parts.append("# Relevant Code")
+        parts.append("")
+
+        # Try to find relevant files
+        relevant_files = self._find_relevant_files(question)
+
+        for file_path in relevant_files[:5]:  # Top 5 most relevant
+            file_info = self.index.files.get(file_path)
+            if file_info and file_info.content:
+                parts.append(f"## File: {file_path}")
+                parts.append(f"```{file_info.language}")
+                parts.append(file_info.content)
+                parts.append("```")
+                parts.append("")
+
+        if extra_context:
+            parts.append("# Additional Context")
+            for key, value in extra_context.items():
+                parts.append(f"{key}: {value}")
+            parts.append("")
+
+        parts.append("# Instructions")
+        parts.append("Answer the question comprehensively using your knowledge of the codebase.")
+        parts.append("Reference specific files, functions, and classes when relevant.")
+        parts.append("Provide code examples if helpful.")
+        parts.append("Use FINAL_VAR(str(your_answer)) to return your answer.")
+
+        return "\n".join(parts)
+
+    def _find_relevant_files(self, question: str) -> List[str]:
+        """Find files relevant to the question"""
+        relevant = []
+        question_lower = question.lower()
+
+        # Check for specific file mentions
+        for file_path in self.index.files.keys():
+            file_name = Path(file_path).name.lower()
+            if file_name in question_lower or file_path in question_lower:
+                relevant.append(file_path)
+
+        # Check for class/function mentions
+        for class_name, file_paths in self.index.classes.items():
+            if class_name.lower() in question_lower:
+                relevant.extend(file_paths)
+
+        for func_name, file_paths in self.index.functions.items():
+            if func_name.lower() in question_lower:
+                relevant.extend(file_paths)
+
+        # Keywords that suggest core files
+        if any(kw in question_lower for kw in ['main', 'core', 'base', 'init']):
+            for file_path in self.index.files.keys():
+                if any(name in file_path for name in ['__init__.py', 'main.py', 'core/', 'rlm.py']):
+                    relevant.append(file_path)
+
+        # Remove duplicates while preserving order
+        seen = set()
+        unique_relevant = []
+        for item in relevant:
+            if item not in seen:
+                seen.add(item)
+                unique_relevant.append(item)
+
+        return unique_relevant if unique_relevant else list(self.index.files.keys())[:10]
+
+    def _extract_sources(self, response: str) -> List[str]:
+        """Extract file references from response"""
+        sources = set()
+
+        for file_path in self.index.files.keys():
+            if file_path in response or Path(file_path).name in response:
+                sources.add(file_path)
+
+        return list(sources)
+
+    def find_function(self, function_name: str) -> OracleResponse:
+        """Ask Oracle where a function is defined"""
+        return self.ask(f"Where is the function '{function_name}' defined? Show me the code.")
+
+    def find_class(self, class_name: str) -> OracleResponse:
+        """Ask Oracle where a class is defined"""
+        return self.ask(f"Where is the class '{class_name}' defined? Show me the code and explain what it does.")
+
+    def explain_file(self, file_path: str) -> OracleResponse:
+        """Ask Oracle to explain a file"""
+        return self.ask(f"Explain what {file_path} does and how it fits into the overall architecture.")
+
+    def find_usage(self, item_name: str) -> OracleResponse:
+        """Ask Oracle how something is used"""
+        return self.ask(f"Show me examples of how '{item_name}' is used in the codebase.")
+
+    def suggest_refactoring(self, description: str) -> OracleResponse:
+        """Ask Oracle for refactoring suggestions"""
+        return self.ask(f"Suggest how to refactor: {description}")
+
+    def find_bugs(self, area: Optional[str] = None) -> OracleResponse:
+        """Ask Oracle to identify potential bugs"""
+        question = f"Identify potential bugs in {area}" if area else "Identify potential bugs in the codebase"
+        return self.ask(question)
+
+    def get_architecture_overview(self) -> OracleResponse:
+        """Ask Oracle for architecture overview"""
+        return self.ask(
+            "Provide a comprehensive overview of the codebase architecture. "
+            "Explain the main components, how they interact, and the overall design pattern."
+        )
+
+    def how_to_add_feature(self, feature: str) -> OracleResponse:
+        """Ask Oracle how to add a feature"""
+        return self.ask(
+            f"How would I add this feature: {feature}? "
+            f"Which files would I need to modify? Show me the steps."
+        )
+
+    def trace_execution(self, entry_point: str) -> OracleResponse:
+        """Ask Oracle to trace execution flow"""
+        return self.ask(
+            f"Trace the execution flow starting from {entry_point}. "
+            f"Show me the call chain and what happens step by step."
+        )
+
+    def explain_imports(self, module: str) -> OracleResponse:
+        """Ask Oracle about import relationships"""
+        return self.ask(f"Explain the import relationships for {module}. Which files import it and why?")

groknroll/setup.py

@@ -0,0 +1,34 @@
+"""
+groknroll setup
+"""
+
+from setuptools import setup, find_packages
+
+setup(
+    name="groknroll",
+    version="2.0.0",
+    packages=find_packages(),
+    python_requires=">=3.9",
+    install_requires=[
+        "click>=8.1.0",
+        "rich>=13.0.0",
+        "blessed>=1.20.0",
+        "inquirer>=3.1.0",
+        "radon>=6.0.0",
+        "bandit>=1.7.0",
+        "gitpython>=3.1.0",
+        "diff-match-patch>=20230430",
+        "sqlalchemy>=2.0.0",
+        "pyyaml>=6.0.0",
+        "python-dotenv>=1.0.0",
+        "jinja2>=3.1.0",
+        "pytest>=7.4.0",
+        "coverage>=7.0.0",
+    ],
+    entry_points={
+        "console_scripts": [
+            "groknroll=groknroll.cli.main:main",
+            "gknr=groknroll.cli.main:main",
+        ],
+    },
+)

groknroll/storage/__init__.py

@@ -0,0 +1,14 @@
+"""
+groknroll Storage - SQLite database for project context and history
+"""
+
+from groknroll.storage.database import Database
+from groknroll.storage.models import (
+    Project,
+    FileIndex,
+    Execution,
+    Session,
+    Analysis
+)
+
+__all__ = ["Database", "Project", "FileIndex", "Execution", "Session", "Analysis"]