jarviscore-framework 0.1.0__py3-none-any.whl

jarviscore/execution/__init__.py

@@ -0,0 +1,94 @@
+"""
+Execution Engine - Complete code generation and execution pipeline
+
+Zero-config components:
+- UnifiedLLMClient: Multi-provider LLM (vLLM, Azure, Gemini, Claude)
+- InternetSearch: Web search and content extraction (DuckDuckGo)
+- CodeGenerator: Natural language → Python code
+- SandboxExecutor: Safe code execution with limits
+- AutonomousRepair: Automatic error fixing
+
+Everything works out of the box - just pass config dict.
+"""
+
+# LLM Client
+from .llm import (
+    UnifiedLLMClient,
+    LLMProvider,
+    TOKEN_PRICING,
+    create_llm_client
+)
+
+# Internet Search
+from .search import (
+    InternetSearch,
+    create_search_client
+)
+
+# Code Generator
+from .generator import (
+    CodeGenerator,
+    create_code_generator
+)
+
+# Sandbox Executor
+from .sandbox import (
+    SandboxExecutor,
+    ExecutionTimeout,
+    create_sandbox_executor
+)
+
+# Autonomous Repair
+from .repair import (
+    AutonomousRepair,
+    create_autonomous_repair
+)
+
+# Result Handler
+from .result_handler import (
+    ResultHandler,
+    ResultStatus,
+    ErrorCategory,
+    create_result_handler
+)
+
+# Code Registry
+from .code_registry import (
+    CodeRegistry,
+    create_code_registry
+)
+
+__all__ = [
+    # LLM
+    'UnifiedLLMClient',
+    'LLMProvider',
+    'TOKEN_PRICING',
+    'create_llm_client',
+
+    # Search
+    'InternetSearch',
+    'create_search_client',
+
+    # Generator
+    'CodeGenerator',
+    'create_code_generator',
+
+    # Sandbox
+    'SandboxExecutor',
+    'ExecutionTimeout',
+    'create_sandbox_executor',
+
+    # Repair
+    'AutonomousRepair',
+    'create_autonomous_repair',
+
+    # Result Handler
+    'ResultHandler',
+    'ResultStatus',
+    'ErrorCategory',
+    'create_result_handler',
+
+    # Code Registry
+    'CodeRegistry',
+    'create_code_registry',
+]

jarviscore/execution/code_registry.py

@@ -0,0 +1,298 @@
+"""
+Code Registry - Store and retrieve generated code functions
+
+Purpose:
+- Store successfully generated code for reuse
+- Index by agent_id, capabilities, task patterns
+- Retrieve similar code for future tasks
+- File-based storage (no external dependencies)
+
+Storage Structure:
+./logs/code_registry/
+├─ index.json  # Metadata index for fast search
+└─ functions/
+    ├─ {function_id}__{hash}.py  # Generated code files
+    └─ ...
+"""
+import json
+import hashlib
+import logging
+import re
+from pathlib import Path
+from typing import Dict, Any, List, Optional
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+
+class CodeRegistry:
+    """
+    Registry for storing and retrieving generated code.
+
+    Features:
+    - Auto-register successful executions
+    - Search by agent capabilities and task patterns
+    - File-based storage with JSON index
+    - No external dependencies
+
+    Example:
+        registry = CodeRegistry()
+
+        # Register generated code
+        registry.register(
+            code="result = 2 + 2",
+            agent_id="calculator-123",
+            task="Calculate 2+2",
+            capabilities=["math", "arithmetic"],
+            output=4
+        )
+
+        # Search for similar code
+        matches = registry.search(
+            capabilities=["math"],
+            task_pattern="factorial"
+        )
+    """
+
+    def __init__(self, registry_dir: str = "./logs/code_registry"):
+        """
+        Initialize code registry.
+
+        Args:
+            registry_dir: Base directory for code storage
+        """
+        self.registry_dir = Path(registry_dir)
+        self.functions_dir = self.registry_dir / "functions"
+        self.index_file = self.registry_dir / "index.json"
+
+        # Create directories
+        self.functions_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load or create index
+        self._index = self._load_index()
+
+        logger.info(f"CodeRegistry initialized: {self.registry_dir}")
+        logger.info(f"Indexed functions: {len(self._index)}")
+
+    def register(
+        self,
+        code: str,
+        agent_id: str,
+        task: str,
+        capabilities: List[str],
+        output: Any,
+        result_id: Optional[str] = None,
+        metadata: Optional[Dict] = None
+    ) -> str:
+        """
+        Register generated code in the registry.
+
+        Args:
+            code: Generated Python code
+            agent_id: ID of agent that generated code
+            task: Original task description
+            capabilities: Agent capabilities list
+            output: Execution output (for verification)
+            result_id: Optional result_id from execution
+            metadata: Additional metadata
+
+        Returns:
+            function_id: Unique identifier for registered function
+        """
+        # Generate function ID
+        code_hash = self._hash_code(code)
+        function_id = f"{agent_id}_{code_hash[:8]}"
+
+        # Check if already registered
+        if function_id in self._index:
+            logger.debug(f"Code already registered: {function_id}")
+            return function_id
+
+        # Store code to file
+        code_file = self.functions_dir / f"{function_id}.py"
+        code_file.write_text(code)
+
+        # Add to index
+        self._index[function_id] = {
+            "function_id": function_id,
+            "agent_id": agent_id,
+            "task": task,
+            "capabilities": capabilities,
+            "code_hash": code_hash,
+            "code_file": str(code_file.relative_to(self.registry_dir)),
+            "output_sample": str(output)[:100],  # First 100 chars
+            "result_id": result_id,
+            "registered_at": datetime.now().isoformat(),
+            "metadata": metadata or {},
+            # Extract task keywords for search
+            "task_keywords": self._extract_keywords(task)
+        }
+
+        # Save index
+        self._save_index()
+
+        logger.info(f"Registered code: {function_id} (task: {task[:50]}...)")
+        return function_id
+
+    def get(self, function_id: str) -> Optional[Dict[str, Any]]:
+        """
+        Get function metadata and code by ID.
+
+        Args:
+            function_id: Function identifier
+
+        Returns:
+            Dictionary with metadata and code, or None if not found
+        """
+        if function_id not in self._index:
+            return None
+
+        entry = self._index[function_id].copy()
+
+        # Load code from file
+        code_file = self.registry_dir / entry["code_file"]
+        if code_file.exists():
+            entry["code"] = code_file.read_text()
+        else:
+            logger.warning(f"Code file missing for {function_id}")
+            entry["code"] = None
+
+        return entry
+
+    def search(
+        self,
+        capabilities: Optional[List[str]] = None,
+        task_pattern: Optional[str] = None,
+        agent_id: Optional[str] = None,
+        limit: int = 10
+    ) -> List[Dict[str, Any]]:
+        """
+        Search for matching code in registry.
+
+        Args:
+            capabilities: Filter by agent capabilities
+            task_pattern: Search in task descriptions
+            agent_id: Filter by agent ID
+            limit: Maximum results to return
+
+        Returns:
+            List of matching function metadata (sorted by relevance)
+        """
+        matches = []
+
+        for function_id, entry in self._index.items():
+            score = 0
+
+            # Filter by agent_id
+            if agent_id and entry["agent_id"] != agent_id:
+                continue
+
+            # Score by capability overlap
+            if capabilities:
+                entry_caps = set(entry["capabilities"])
+                query_caps = set(capabilities)
+                overlap = len(entry_caps & query_caps)
+                if overlap == 0:
+                    continue  # Must have at least one matching capability
+                score += overlap * 10
+
+            # Score by task keyword match
+            if task_pattern:
+                pattern_keywords = self._extract_keywords(task_pattern)
+                entry_keywords = set(entry["task_keywords"])
+                keyword_overlap = len(set(pattern_keywords) & entry_keywords)
+                if keyword_overlap > 0:
+                    score += keyword_overlap * 5
+
+            # Add to matches with score
+            match = entry.copy()
+            match["_score"] = score
+            matches.append(match)
+
+        # Sort by score (descending)
+        matches.sort(key=lambda x: x["_score"], reverse=True)
+
+        # Return top matches
+        return matches[:limit]
+
+    def list_all(self, limit: int = 100) -> List[Dict[str, Any]]:
+        """
+        List all registered functions (most recent first).
+
+        Args:
+            limit: Maximum results to return
+
+        Returns:
+            List of function metadata
+        """
+        entries = list(self._index.values())
+
+        # Sort by registration time (most recent first)
+        entries.sort(
+            key=lambda x: x.get("registered_at", ""),
+            reverse=True
+        )
+
+        return entries[:limit]
+
+    def _hash_code(self, code: str) -> str:
+        """Generate hash of code for deduplication."""
+        # Normalize code (remove whitespace variations)
+        normalized = re.sub(r'\s+', ' ', code.strip())
+        return hashlib.md5(normalized.encode()).hexdigest()
+
+    def _extract_keywords(self, text: str) -> List[str]:
+        """Extract meaningful keywords from text."""
+        # Convert to lowercase
+        text = text.lower()
+
+        # Remove common words
+        stopwords = {
+            'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+            'of', 'with', 'by', 'from', 'as', 'is', 'was', 'are', 'were', 'been',
+            'be', 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would',
+            'should', 'could', 'may', 'might', 'must', 'can'
+        }
+
+        # Extract words (alphanumeric only)
+        words = re.findall(r'\b[a-z0-9]+\b', text)
+
+        # Filter stopwords and short words
+        keywords = [w for w in words if w not in stopwords and len(w) > 2]
+
+        return keywords
+
+    def _load_index(self) -> Dict:
+        """Load index from file."""
+        if self.index_file.exists():
+            try:
+                with open(self.index_file, 'r') as f:
+                    index = json.load(f)
+                    logger.debug(f"Loaded index with {len(index)} entries")
+                    return index
+            except Exception as e:
+                logger.error(f"Failed to load index: {e}")
+                return {}
+        return {}
+
+    def _save_index(self):
+        """Save index to file."""
+        try:
+            with open(self.index_file, 'w') as f:
+                json.dump(self._index, f, indent=2)
+            logger.debug(f"Saved index with {len(self._index)} entries")
+        except Exception as e:
+            logger.error(f"Failed to save index: {e}")
+
+
+def create_code_registry(registry_dir: str = "./logs/code_registry") -> CodeRegistry:
+    """
+    Factory function to create code registry.
+
+    Args:
+        registry_dir: Directory for code storage
+
+    Returns:
+        CodeRegistry instance
+    """
+    return CodeRegistry(registry_dir)

jarviscore/execution/generator.py

@@ -0,0 +1,268 @@
+"""
+Code Generator - LLM-based Python code generation from natural language
+Auto-injects internet search capabilities when needed
+"""
+import ast
+import logging
+from typing import Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class CodeGenerator:
+    """
+    Zero-config code generator using LLM.
+
+    Philosophy:
+    - Developer writes natural language task
+    - Framework generates executable Python code
+    - Auto-injects search tools if task needs web data
+    - Validates syntax before returning
+
+    Example:
+        gen = CodeGenerator(llm_client, search_client)
+        code = await gen.generate(
+            task="Search for Python tutorials and count results",
+            system_prompt="You are a Python expert"
+        )
+    """
+
+    def __init__(self, llm_client, search_client=None):
+        """
+        Initialize code generator.
+
+        Args:
+            llm_client: UnifiedLLMClient instance
+            search_client: Optional InternetSearch instance (auto-injects if provided)
+        """
+        self.llm = llm_client
+        self.search = search_client
+
+        # Prompt templates
+        self.base_template = self._load_base_template()
+        self.search_template = self._load_search_template()
+
+    def _load_base_template(self) -> str:
+        """Base prompt template for code generation."""
+        return """You are an expert Python programmer. Generate clean, working Python code to accomplish the given task.
+
+CRITICAL REQUIREMENTS:
+1. Write ONLY executable Python code (no explanations, markdown, or comments outside code)
+2. Store the final result in a variable named 'result'
+3. Use standard libraries when possible (requests, json, re, etc.)
+4. Handle errors gracefully with try/except
+5. Do NOT use input() or any blocking operations
+6. Do NOT print anything unless explicitly requested
+7. Make the code self-contained and complete
+
+AVAILABLE TOOLS:
+- Standard library: os, sys, json, re, datetime, math, random, etc.
+- HTTP requests: import requests
+- Data processing: import pandas (if installed)
+- Any pip-installable package (assume it's available)
+
+TASK:
+{task}
+
+AGENT CONTEXT:
+{system_prompt}
+
+Generate the Python code now (code only, no explanations):
+"""
+
+    def _load_search_template(self) -> str:
+        """Template when internet search is available."""
+        return """You are an expert Python programmer with internet search capabilities. Generate clean, working Python code to accomplish the given task.
+
+CRITICAL REQUIREMENTS:
+1. Write ONLY executable Python code (no explanations, markdown, or comments outside code)
+2. Store the final result in a variable named 'result'
+3. Use standard libraries when possible
+4. Handle errors gracefully with try/except
+5. Do NOT use input() or any blocking operations
+6. Do NOT print anything unless explicitly requested
+
+INTERNET SEARCH AVAILABLE:
+You have access to a 'search' object with these async methods:
+- await search.search(query, max_results=5) -> List[Dict] with 'title', 'snippet', 'url'
+- await search.extract_content(url, max_length=10000) -> Dict with 'title', 'content', 'success'
+- await search.search_and_extract(query, num_results=3) -> List[Dict] combined results
+
+Example using search (define async main, sandbox will execute it):
+```python
+async def main():
+    # Search the web
+    results = await search.search("Python async tutorial")
+
+    # Extract content from first result
+    if results:
+        content = await search.extract_content(results[0]['url'])
+
+        # Process and return result
+        result = {{
+            'title': content['title'],
+            'summary': content['content'][:500]
+        }}
+        return result
+    return None
+```
+
+IMPORTANT:
+- Define async def main() - the sandbox will call it automatically
+- Do NOT use asyncio.run() - you're already in an async context
+- Store final result by returning from main() or assigning to 'result' variable
+
+TASK:
+{task}
+
+AGENT CONTEXT:
+{system_prompt}
+
+Generate the Python code now (code only, no explanations):
+"""
+
+    async def generate(
+        self,
+        task: Dict[str, Any],
+        system_prompt: str,
+        context: Optional[Dict] = None,
+        enable_search: bool = True
+    ) -> str:
+        """
+        Generate Python code for a task.
+
+        Args:
+            task: Task dict with 'task' key (natural language description)
+            system_prompt: Agent's system prompt (role/expertise)
+            context: Optional context (dependencies, previous results)
+            enable_search: Auto-inject search tools if available (default True)
+
+        Returns:
+            Generated Python code as string
+
+        Raises:
+            ValueError: If generated code has syntax errors
+            RuntimeError: If LLM generation fails
+
+        Example:
+            code = await gen.generate(
+                task={"task": "Calculate factorial of 10"},
+                system_prompt="You are a math expert"
+            )
+        """
+        task_description = task.get('task', '')
+        logger.info(f"Generating code for: {task_description[:80]}...")
+
+        # Decide whether to inject search tools
+        use_search = enable_search and self.search is not None
+        if use_search:
+            logger.debug("Search tools available - using search template")
+            template = self.search_template
+        else:
+            template = self.base_template
+
+        # Build prompt
+        prompt = template.format(
+            task=task_description,
+            system_prompt=system_prompt
+        )
+
+        # Add context if provided
+        if context:
+            prompt += f"\n\nCONTEXT FROM PREVIOUS STEPS:\n{self._format_context(context)}\n"
+
+        # Generate code via LLM
+        try:
+            response = await self.llm.generate(
+                prompt=prompt,
+                temperature=0.3,  # Lower temp for code generation
+                max_tokens=4000
+            )
+
+            code = response['content']
+            logger.debug(f"Generated {len(code)} chars of code")
+
+            # Clean up code (remove markdown blocks if present)
+            code = self._clean_code(code)
+
+            # Validate syntax
+            self._validate_code(code)
+
+            logger.info("Code generation successful")
+            return code
+
+        except Exception as e:
+            logger.error(f"Code generation failed: {e}")
+            raise RuntimeError(f"Failed to generate code: {e}")
+
+    def _clean_code(self, code: str) -> str:
+        """
+        Clean up generated code (remove markdown, explanations).
+
+        Args:
+            code: Raw LLM output
+
+        Returns:
+            Clean Python code
+        """
+        # Remove markdown code blocks
+        import re
+
+        # Pattern: ```python ... ``` or ```...```
+        pattern = r'```(?:python)?\s*\n(.*?)\n```'
+        matches = re.findall(pattern, code, re.DOTALL)
+
+        if matches:
+            # Use the first code block found
+            code = matches[0]
+
+        # Remove leading/trailing whitespace
+        code = code.strip()
+
+        # Remove any lines that are just comments at the start
+        lines = code.split('\n')
+        while lines and lines[0].strip().startswith('#'):
+            lines.pop(0)
+
+        code = '\n'.join(lines)
+
+        return code
+
+    def _validate_code(self, code: str):
+        """
+        Validate Python syntax.
+
+        Args:
+            code: Python code string
+
+        Raises:
+            ValueError: If code has syntax errors
+        """
+        try:
+            ast.parse(code)
+            logger.debug("Code syntax validation passed")
+        except SyntaxError as e:
+            logger.error(f"Syntax error in generated code: {e}")
+            logger.error(f"Problematic code:\n{code}")
+            raise ValueError(f"Generated code has syntax errors: {e}")
+
+    def _format_context(self, context: Dict) -> str:
+        """Format context from previous steps for prompt."""
+        parts = []
+        for key, value in context.items():
+            parts.append(f"{key}: {value}")
+        return "\n".join(parts)
+
+
+def create_code_generator(llm_client, search_client=None) -> CodeGenerator:
+    """
+    Factory function to create code generator.
+
+    Args:
+        llm_client: LLM client instance
+        search_client: Optional search client (auto-injected if provided)
+
+    Returns:
+        CodeGenerator instance
+    """
+    return CodeGenerator(llm_client, search_client)