jarviscore-framework 0.1.0__py3-none-any.whl

jarviscore/execution/search.py

@@ -0,0 +1,281 @@
+"""
+Internet Search - Zero-config web search and content extraction
+Uses DuckDuckGo (no API key required)
+"""
+import logging
+import aiohttp
+import asyncio
+import re
+from typing import Dict, Any, List, Optional
+from bs4 import BeautifulSoup
+from urllib.parse import quote_plus, urlparse
+
+logger = logging.getLogger(__name__)
+
+
+class InternetSearch:
+    """
+    Zero-config internet search with content extraction.
+
+    Features:
+    - Search web using DuckDuckGo (no API key needed)
+    - Extract clean text from web pages
+    - Automatic error handling and retries
+    - Combined search + extract in one call
+
+    Example:
+        search = InternetSearch()
+        results = await search.search("Python async programming")
+        content = await search.extract_content(results[0]['url'])
+    """
+
+    def __init__(self, user_agent: Optional[str] = None):
+        """
+        Initialize search module.
+
+        Args:
+            user_agent: Optional custom user agent (auto-set if None)
+        """
+        self.session = None
+        self.user_agent = user_agent or (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        )
+
+    async def initialize(self):
+        """Initialize HTTP session (auto-called on first use)."""
+        if self.session is None or self.session.closed:
+            self.session = aiohttp.ClientSession(
+                timeout=aiohttp.ClientTimeout(total=30),
+                headers={
+                    "User-Agent": self.user_agent,
+                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                    "Accept-Language": "en-US,en;q=0.9"
+                }
+            )
+            logger.debug("HTTP session initialized")
+
+    async def close(self):
+        """Close HTTP session."""
+        if self.session and not self.session.closed:
+            await self.session.close()
+            self.session = None
+            logger.debug("HTTP session closed")
+
+    async def __aenter__(self):
+        """Async context manager entry."""
+        await self.initialize()
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit."""
+        await self.close()
+        return False
+
+    async def search(self, query: str, max_results: int = 5) -> List[Dict[str, Any]]:
+        """
+        Search the web using DuckDuckGo.
+
+        Args:
+            query: Search query
+            max_results: Maximum number of results (default 5)
+
+        Returns:
+            List of results: [{"title": "...", "snippet": "...", "url": "..."}]
+
+        Example:
+            results = await search.search("Python async tutorial")
+            print(results[0]['title'])
+        """
+        await self.initialize()
+
+        encoded_query = quote_plus(query)
+        logger.info(f"Searching: {query}")
+
+        try:
+            results = await self._search_duckduckgo(encoded_query)
+
+            # Normalize URLs
+            for result in results:
+                url = result.get('url', '')
+                if url and not url.startswith(('http://', 'https://')):
+                    result['url'] = 'https://' + url
+
+            results = results[:max_results]
+            logger.info(f"Found {len(results)} results")
+            return results
+
+        except Exception as e:
+            logger.error(f"Search failed: {e}")
+            return []
+
+    async def _search_duckduckgo(self, encoded_query: str) -> List[Dict[str, Any]]:
+        """Perform DuckDuckGo search."""
+        url = f"https://html.duckduckgo.com/html/?q={encoded_query}"
+
+        try:
+            async with self.session.get(url) as response:
+                if response.status != 200:
+                    raise RuntimeError(f"DuckDuckGo returned {response.status}")
+
+                html = await response.text()
+                soup = BeautifulSoup(html, 'html.parser')
+
+                results = []
+                for result_elem in soup.select('.result'):
+                    title_elem = result_elem.select_one('.result__title')
+                    snippet_elem = result_elem.select_one('.result__snippet')
+                    url_elem = result_elem.select_one('.result__url')
+
+                    if title_elem and url_elem:
+                        # Clean up DuckDuckGo's tracking URLs
+                        url_text = url_elem.get_text(strip=True)
+                        # Remove "duckduckgo.com" prefix if present
+                        url_text = re.sub(r'^.*?://', '', url_text)
+                        url_text = url_text.split('?')[0]  # Remove query params
+
+                        results.append({
+                            "title": title_elem.get_text(strip=True),
+                            "snippet": snippet_elem.get_text(strip=True) if snippet_elem else "",
+                            "url": url_text
+                        })
+
+                return results
+
+        except Exception as e:
+            logger.error(f"DuckDuckGo search error: {e}")
+            return []
+
+    async def extract_content(self, url: str, max_length: int = 10000) -> Dict[str, Any]:
+        """
+        Extract clean text content from a webpage.
+
+        Args:
+            url: URL to extract from
+            max_length: Maximum content length (default 10k chars)
+
+        Returns:
+            {
+                "url": "https://...",
+                "title": "Page Title",
+                "content": "Clean extracted text...",
+                "success": True,
+                "word_count": 1234
+            }
+
+        Example:
+            content = await search.extract_content("https://example.com")
+            print(content['content'])
+        """
+        await self.initialize()
+
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        logger.info(f"Extracting content from: {url}")
+
+        try:
+            async with self.session.get(url) as response:
+                if response.status != 200:
+                    return {
+                        "url": url,
+                        "success": False,
+                        "error": f"HTTP {response.status}"
+                    }
+
+                html = await response.text()
+                soup = BeautifulSoup(html, 'html.parser')
+
+                # Extract title
+                title = soup.title.string if soup.title else urlparse(url).netloc
+
+                # Remove script, style, and other non-content elements
+                for element in soup(['script', 'style', 'nav', 'footer', 'header', 'aside']):
+                    element.decompose()
+
+                # Extract main content
+                # Try to find main content area
+                main_content = (
+                    soup.find('main') or
+                    soup.find('article') or
+                    soup.find('div', class_=re.compile(r'content|main|article', re.I)) or
+                    soup.body
+                )
+
+                if main_content:
+                    text = main_content.get_text(separator='\n', strip=True)
+                else:
+                    text = soup.get_text(separator='\n', strip=True)
+
+                # Clean up text
+                lines = [line.strip() for line in text.split('\n') if line.strip()]
+                clean_text = '\n'.join(lines)
+
+                # Truncate if too long
+                if len(clean_text) > max_length:
+                    clean_text = clean_text[:max_length] + "... [truncated]"
+
+                word_count = len(clean_text.split())
+
+                logger.info(f"Extracted {word_count} words from {url}")
+
+                return {
+                    "url": url,
+                    "title": title.strip() if title else "",
+                    "content": clean_text,
+                    "success": True,
+                    "word_count": word_count
+                }
+
+        except Exception as e:
+            logger.error(f"Content extraction failed for {url}: {e}")
+            return {
+                "url": url,
+                "success": False,
+                "error": str(e)
+            }
+
+    async def search_and_extract(
+        self,
+        query: str,
+        num_results: int = 3,
+        max_content_length: int = 5000
+    ) -> List[Dict[str, Any]]:
+        """
+        Search and automatically extract content from top results.
+
+        Args:
+            query: Search query
+            num_results: Number of results to extract (default 3)
+            max_content_length: Max content per page (default 5k chars)
+
+        Returns:
+            List of extracted content from top search results
+
+        Example:
+            results = await search.search_and_extract("Python asyncio tutorial")
+            for result in results:
+                print(f"{result['title']}: {result['content'][:100]}...")
+        """
+        # Search first
+        search_results = await self.search(query, max_results=num_results)
+
+        # Extract content from each result
+        extracted = []
+        for result in search_results:
+            content = await self.extract_content(result['url'], max_content_length)
+            if content.get('success'):
+                # Merge search metadata with extracted content
+                extracted.append({
+                    **result,
+                    **content
+                })
+
+        logger.info(f"Extracted content from {len(extracted)}/{len(search_results)} results")
+        return extracted
+
+
+def create_search_client() -> InternetSearch:
+    """Factory function to create search client."""
+    return InternetSearch()

jarviscore/orchestration/__init__.py

@@ -0,0 +1,18 @@
+"""
+Orchestration module for JarvisCore Framework
+
+Workflow execution engine with dependency management.
+"""
+
+from .engine import WorkflowEngine
+from .claimer import StepClaimer
+from .dependency import DependencyManager
+from .status import StatusManager, StepStatus
+
+__all__ = [
+    'WorkflowEngine',
+    'StepClaimer',
+    'DependencyManager',
+    'StatusManager',
+    'StepStatus',
+]

jarviscore/orchestration/claimer.py

@@ -0,0 +1,101 @@
+"""
+Step Claimer - Matches workflow steps to capable agents
+
+Simplified from integration-agent (260 lines → 80 lines)
+Focused on capability matching without Kafka integration.
+"""
+import logging
+from typing import List, Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class StepClaimer:
+    """
+    Matches workflow steps to agents based on capabilities.
+
+    Simplified from integration-agent's version.
+    Removes: Lazy evaluation, external dependency tracking, Kafka claims
+    Keeps: Core capability matching logic
+    """
+
+    def __init__(self, agents: List):
+        """
+        Initialize step claimer with available agents.
+
+        Args:
+            agents: List of Agent instances available for claiming
+        """
+        self.agents = agents
+        self._capability_index = self._build_capability_index()
+        logger.info(f"Step claimer initialized with {len(agents)} agents")
+
+    def _build_capability_index(self) -> Dict[str, List]:
+        """Build index of capabilities to agents."""
+        index = {}
+        for agent in self.agents:
+            # Index by role
+            if agent.role not in index:
+                index[agent.role] = []
+            index[agent.role].append(agent)
+
+            # Index by each capability
+            for cap in agent.capabilities:
+                if cap not in index:
+                    index[cap] = []
+                index[cap].append(agent)
+
+        logger.debug(f"Built capability index with {len(index)} entries")
+        return index
+
+    def find_agent(self, step: Dict[str, Any]) -> Optional[Any]:
+        """
+        Find agent capable of executing this step.
+
+        Args:
+            step: Step specification containing:
+                - agent: Role or capability required
+                - role: Alternative key for agent role
+                - capability: Alternative key for capability
+
+        Returns:
+            Agent instance that can handle the step, or None
+
+        Example:
+            step = {"agent": "scraper", "task": "Scrape website"}
+            agent = claimer.find_agent(step)
+        """
+        # Try different keys for agent requirement
+        required = (
+            step.get("agent") or
+            step.get("role") or
+            step.get("capability")
+        )
+
+        if not required:
+            logger.warning(f"Step has no agent/role/capability specified: {step}")
+            return None
+
+        # Look up in capability index
+        agents = self._capability_index.get(required, [])
+
+        if not agents:
+            logger.warning(f"No agent found for requirement: {required}")
+            return None
+
+        # Return first matching agent
+        agent = agents[0]
+        logger.debug(f"Matched step to agent: {agent.agent_id}")
+        return agent
+
+    def find_all_agents(self, capability: str) -> List[Any]:
+        """
+        Find all agents with a specific capability.
+
+        Args:
+            capability: Required capability
+
+        Returns:
+            List of agents with the capability
+        """
+        return self._capability_index.get(capability, [])

jarviscore/orchestration/dependency.py

@@ -0,0 +1,143 @@
+"""
+Dependency Manager - Resolves step dependencies
+
+Simplified from integration-agent
+Removes: Kafka integration, complex P2P queries
+Keeps: Memory cache, basic waiting logic
+"""
+import logging
+import asyncio
+from typing import Dict, List, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class DependencyManager:
+    """
+    Manages step dependencies and resolution.
+
+    Simplified from integration-agent's 3-tier system:
+    - Tier 1: Memory cache (kept)
+    - Tier 2: P2P queries (simplified - future)
+    - Tier 3: Kafka (removed for MVP)
+    """
+
+    def __init__(self, memory_cache: Optional[Dict] = None):
+        """
+        Initialize dependency manager.
+
+        Args:
+            memory_cache: Optional shared memory cache for step outputs
+        """
+        self.memory = memory_cache or {}
+        self.waiting_steps: Dict[str, List[str]] = {}  # step_id -> [dep_ids]
+        logger.info("Dependency manager initialized")
+
+    async def wait_for(
+        self,
+        dependencies: List[str],
+        memory: Dict[str, Any],
+        timeout: float = 300.0
+    ) -> Dict[str, Any]:
+        """
+        Wait for dependencies to be satisfied.
+
+        Args:
+            dependencies: List of step IDs this step depends on
+            memory: Workflow memory containing step outputs
+            timeout: Maximum time to wait in seconds
+
+        Returns:
+            Dictionary of dependency_id -> output
+
+        Raises:
+            TimeoutError: If dependencies not satisfied within timeout
+            ValueError: If required dependency not found
+
+        Example:
+            # Step 2 depends on step 1
+            deps = await manager.wait_for(['step1'], memory)
+            input_data = deps['step1']['output']
+        """
+        if not dependencies:
+            return {}
+
+        logger.info(f"Waiting for {len(dependencies)} dependencies: {dependencies}")
+
+        start_time = asyncio.get_event_loop().time()
+        resolved = {}
+
+        for dep_id in dependencies:
+            # Check if already in memory
+            if dep_id in memory:
+                resolved[dep_id] = memory[dep_id]
+                logger.debug(f"Dependency {dep_id} found in memory")
+                continue
+
+            # Wait for dependency to appear in memory
+            logger.info(f"Waiting for dependency: {dep_id}")
+            while dep_id not in memory:
+                if asyncio.get_event_loop().time() - start_time > timeout:
+                    raise TimeoutError(
+                        f"Dependency {dep_id} not satisfied within {timeout}s"
+                    )
+
+                await asyncio.sleep(0.5)  # Poll every 500ms
+
+            resolved[dep_id] = memory[dep_id]
+            logger.debug(f"Dependency {dep_id} satisfied")
+
+        logger.info(f"All dependencies satisfied: {list(resolved.keys())}")
+        return resolved
+
+    def check_dependencies(
+        self,
+        dependencies: List[str],
+        memory: Dict[str, Any]
+    ) -> tuple[bool, List[str]]:
+        """
+        Check if dependencies are satisfied (non-blocking).
+
+        Args:
+            dependencies: List of step IDs to check
+            memory: Workflow memory
+
+        Returns:
+            Tuple of (all_satisfied, missing_deps)
+        """
+        if not dependencies:
+            return True, []
+
+        missing = [dep for dep in dependencies if dep not in memory]
+
+        if missing:
+            logger.debug(f"Missing dependencies: {missing}")
+            return False, missing
+
+        return True, []
+
+    def register_waiting(self, step_id: str, dependencies: List[str]):
+        """
+        Register a step as waiting for dependencies.
+
+        Args:
+            step_id: Step that is waiting
+            dependencies: List of dependency step IDs
+        """
+        self.waiting_steps[step_id] = dependencies
+        logger.debug(f"Step {step_id} waiting for: {dependencies}")
+
+    def resolve_step(self, step_id: str):
+        """
+        Mark a step as resolved (completed).
+
+        Args:
+            step_id: Step that has been completed
+        """
+        if step_id in self.waiting_steps:
+            del self.waiting_steps[step_id]
+            logger.debug(f"Step {step_id} resolved")
+
+    def get_waiting_steps(self) -> Dict[str, List[str]]:
+        """Get all steps currently waiting for dependencies."""
+        return self.waiting_steps.copy()