crca 1.4.0__py3-none-any.whl → 1.5.0__py3-none-any.whl

utils/agent_discovery.py

@@ -0,0 +1,311 @@
+"""
+Agent discovery utilities.
+
+Provides functionality for:
+- Auto-discovery of AOP instances
+- Auto-discovery of router instances
+- Agent listing and metadata
+- Route-first routing helpers
+"""
+
+import inspect
+import sys
+from typing import Any, Dict, List, Optional, Union
+from loguru import logger
+
+# Try to import AOP and Router
+try:
+    from utils.aop import AOP
+    AOP_AVAILABLE = True
+except ImportError:
+    AOP = None
+    AOP_AVAILABLE = False
+    logger.debug("AOP not available for agent discovery")
+
+try:
+    from utils.router import SwarmRouter
+    ROUTER_AVAILABLE = True
+except ImportError:
+    SwarmRouter = None
+    ROUTER_AVAILABLE = False
+    logger.debug("Router not available for agent discovery")
+
+
+def discover_aop_instances() -> List[Any]:
+    """Auto-discover AOP instances in the current runtime.
+    
+    Searches for AOP instances in:
+    - Global namespace
+    - Module-level variables
+    - Active objects
+    
+    Returns:
+        List of discovered AOP instances
+    """
+    if not AOP_AVAILABLE:
+        return []
+    
+    instances = []
+    
+    # Search in global namespace
+    try:
+        frame = sys._getframe(1)
+        while frame:
+            for name, obj in frame.f_globals.items():
+                if isinstance(obj, AOP):
+                    instances.append(obj)
+            frame = frame.f_back
+    except Exception as e:
+        logger.debug(f"Error discovering AOP instances from frames: {e}")
+    
+    # Search in module-level variables (limited approach)
+    # This is a best-effort discovery
+    try:
+        import gc
+        for obj in gc.get_objects():
+            if isinstance(obj, AOP):
+                if obj not in instances:
+                    instances.append(obj)
+    except Exception as e:
+        logger.debug(f"Error discovering AOP instances from GC: {e}")
+    
+    logger.debug(f"Discovered {len(instances)} AOP instance(s)")
+    return instances
+
+
+def discover_router_instances() -> List[Any]:
+    """Auto-discover router instances in the current runtime.
+    
+    Searches for router instances in:
+    - Global namespace
+    - Module-level variables
+    - Active objects
+    
+    Returns:
+        List of discovered router instances
+    """
+    if not ROUTER_AVAILABLE:
+        return []
+    
+    instances = []
+    
+    # Search in global namespace
+    try:
+        frame = sys._getframe(1)
+        while frame:
+            for name, obj in frame.f_globals.items():
+                if isinstance(obj, SwarmRouter):
+                    instances.append(obj)
+            frame = frame.f_back
+    except Exception as e:
+        logger.debug(f"Error discovering router instances from frames: {e}")
+    
+    # Search in module-level variables
+    try:
+        import gc
+        for obj in gc.get_objects():
+            if isinstance(obj, SwarmRouter):
+                if obj not in instances:
+                    instances.append(obj)
+    except Exception as e:
+        logger.debug(f"Error discovering router instances from GC: {e}")
+    
+    logger.debug(f"Discovered {len(instances)} router instance(s)")
+    return instances
+
+
+def get_agents_from_aop(aop_instance: Any) -> Dict[str, Any]:
+    """Get list of agents from an AOP instance.
+    
+    Args:
+        aop_instance: AOP instance to query
+        
+    Returns:
+        Dictionary mapping agent names to agent metadata
+    """
+    if not AOP_AVAILABLE or not isinstance(aop_instance, AOP):
+        return {}
+    
+    agents = {}
+    
+    try:
+        if hasattr(aop_instance, 'agents'):
+            for agent_name, agent in aop_instance.agents.items():
+                agents[agent_name] = {
+                    "name": agent_name,
+                    "type": type(agent).__name__,
+                    "description": getattr(agent, 'agent_description', 'No description'),
+                    "available": True,
+                }
+        
+        if hasattr(aop_instance, 'tool_configs'):
+            for tool_name, config in aop_instance.tool_configs.items():
+                if tool_name not in agents:
+                    agents[tool_name] = {
+                        "name": tool_name,
+                        "type": "tool",
+                        "description": getattr(config, 'tool_description', 'No description'),
+                        "available": True,
+                    }
+    except Exception as e:
+        logger.error(f"Error getting agents from AOP instance: {e}")
+    
+    return agents
+
+
+def get_agents_from_router(router_instance: Any) -> Dict[str, Any]:
+    """Get list of agents from a router instance.
+    
+    Args:
+        router_instance: Router instance to query
+        
+    Returns:
+        Dictionary mapping agent names to agent metadata
+    """
+    if not ROUTER_AVAILABLE or not isinstance(router_instance, SwarmRouter):
+        return {}
+    
+    agents = {}
+    
+    try:
+        if hasattr(router_instance, 'agents'):
+            for i, agent in enumerate(router_instance.agents):
+                agent_name = getattr(agent, 'agent_name', f"agent_{i}")
+                agents[agent_name] = {
+                    "name": agent_name,
+                    "type": type(agent).__name__,
+                    "description": getattr(agent, 'agent_description', 'No description'),
+                    "available": True,
+                }
+    except Exception as e:
+        logger.error(f"Error getting agents from router instance: {e}")
+    
+    return agents
+
+
+def discover_all_agents(
+    aop_instances: Optional[List[Any]] = None,
+    router_instances: Optional[List[Any]] = None,
+) -> Dict[str, Any]:
+    """Discover all available agents from AOP and router instances.
+    
+    Args:
+        aop_instances: Optional list of AOP instances (auto-discovered if None)
+        router_instances: Optional list of router instances (auto-discovered if None)
+        
+    Returns:
+        Dictionary mapping agent names to agent metadata
+    """
+    all_agents = {}
+    
+    # Discover AOP instances if not provided
+    if aop_instances is None:
+        aop_instances = discover_aop_instances()
+    
+    # Discover router instances if not provided
+    if router_instances is None:
+        router_instances = discover_router_instances()
+    
+    # Get agents from AOP instances
+    for aop_instance in aop_instances:
+        agents = get_agents_from_aop(aop_instance)
+        all_agents.update(agents)
+    
+    # Get agents from router instances
+    for router_instance in router_instances:
+        agents = get_agents_from_router(router_instance)
+        all_agents.update(agents)
+    
+    logger.info(f"Discovered {len(all_agents)} total agent(s)")
+    return all_agents
+
+
+def find_best_agent_for_task(
+    task: str,
+    available_agents: Dict[str, Any],
+    aop_instances: Optional[List[Any]] = None,
+    router_instances: Optional[List[Any]] = None,
+) -> Optional[tuple[str, Any, str]]:
+    """Find the best agent for a given task (route-first strategy).
+    
+    Args:
+        task: Task description
+        available_agents: Dictionary of available agents
+        aop_instances: Optional list of AOP instances
+        router_instances: Optional list of router instances
+        
+    Returns:
+        Tuple of (agent_name, agent_instance, source) or None if no suitable agent found
+        source is either 'aop' or 'router'
+    """
+    if not available_agents:
+        return None
+    
+    # Simple keyword-based matching (can be enhanced with LLM-based routing)
+    task_lower = task.lower()
+    
+    # Check for specialized agents first (route-first strategy)
+    for agent_name, agent_info in available_agents.items():
+        description = agent_info.get("description", "").lower()
+        agent_type = agent_info.get("type", "").lower()
+        
+        # Simple matching logic
+        if any(keyword in description or keyword in agent_type for keyword in task_lower.split()):
+            # Try to get the actual agent instance
+            if aop_instances:
+                for aop in aop_instances:
+                    if hasattr(aop, 'agents') and agent_name in aop.agents:
+                        return (agent_name, aop.agents[agent_name], "aop")
+            
+            if router_instances:
+                for router in router_instances:
+                    if hasattr(router, 'agents'):
+                        for agent in router.agents:
+                            if getattr(agent, 'agent_name', None) == agent_name:
+                                return (agent_name, agent, "router")
+    
+    # If no match found, return None (fallback to direct handling)
+    return None
+
+
+def route_to_agent(
+    agent_name: str,
+    task: str,
+    aop_instances: Optional[List[Any]] = None,
+    router_instances: Optional[List[Any]] = None,
+) -> Optional[Any]:
+    """Route a task to a specific agent.
+    
+    Args:
+        agent_name: Name of the agent to route to
+        task: Task to execute
+        aop_instances: Optional list of AOP instances
+        router_instances: Optional list of router instances
+        
+    Returns:
+        Agent response or None if agent not found
+    """
+    # Discover instances if not provided
+    if aop_instances is None:
+        aop_instances = discover_aop_instances()
+    
+    if router_instances is None:
+        router_instances = discover_router_instances()
+    
+    # Try AOP instances first
+    for aop in aop_instances:
+        if hasattr(aop, 'agents') and agent_name in aop.agents:
+            agent = aop.agents[agent_name]
+            if hasattr(agent, 'run'):
+                return agent.run(task)
+    
+    # Try router instances
+    for router in router_instances:
+        if hasattr(router, 'agents'):
+            for agent in router.agents:
+                if getattr(agent, 'agent_name', None) == agent_name:
+                    if hasattr(agent, 'run'):
+                        return agent.run(task)
+    
+    logger.warning(f"Agent '{agent_name}' not found for routing")
+    return None

utils/batch_processor.py

@@ -0,0 +1,317 @@
+"""
+Batch processing utilities.
+
+Provides functionality for:
+- Parallel task execution
+- Batch rate limiting
+- Progress tracking
+- Error aggregation
+"""
+
+import asyncio
+import concurrent.futures
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Tuple
+from loguru import logger
+
+try:
+    from utils.rate_limiter import RateLimiter, RateLimitConfig
+    RATE_LIMITER_AVAILABLE = True
+except ImportError:
+    RATE_LIMITER_AVAILABLE = False
+    logger.debug("Rate limiter not available for batch processing")
+
+
+@dataclass
+class BatchResult:
+    """Result of a batch processing operation.
+    
+    Attributes:
+        task_id: Task identifier
+        success: Whether task succeeded
+        result: Task result (if successful)
+        error: Error message (if failed)
+        execution_time: Time taken to execute task
+    """
+    task_id: str
+    success: bool
+    result: Any = None
+    error: Optional[str] = None
+    execution_time: float = 0.0
+
+
+@dataclass
+class BatchStats:
+    """Statistics for batch processing.
+    
+    Attributes:
+        total_tasks: Total number of tasks
+        completed_tasks: Number of completed tasks
+        failed_tasks: Number of failed tasks
+        total_time: Total execution time
+        average_time: Average execution time per task
+    """
+    total_tasks: int = 0
+    completed_tasks: int = 0
+    failed_tasks: int = 0
+    total_time: float = 0.0
+    average_time: float = 0.0
+
+
+class BatchProcessor:
+    """Batch processor for parallel task execution.
+    
+    Provides functionality for:
+    - Parallel task execution with configurable workers
+    - Batch rate limiting
+    - Progress tracking
+    - Error aggregation
+    """
+    
+    def __init__(
+        self,
+        max_workers: int = 4,
+        rate_limiter: Optional[Any] = None,
+        rate_limit_config: Optional[Any] = None,
+    ):
+        """Initialize batch processor.
+        
+        Args:
+            max_workers: Maximum number of parallel workers
+            rate_limiter: Optional rate limiter instance
+            rate_limit_config: Optional rate limit configuration
+        """
+        self.max_workers = max_workers
+        
+        # Set up rate limiting
+        if rate_limiter is not None:
+            self.rate_limiter = rate_limiter
+        elif RATE_LIMITER_AVAILABLE and rate_limit_config is not None:
+            self.rate_limiter = RateLimiter(rate_limit_config)
+        elif RATE_LIMITER_AVAILABLE:
+            # Default rate limiting for batch processing
+            config = RateLimitConfig(
+                requests_per_minute=100,
+                requests_per_hour=5000,
+            )
+            self.rate_limiter = RateLimiter(config)
+        else:
+            self.rate_limiter = None
+        
+        logger.debug(f"Initialized BatchProcessor with {max_workers} workers")
+    
+    def process_batch(
+        self,
+        tasks: List[Any],
+        task_fn: Callable,
+        task_ids: Optional[List[str]] = None,
+        user_id: str = "default",
+        show_progress: bool = True,
+    ) -> Tuple[List[BatchResult], BatchStats]:
+        """Process a batch of tasks in parallel.
+        
+        Args:
+            tasks: List of task inputs
+            task_fn: Function to execute for each task
+            task_ids: Optional list of task identifiers
+            user_id: User identifier for rate limiting
+            show_progress: Whether to show progress updates
+            
+        Returns:
+            Tuple of (results, stats)
+        """
+        if not tasks:
+            return [], BatchStats()
+        
+        start_time = time.time()
+        results: List[BatchResult] = []
+        
+        # Generate task IDs if not provided
+        if task_ids is None:
+            task_ids = [f"task_{i}" for i in range(len(tasks))]
+        
+        if len(task_ids) != len(tasks):
+            logger.warning("Task IDs length doesn't match tasks length, generating new IDs")
+            task_ids = [f"task_{i}" for i in range(len(tasks))]
+        
+        def process_single_task(task: Any, task_id: str) -> BatchResult:
+            """Process a single task with error handling."""
+            task_start = time.time()
+            
+            try:
+                # Apply rate limiting if available
+                if self.rate_limiter:
+                    is_allowed, error_msg = self.rate_limiter.check_rate_limit(user_id)
+                    if not is_allowed:
+                        # Wait if rate limited
+                        self.rate_limiter.wait_if_rate_limited(user_id, max_wait=60.0)
+                
+                # Execute task
+                result = task_fn(task)
+                execution_time = time.time() - task_start
+                
+                return BatchResult(
+                    task_id=task_id,
+                    success=True,
+                    result=result,
+                    execution_time=execution_time,
+                )
+            except Exception as e:
+                execution_time = time.time() - task_start
+                error_msg = str(e)
+                logger.error(f"Error processing task {task_id}: {error_msg}")
+                
+                return BatchResult(
+                    task_id=task_id,
+                    success=False,
+                    error=error_msg,
+                    execution_time=execution_time,
+                )
+        
+        # Process tasks in parallel
+        with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            # Submit all tasks
+            future_to_task = {
+                executor.submit(process_single_task, task, task_id): (task, task_id)
+                for task, task_id in zip(tasks, task_ids)
+            }
+            
+            # Collect results as they complete
+            completed = 0
+            for future in concurrent.futures.as_completed(future_to_task):
+                result = future.result()
+                results.append(result)
+                completed += 1
+                
+                if show_progress:
+                    logger.info(f"Batch progress: {completed}/{len(tasks)} tasks completed")
+        
+        # Calculate statistics
+        total_time = time.time() - start_time
+        completed_tasks = sum(1 for r in results if r.success)
+        failed_tasks = len(results) - completed_tasks
+        average_time = total_time / len(results) if results else 0.0
+        
+        stats = BatchStats(
+            total_tasks=len(tasks),
+            completed_tasks=completed_tasks,
+            failed_tasks=failed_tasks,
+            total_time=total_time,
+            average_time=average_time,
+        )
+        
+        logger.info(
+            f"Batch processing complete: {completed_tasks}/{len(tasks)} succeeded, "
+            f"{failed_tasks} failed, {total_time:.2f}s total"
+        )
+        
+        return results, stats
+    
+    async def process_batch_async(
+        self,
+        tasks: List[Any],
+        task_fn: Callable,
+        task_ids: Optional[List[str]] = None,
+        user_id: str = "default",
+        show_progress: bool = True,
+    ) -> Tuple[List[BatchResult], BatchStats]:
+        """Process a batch of tasks asynchronously.
+        
+        Args:
+            tasks: List of task inputs
+            task_fn: Async function to execute for each task
+            task_ids: Optional list of task identifiers
+            user_id: User identifier for rate limiting
+            show_progress: Whether to show progress updates
+            
+        Returns:
+            Tuple of (results, stats)
+        """
+        if not tasks:
+            return [], BatchStats()
+        
+        start_time = time.time()
+        results: List[BatchResult] = []
+        
+        # Generate task IDs if not provided
+        if task_ids is None:
+            task_ids = [f"task_{i}" for i in range(len(tasks))]
+        
+        if len(task_ids) != len(tasks):
+            logger.warning("Task IDs length doesn't match tasks length, generating new IDs")
+            task_ids = [f"task_{i}" for i in range(len(tasks))]
+        
+        async def process_single_task_async(task: Any, task_id: str) -> BatchResult:
+            """Process a single task asynchronously with error handling."""
+            task_start = time.time()
+            
+            try:
+                # Apply rate limiting if available
+                if self.rate_limiter:
+                    is_allowed, error_msg = self.rate_limiter.check_rate_limit(user_id)
+                    if not is_allowed:
+                        # Wait if rate limited (async sleep)
+                        await asyncio.sleep(1.0)
+                        # Try again
+                        is_allowed, error_msg = self.rate_limiter.check_rate_limit(user_id)
+                        if not is_allowed:
+                            await asyncio.sleep(5.0)
+                
+                # Execute task (assume it's async or can be awaited)
+                if asyncio.iscoroutinefunction(task_fn):
+                    result = await task_fn(task)
+                else:
+                    # Run sync function in executor
+                    try:
+                        loop = asyncio.get_running_loop()
+                    except RuntimeError:
+                        # No running loop, create new one
+                        loop = asyncio.new_event_loop()
+                        asyncio.set_event_loop(loop)
+                    result = await loop.run_in_executor(None, task_fn, task)
+                
+                execution_time = time.time() - task_start
+                
+                return BatchResult(
+                    task_id=task_id,
+                    success=True,
+                    result=result,
+                    execution_time=execution_time,
+                )
+            except Exception as e:
+                execution_time = time.time() - task_start
+                error_msg = str(e)
+                logger.error(f"Error processing task {task_id}: {error_msg}")
+                
+                return BatchResult(
+                    task_id=task_id,
+                    success=False,
+                    error=error_msg,
+                    execution_time=execution_time,
+                )
+        
+        # Process tasks concurrently
+        tasks_to_run = [process_single_task_async(task, task_id) for task, task_id in zip(tasks, task_ids)]
+        results = await asyncio.gather(*tasks_to_run)
+        
+        # Calculate statistics
+        total_time = time.time() - start_time
+        completed_tasks = sum(1 for r in results if r.success)
+        failed_tasks = len(results) - completed_tasks
+        average_time = total_time / len(results) if results else 0.0
+        
+        stats = BatchStats(
+            total_tasks=len(tasks),
+            completed_tasks=completed_tasks,
+            failed_tasks=failed_tasks,
+            total_time=total_time,
+            average_time=average_time,
+        )
+        
+        logger.info(
+            f"Async batch processing complete: {completed_tasks}/{len(tasks)} succeeded, "
+            f"{failed_tasks} failed, {total_time:.2f}s total"
+        )
+        
+        return results, stats

utils/conversation.py

@@ -1192,4 +1192,82 @@ class Conversation:
         except Exception as e:
             logger.error(f"Dynamic auto chunking failed: {e}")
             return self._return_history_as_string_worker()
+    
+    def get_conversation_stats(self) -> Dict[str, Any]:
+        """Get statistics about the conversation.
+        
+        Returns:
+            Dictionary with conversation statistics
+        """
+        stats = {
+            "total_messages": len(self.conversation_history),
+            "total_tokens": 0,
+            "messages_by_role": {},
+            "created_at": getattr(self, "created_at", None),
+            "last_updated": None,
+        }
+        
+        # Count messages by role
+        for message in self.conversation_history:
+            role = message.get("role", "unknown")
+            stats["messages_by_role"][role] = stats["messages_by_role"].get(role, 0) + 1
+            
+            # Sum token counts if available
+            if "token_count" in message:
+                stats["total_tokens"] += message["token_count"]
+            
+            # Get last updated timestamp
+            if "timestamp" in message:
+                stats["last_updated"] = message["timestamp"]
+        
+        return stats
+    
+    def get_enhanced_metadata(self) -> Dict[str, Any]:
+        """Get enhanced metadata about the conversation.
+        
+        Returns:
+            Dictionary with enhanced metadata
+        """
+        metadata = {
+            "id": self.id,
+            "name": self.name,
+            "created_at": getattr(self, "created_at", None),
+            "context_length": self.context_length,
+            "export_method": self.export_method,
+            "save_filepath": self.save_filepath,
+            "stats": self.get_conversation_stats(),
+        }
+        
+        return metadata
+    
+    def save_with_metadata(self, filepath: Optional[str] = None, force: bool = True) -> None:
+        """Save conversation with enhanced metadata.
+        
+        Args:
+            filepath: Optional filepath to save to (uses default if None)
+            force: If True, saves regardless of autosave setting
+        """
+        if filepath:
+            self.save_filepath = filepath
+        
+        # Prepare data with metadata
+        data = {
+            "metadata": self.get_enhanced_metadata(),
+            "history": self.conversation_history,
+        }
+        
+        try:
+            self._ensure_save_path()
+            
+            if self.export_method == "json":
+                with open(self.save_filepath, "w", encoding="utf-8") as f:
+                    json.dump(data, f, indent=4, default=str)
+            else:
+                with open(self.save_filepath, "w", encoding="utf-8") as f:
+                    yaml.dump(data, f, indent=4, default_flow_style=False, sort_keys=False)
+            
+            logger.info(f"Conversation with metadata saved to {self.save_filepath}")
+        except Exception as e:
+            logger.error(f"Failed to save conversation with metadata: {str(e)}\n{traceback.format_exc()}")
+            raise