camel-ai 0.2.75a6__py3-none-any.whl → 0.2.76__py3-none-any.whl

camel/societies/workforce/single_agent_worker.py

@@ -15,14 +15,19 @@ from __future__ import annotations
 
 import asyncio
 import datetime
+import glob
+import os
+import re
 import time
 from collections import deque
-from typing import Any, List, Optional
+from pathlib import Path
+from typing import Any, Dict, List, Optional
 
 from colorama import Fore
 
 from camel.agents import ChatAgent
 from camel.agents.chat_agent import AsyncStreamingChatAgentResponse
+from camel.logger import get_logger
 from camel.societies.workforce.prompts import PROCESS_TASK_PROMPT
 from camel.societies.workforce.structured_output_handler import (
     StructuredOutputHandler,
@@ -30,13 +35,16 @@ from camel.societies.workforce.structured_output_handler import (
 from camel.societies.workforce.utils import TaskResult
 from camel.societies.workforce.worker import Worker
 from camel.tasks.task import Task, TaskState, is_task_result_insufficient
+from camel.utils.context_utils import ContextUtility, WorkflowSummary
+
+logger = get_logger(__name__)
 
 
 class AgentPool:
     r"""A pool of agent instances for efficient reuse.
 
-    This pool manages a collection of pre-cloned agents. It supports
-    auto-scaling based ondemand and intelligent reuse of existing agents.
+    This pool manages a collection of pre-cloned agents with automatic
+    scaling and idle timeout cleanup.
 
     Args:
         base_agent (ChatAgent): The base agent to clone from.
@@ -48,6 +56,8 @@ class AgentPool:
             (default: :obj:`True`)
         idle_timeout (float): Time in seconds after which idle agents are
             removed. (default: :obj:`180.0`)
+        cleanup_interval (float): Fixed interval in seconds between cleanup
+            checks. (default: :obj:`60.0`)
     """
 
     def __init__(
@@ -56,12 +66,14 @@ class AgentPool:
         initial_size: int = 1,
         max_size: int = 10,
         auto_scale: bool = True,
-        idle_timeout: float = 180.0,  # 3 minutes
+        idle_timeout: float = 180.0,
+        cleanup_interval: float = 60.0,
     ):
         self.base_agent = base_agent
         self.max_size = max_size
         self.auto_scale = auto_scale
         self.idle_timeout = idle_timeout
+        self.cleanup_interval = cleanup_interval
 
         # Pool management
         self._available_agents: deque = deque()
@@ -73,6 +85,7 @@ class AgentPool:
         self._total_borrows = 0
         self._total_clones_created = 0
         self._pool_hits = 0
+        self._agents_cleaned = 0
 
         # Initialize pool
         self._initialize_pool(initial_size)
@@ -82,6 +95,7 @@ class AgentPool:
         for _ in range(min(size, self.max_size)):
             agent = self._create_fresh_agent()
             self._available_agents.append(agent)
+            self._agent_last_used[id(agent)] = time.time()
 
     def _create_fresh_agent(self) -> ChatAgent:
         r"""Create a fresh agent instance."""
@@ -94,53 +108,46 @@ class AgentPool:
         async with self._lock:
             self._total_borrows += 1
 
-            # Try to get from available agents first
             if self._available_agents:
                 agent = self._available_agents.popleft()
                 self._in_use_agents.add(id(agent))
                 self._pool_hits += 1
-
-                # Reset the agent state
-                agent.reset()
                 return agent
 
-            # Check if we can create new agents
-            total_agents = len(self._available_agents) + len(
-                self._in_use_agents
-            )
-            if total_agents < self.max_size:
+            # Check if we can create a new agent
+            if len(self._in_use_agents) < self.max_size or self.auto_scale:
                 agent = self._create_fresh_agent()
                 self._in_use_agents.add(id(agent))
                 return agent
 
-            # Pool exhausted, wait and retry or create temporary agent
-            if self.auto_scale:
-                # Create a temporary agent that won't be returned to pool
-                return self._create_fresh_agent()
-            else:
-                # Wait for an agent to become available
-                while not self._available_agents:
-                    await asyncio.sleep(0.1)
-
-                agent = self._available_agents.popleft()
-                self._in_use_agents.add(id(agent))
-                agent.reset()
-                return agent
+        # Wait for available agent
+        while True:
+            async with self._lock:
+                if self._available_agents:
+                    agent = self._available_agents.popleft()
+                    self._in_use_agents.add(id(agent))
+                    self._pool_hits += 1
+                    return agent
+            await asyncio.sleep(0.05)
 
     async def return_agent(self, agent: ChatAgent) -> None:
         r"""Return an agent to the pool."""
+        agent_id = id(agent)
+
         async with self._lock:
-            agent_id = id(agent)
+            if agent_id not in self._in_use_agents:
+                return
 
-            if agent_id in self._in_use_agents:
-                self._in_use_agents.remove(agent_id)
+            self._in_use_agents.discard(agent_id)
 
-                # Only return to pool if we're under max size
-                if len(self._available_agents) < self.max_size:
-                    # Reset agent state before returning to pool
-                    agent.reset()
-                    self._available_agents.append(agent)
-                    self._agent_last_used[agent_id] = time.time()
+            # Only add back to pool if under max size
+            if len(self._available_agents) < self.max_size:
+                agent.reset()
+                self._agent_last_used[agent_id] = time.time()
+                self._available_agents.append(agent)
+            else:
+                # Remove tracking for agents not returned to pool
+                self._agent_last_used.pop(agent_id, None)
 
     async def cleanup_idle_agents(self) -> None:
         r"""Remove idle agents from the pool to free memory."""
@@ -148,30 +155,35 @@ class AgentPool:
             return
 
         async with self._lock:
+            if not self._available_agents:
+                return
+
             current_time = time.time()
             agents_to_remove = []
 
             for agent in list(self._available_agents):
                 agent_id = id(agent)
                 last_used = self._agent_last_used.get(agent_id, current_time)
-
                 if current_time - last_used > self.idle_timeout:
                     agents_to_remove.append(agent)
 
             for agent in agents_to_remove:
                 self._available_agents.remove(agent)
-                agent_id = id(agent)
-                self._agent_last_used.pop(agent_id, None)
+                self._agent_last_used.pop(id(agent), None)
+                self._agents_cleaned += 1
 
     def get_stats(self) -> dict:
         r"""Get pool statistics."""
         return {
             "available_agents": len(self._available_agents),
             "in_use_agents": len(self._in_use_agents),
+            "pool_size": len(self._available_agents)
+            + len(self._in_use_agents),
             "total_borrows": self._total_borrows,
             "total_clones_created": self._total_clones_created,
             "pool_hits": self._pool_hits,
             "hit_rate": self._pool_hits / max(self._total_borrows, 1),
+            "agents_cleaned_up": self._agents_cleaned,
         }
 
 
@@ -197,6 +209,16 @@ class SingleAgentWorker(Worker):
             support native structured output. When disabled, the workforce
             uses the native response_format parameter.
             (default: :obj:`True`)
+        context_utility (ContextUtility, optional): Shared context utility
+            instance for workflow management. If provided, all workflow
+            operations will use this shared instance instead of creating
+            a new one. This ensures multiple workers share the same session
+            directory. (default: :obj:`None`)
+        enable_workflow_memory (bool, optional): Whether to enable workflow
+            memory accumulation during task execution. When enabled,
+            conversations from all task executions are accumulated for
+            potential workflow saving. Set to True if you plan to call
+            save_workflow_memories(). (default: :obj:`False`)
     """
 
     def __init__(
@@ -208,6 +230,8 @@ class SingleAgentWorker(Worker):
         pool_max_size: int = 10,
         auto_scale_pool: bool = True,
         use_structured_output_handler: bool = True,
+        context_utility: Optional[ContextUtility] = None,
+        enable_workflow_memory: bool = False,
     ) -> None:
         node_id = worker.agent_id
         super().__init__(
@@ -222,6 +246,18 @@ class SingleAgentWorker(Worker):
         )
         self.worker = worker
         self.use_agent_pool = use_agent_pool
+        self.enable_workflow_memory = enable_workflow_memory
+        self._shared_context_utility = context_utility
+        self._context_utility: Optional[ContextUtility] = (
+            None  # Will be initialized when needed
+        )
+
+        # accumulator agent for collecting conversations
+        # from all task processing
+        self._conversation_accumulator: Optional[ChatAgent] = None
+
+        # note: context utility is set on the worker agent during save/load
+        # operations to avoid creating session folders during initialization
 
         self.agent_pool: Optional[AgentPool] = None
         self._cleanup_task: Optional[asyncio.Task] = None
@@ -264,6 +300,24 @@ class SingleAgentWorker(Worker):
             await self.agent_pool.return_agent(agent)
         # If not using pool, agent will be garbage collected
 
+    def _get_context_utility(self) -> ContextUtility:
+        r"""Get context utility with lazy initialization."""
+        if self._context_utility is None:
+            self._context_utility = (
+                self._shared_context_utility
+                or ContextUtility.get_workforce_shared()
+            )
+        return self._context_utility
+
+    def _get_conversation_accumulator(self) -> ChatAgent:
+        r"""Get or create the conversation accumulator agent."""
+        if self._conversation_accumulator is None:
+            # create a clone of the original worker to serve as accumulator
+            self._conversation_accumulator = self.worker.clone(
+                with_memory=False
+            )
+        return self._conversation_accumulator
+
     async def _process_task(
         self, task: Task, dependencies: List[Task]
     ) -> TaskState:
@@ -381,10 +435,36 @@ class SingleAgentWorker(Worker):
                 usage_info.get("total_tokens", 0) if usage_info else 0
             )
 
+            # collect conversation from working agent to
+            # accumulator for workflow memory
+            # Only transfer memory if workflow memory is enabled
+            if self.enable_workflow_memory:
+                accumulator = self._get_conversation_accumulator()
+
+                # transfer all memory records from working agent to accumulator
+                try:
+                    # retrieve all context records from the working agent
+                    work_records = worker_agent.memory.retrieve()
+
+                    # write these records to the accumulator's memory
+                    memory_records = [
+                        record.memory_record for record in work_records
+                    ]
+                    accumulator.memory.write_records(memory_records)
+
+                    logger.debug(
+                        f"Transferred {len(memory_records)} memory records to "
+                        f"accumulator"
+                    )
+
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to transfer conversation to accumulator: {e}"
+                    )
+
         except Exception as e:
-            print(
-                f"{Fore.RED}Error processing task {task.id}: "
-                f"{type(e).__name__}: {e}{Fore.RESET}"
+            logger.error(
+                f"Error processing task {task.id}: {type(e).__name__}: {e}"
             )
             # Store error information in task result
             task.result = f"{type(e).__name__}: {e!s}"
@@ -429,13 +509,13 @@ class SingleAgentWorker(Worker):
         task.additional_info["token_usage"] = {"total_tokens": total_tokens}
 
         print(f"======\n{Fore.GREEN}Response from {self}:{Fore.RESET}")
+        logger.info(f"Response from {self}:")
 
         if not self.use_structured_output_handler:
             # Handle native structured output parsing
             if task_result is None:
-                print(
-                    f"{Fore.RED}Error in worker step execution: Invalid "
-                    f"task result{Fore.RESET}"
+                logger.error(
+                    "Error in worker step execution: Invalid task result"
                 )
                 task_result = TaskResult(
                     content="Failed to generate valid task result.",
@@ -446,6 +526,10 @@ class SingleAgentWorker(Worker):
         print(
             f"\n{color}{task_result.content}{Fore.RESET}\n======",  # type: ignore[union-attr]
         )
+        if task_result.failed:  # type: ignore[union-attr]
+            logger.error(f"{task_result.content}")  # type: ignore[union-attr]
+        else:
+            logger.info(f"{task_result.content}")  # type: ignore[union-attr]
 
         task.result = task_result.content  # type: ignore[union-attr]
 
@@ -453,9 +537,9 @@ class SingleAgentWorker(Worker):
             return TaskState.FAILED
 
         if is_task_result_insufficient(task):
-            print(
-                f"{Fore.RED}Task {task.id}: Content validation failed - "
-                f"task marked as failed{Fore.RESET}"
+            logger.warning(
+                f"Task {task.id}: Content validation failed - "
+                f"task marked as failed"
             )
             return TaskState.FAILED
         return TaskState.DONE
@@ -477,16 +561,317 @@ class SingleAgentWorker(Worker):
         r"""Periodically clean up idle agents from the pool."""
         while True:
             try:
-                await asyncio.sleep(60)  # Cleanup every minute
+                # Fixed interval cleanup
+                await asyncio.sleep(self.agent_pool.cleanup_interval)
+
                 if self.agent_pool:
                     await self.agent_pool.cleanup_idle_agents()
             except asyncio.CancelledError:
                 break
             except Exception as e:
-                print(f"Error in pool cleanup: {e}")
+                logger.warning(f"Error in pool cleanup: {e}")
 
     def get_pool_stats(self) -> Optional[dict]:
         r"""Get agent pool statistics if pool is enabled."""
         if self.use_agent_pool and self.agent_pool:
             return self.agent_pool.get_stats()
         return None
+
+    def save_workflow_memories(self) -> Dict[str, Any]:
+        r"""Save the worker's current workflow memories using agent
+        summarization.
+
+        This method generates a workflow summary from the worker agent's
+        conversation history and saves it to a markdown file. The filename
+        is based on the worker's description for easy loading later.
+
+        Returns:
+            Dict[str, Any]: Result dictionary with keys:
+                - status (str): "success" or "error"
+                - summary (str): Generated workflow summary
+                - file_path (str): Path to saved file
+                - worker_description (str): Worker description used
+        """
+        try:
+            # validate requirements
+            validation_error = self._validate_workflow_save_requirements()
+            if validation_error:
+                return validation_error
+
+            # setup context utility and agent
+            context_util = self._get_context_utility()
+            self.worker.set_context_utility(context_util)
+
+            # prepare workflow summarization components
+            filename = self._generate_workflow_filename()
+            structured_prompt = self._prepare_workflow_prompt()
+            agent_to_summarize = self._select_agent_for_summarization(
+                context_util
+            )
+
+            # generate and save workflow summary
+            result = agent_to_summarize.summarize(
+                filename=filename,
+                summary_prompt=structured_prompt,
+                response_format=WorkflowSummary,
+            )
+
+            # add worker metadata and cleanup
+            result["worker_description"] = self.description
+            if self._conversation_accumulator is not None:
+                logger.info(
+                    "Cleaning up conversation accumulator after workflow "
+                    "summarization"
+                )
+                self._conversation_accumulator = None
+
+            return result
+
+        except Exception as e:
+            return {
+                "status": "error",
+                "summary": "",
+                "file_path": None,
+                "worker_description": self.description,
+                "message": f"Failed to save workflow memories: {e!s}",
+            }
+
+    def load_workflow_memories(
+        self,
+        pattern: Optional[str] = None,
+        max_files_to_load: int = 3,
+        session_id: Optional[str] = None,
+    ) -> bool:
+        r"""Load workflow memories matching worker description
+        from saved files.
+
+        This method searches for workflow memory files that match the worker's
+        description and loads them into the agent's memory using
+        ContextUtility.
+
+        Args:
+            pattern (Optional[str]): Custom search pattern for workflow
+                memory files.
+                If None, uses worker description to generate pattern.
+            max_files_to_load (int): Maximum number of workflow files to load.
+                (default: :obj:`3`)
+            session_id (Optional[str]): Specific workforce session ID to load
+                from. If None, searches across all sessions.
+                (default: :obj:`None`)
+
+        Returns:
+            bool: True if workflow memories were successfully loaded, False
+                otherwise.
+        """
+        try:
+            # reset system message to original state before loading
+            # this prevents duplicate workflow context on multiple calls
+            if isinstance(self.worker, ChatAgent):
+                self.worker.reset_to_original_system_message()
+
+            # Find workflow memory files matching the pattern
+            workflow_files = self._find_workflow_files(pattern, session_id)
+            if not workflow_files:
+                return False
+
+            # Load the workflow memory files
+            loaded_count = self._load_workflow_files(
+                workflow_files, max_files_to_load
+            )
+
+            # Report results
+            logger.info(
+                f"Successfully loaded {loaded_count} workflow file(s) for "
+                f"{self.description}"
+            )
+            return loaded_count > 0
+
+        except Exception as e:
+            logger.warning(
+                f"Error loading workflow memories for {self.description}: "
+                f"{e!s}"
+            )
+            return False
+
+    def _find_workflow_files(
+        self, pattern: Optional[str], session_id: Optional[str] = None
+    ) -> List[str]:
+        r"""Find and return sorted workflow files matching the pattern.
+
+        Args:
+            pattern (Optional[str]): Custom search pattern for workflow files.
+                If None, uses worker description to generate pattern.
+            session_id (Optional[str]): Specific session ID to search in.
+                If None, searches across all sessions.
+
+        Returns:
+            List[str]: Sorted list of workflow file paths (empty if
+                validation fails).
+        """
+        # Ensure we have a ChatAgent worker
+        if not isinstance(self.worker, ChatAgent):
+            logger.warning(
+                f"Cannot load workflow: {self.description} worker is not "
+                "a ChatAgent"
+            )
+            return []
+
+        # generate filename-safe search pattern from worker description
+        if pattern is None:
+            # sanitize description: spaces to underscores, remove special chars
+            clean_desc = self.description.lower().replace(" ", "_")
+            clean_desc = re.sub(r'[^a-z0-9_]', '', clean_desc)
+            pattern = f"{clean_desc}_workflow*.md"
+
+        # Get the base workforce_workflows directory
+        camel_workdir = os.environ.get("CAMEL_WORKDIR")
+        if camel_workdir:
+            base_dir = os.path.join(camel_workdir, "workforce_workflows")
+        else:
+            base_dir = "workforce_workflows"
+
+        # search for workflow files in specified or all session directories
+        if session_id:
+            search_path = str(Path(base_dir) / session_id / pattern)
+        else:
+            # search across all session directories using wildcard pattern
+            search_path = str(Path(base_dir) / "*" / pattern)
+        workflow_files = glob.glob(search_path)
+
+        if not workflow_files:
+            logger.info(f"No workflow files found for pattern: {pattern}")
+            return []
+
+        # prioritize most recent sessions by session timestamp in
+        # directory name
+        def extract_session_timestamp(filepath: str) -> str:
+            match = re.search(r'session_(\d{8}_\d{6}_\d{6})', filepath)
+            return match.group(1) if match else ""
+
+        workflow_files.sort(key=extract_session_timestamp, reverse=True)
+        return workflow_files
+
+    def _load_workflow_files(
+        self, workflow_files: List[str], max_files_to_load: int
+    ) -> int:
+        r"""Load workflow files and return count of successful loads.
+
+        Args:
+            workflow_files (List[str]): List of workflow file paths to load.
+
+        Returns:
+            int: Number of successfully loaded workflow files.
+        """
+        loaded_count = 0
+        # limit loading to prevent context overflow
+        for file_path in workflow_files[:max_files_to_load]:
+            try:
+                # extract file and session info from full path
+                filename = os.path.basename(file_path).replace('.md', '')
+                session_dir = os.path.dirname(file_path)
+                session_id = os.path.basename(session_dir)
+
+                # create context utility for the specific session
+                # where file exists
+                temp_utility = ContextUtility.get_workforce_shared(session_id)
+
+                status = temp_utility.load_markdown_context_to_memory(
+                    self.worker, filename
+                )
+
+                if "Context appended" in status:
+                    loaded_count += 1
+                    logger.info(f"Loaded workflow: {filename}")
+                else:
+                    logger.warning(
+                        f"Failed to load workflow {filename}: {status}"
+                    )
+
+            except Exception as e:
+                logger.warning(
+                    f"Failed to load workflow file {file_path}: {e!s}"
+                )
+                continue
+
+        return loaded_count
+
+    def _validate_workflow_save_requirements(self) -> Optional[Dict[str, Any]]:
+        r"""Validate requirements for workflow saving.
+
+        Returns:
+            Optional[Dict[str, Any]]: Error result dict if validation fails,
+                None if validation passes.
+        """
+        if not isinstance(self.worker, ChatAgent):
+            return {
+                "status": "error",
+                "summary": "",
+                "file_path": None,
+                "worker_description": self.description,
+                "message": (
+                    "Worker must be a ChatAgent instance to save workflow "
+                    "memories"
+                ),
+            }
+        return None
+
+    def _generate_workflow_filename(self) -> str:
+        r"""Generate a filename for the workflow based on worker description.
+
+        Returns:
+            str: Sanitized filename without timestamp (session already has
+                timestamp).
+        """
+        clean_desc = self.description.lower().replace(" ", "_")
+        clean_desc = re.sub(r'[^a-z0-9_]', '', clean_desc)
+        return f"{clean_desc}_workflow"
+
+    def _prepare_workflow_prompt(self) -> str:
+        r"""Prepare the structured prompt for workflow summarization.
+
+        Returns:
+            str: Structured prompt for workflow summary.
+        """
+        workflow_prompt = WorkflowSummary.get_instruction_prompt()
+        return StructuredOutputHandler.generate_structured_prompt(
+            base_prompt=workflow_prompt, schema=WorkflowSummary
+        )
+
+    def _select_agent_for_summarization(
+        self, context_util: ContextUtility
+    ) -> ChatAgent:
+        r"""Select the best agent for workflow summarization.
+
+        Args:
+            context_util: Context utility to set on selected agent.
+
+        Returns:
+            ChatAgent: Agent to use for summarization.
+        """
+        agent_to_summarize = self.worker
+
+        if self._conversation_accumulator is not None:
+            accumulator_messages, _ = (
+                self._conversation_accumulator.memory.get_context()
+            )
+            if accumulator_messages:
+                self._conversation_accumulator.set_context_utility(
+                    context_util
+                )
+                agent_to_summarize = self._conversation_accumulator
+                logger.info(
+                    f"Using conversation accumulator with "
+                    f"{len(accumulator_messages)} messages for workflow "
+                    f"summary"
+                )
+            else:
+                logger.info(
+                    "Using original worker for workflow summary (no "
+                    "accumulated conversations)"
+                )
+        else:
+            logger.info(
+                "Using original worker for workflow summary (no accumulator)"
+            )
+
+        return agent_to_summarize