camel-ai 0.2.78__py3-none-any.whl → 0.2.79a1__py3-none-any.whl

camel/toolkits/message_integration.py

@@ -148,12 +148,10 @@ class ToolkitMessageIntegration:
         """
         return FunctionTool(self.send_message_to_user)
 
-    def register_toolkits(
-        self, toolkit: BaseToolkit, tool_names: Optional[List[str]] = None
-    ) -> BaseToolkit:
-        r"""Add messaging capabilities to toolkit methods.
+    def register_toolkits(self, toolkit: BaseToolkit) -> BaseToolkit:
+        r"""Add messaging capabilities to all toolkit methods.
 
-        This method modifies a toolkit so that specified tools can send
+        This method modifies a toolkit so that all its tools can send
         status messages to users while executing their primary function.
         The tools will accept optional messaging parameters:
         - message_title: Title of the status message
@@ -162,20 +160,18 @@ class ToolkitMessageIntegration:
 
         Args:
             toolkit: The toolkit to add messaging capabilities to
-            tool_names: List of specific tool names to modify.
-                       If None, messaging is added to all tools.
 
         Returns:
-            The toolkit with messaging capabilities added
+            The same toolkit instance with messaging capabilities added to
+                all methods.
         """
         original_tools = toolkit.get_tools()
         enhanced_methods = {}
         for tool in original_tools:
             method_name = tool.func.__name__
-            if tool_names is None or method_name in tool_names:
-                enhanced_func = self._add_messaging_to_tool(tool.func)
-                enhanced_methods[method_name] = enhanced_func
-                setattr(toolkit, method_name, enhanced_func)
+            enhanced_func = self._add_messaging_to_tool(tool.func)
+            enhanced_methods[method_name] = enhanced_func
+            setattr(toolkit, method_name, enhanced_func)
         original_get_tools_method = toolkit.get_tools
 
         def enhanced_get_tools() -> List[FunctionTool]:
@@ -201,7 +197,7 @@ class ToolkitMessageIntegration:
             def enhanced_clone_for_new_session(new_session_id=None):
                 cloned_toolkit = original_clone_method(new_session_id)
                 return message_integration_instance.register_toolkits(
-                    cloned_toolkit, tool_names
+                    cloned_toolkit
                 )
 
             toolkit.clone_for_new_session = enhanced_clone_for_new_session
@@ -300,6 +296,12 @@ class ToolkitMessageIntegration:
         This internal method modifies the function signature and docstring
         to include optional messaging parameters that trigger status updates.
         """
+        if getattr(func, "__message_integration_enhanced__", False):
+            logger.debug(
+                f"Function {func.__name__} already enhanced, skipping"
+            )
+            return func
+
         # Get the original signature
         original_sig = inspect.signature(func)
 

camel/toolkits/terminal_toolkit/terminal_toolkit.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-import atexit
 import os
 import platform
 import select
@@ -42,12 +41,10 @@ logger = get_logger(__name__)
 try:
     import docker
     from docker.errors import APIError, NotFound
-    from docker.models.containers import Container
 except ImportError:
     docker = None
     NotFound = None
     APIError = None
-    Container = None
 
 
 def _to_plain(text: str) -> str:
@@ -149,8 +146,6 @@ class TerminalToolkit(BaseToolkit):
         self.initial_env_path: Optional[str] = None
         self.python_executable = sys.executable
 
-        atexit.register(self.__del__)
-
         self.log_dir = os.path.abspath(
             session_logs_dir or os.path.join(self.working_dir, "terminal_logs")
         )
@@ -400,9 +395,11 @@ class TerminalToolkit(BaseToolkit):
                     with self._session_lock:
                         if session_id in self.shell_sessions:
                             self.shell_sessions[session_id]["error"] = str(e)
-                except Exception:
-                    # Swallow any secondary errors during cleanup
-                    pass
+                except Exception as cleanup_error:
+                    logger.warning(
+                        f"[SESSION {session_id}] Failed to store error state: "
+                        f"{cleanup_error}"
+                    )
             finally:
                 try:
                     with self._session_lock:
@@ -480,39 +477,40 @@ class TerminalToolkit(BaseToolkit):
 
         warning_message = (
             "\n--- WARNING: Process is still actively outputting "
-            "after max wait time. Consider using shell_wait() "
-            "before sending the next command. ---"
+            "after max wait time. Consider waiting before "
+            "sending the next command. ---"
         )
         return "".join(output_parts) + warning_message
 
     def shell_exec(self, id: str, command: str, block: bool = True) -> str:
-        r"""This function executes a shell command. The command can run in
-        blocking mode (waits for completion) or non-blocking mode
-        (runs in the background). A unique session ID is created for
-        each session.
+        r"""Executes a shell command in blocking or non-blocking mode.
 
         Args:
-            command (str): The command to execute.
-            block (bool): If True, the command runs synchronously,
-                waiting for it to complete or time out, and returns
-                its full output. If False, the command runs
-                asynchronously in the background.
-            id (Optional[str]): A specific ID for the session. If not provided,
-                a unique ID is generated for non-blocking sessions.
+            id (str): A unique identifier for the command's session. This ID is
+                used to interact with non-blocking processes.
+            command (str): The shell command to execute.
+            block (bool, optional): Determines the execution mode. Defaults to
+                True. If `True` (blocking mode), the function waits for the
+                command to complete and returns the full output. Use this for
+                most commands . If `False` (non-blocking mode), the function
+                starts the command in the background. Use this only for
+                interactive sessions or long-running tasks, or servers.
 
         Returns:
-            str: If block is True, returns the complete stdout and stderr.
-                 If block is False, returns a message containing the new
-                 session ID and the initial output from the command after
-                 it goes idle.
+            str: The output of the command execution, which varies by mode.
+                In blocking mode, returns the complete standard output and
+                standard error from the command.
+                In non-blocking mode, returns a confirmation message with the
+                session `id`. To interact with the background process, use
+                other functions: `shell_view(id)` to see output,
+                `shell_write_to_process(id, "input")` to send input, and
+                `shell_kill_process(id)` to terminate.
         """
         if self.safe_mode:
             is_safe, message = self._sanitize_command(command)
             if not is_safe:
                 return f"Error: {message}"
             command = message
-        else:
-            command = command
 
         if self.use_docker_backend:
             # For Docker, we always run commands in a shell
@@ -570,7 +568,10 @@ class TerminalToolkit(BaseToolkit):
                 log_entry += f"--- Error ---\n{error_msg}\n"
                 return error_msg
             except Exception as e:
-                if "Read timed out" in str(e):
+                if (
+                    isinstance(e, (subprocess.TimeoutExpired, TimeoutError))
+                    or "timed out" in str(e).lower()
+                ):
                     error_msg = (
                         f"Error: Command timed out after "
                         f"{self.timeout} seconds."
@@ -593,6 +594,14 @@ class TerminalToolkit(BaseToolkit):
                 f"> {command}\n",
             )
 
+            # PYTHONUNBUFFERED=1 for real-time output
+            # Without this, Python subprocesses buffer output (4KB buffer)
+            # and shell_view() won't see output until buffer fills or process
+            # exits
+            env_vars = os.environ.copy()
+            env_vars["PYTHONUNBUFFERED"] = "1"
+            docker_env = {"PYTHONUNBUFFERED": "1"}
+
             with self._session_lock:
                 self.shell_sessions[session_id] = {
                     "id": session_id,
@@ -606,6 +615,8 @@ class TerminalToolkit(BaseToolkit):
                     else "local",
                 }
 
+            process = None
+            exec_socket = None
             try:
                 if not self.use_docker_backend:
                     process = subprocess.Popen(
@@ -617,6 +628,7 @@ class TerminalToolkit(BaseToolkit):
                         text=True,
                         cwd=self.working_dir,
                         encoding="utf-8",
+                        env=env_vars,
                     )
                     with self._session_lock:
                         self.shell_sessions[session_id]["process"] = process
@@ -630,6 +642,7 @@ class TerminalToolkit(BaseToolkit):
                         stdin=True,
                         tty=True,
                         workdir=self.docker_workdir,
+                        environment=docker_env,
                     )
                     exec_id = exec_instance['Id']
                     exec_socket = self.docker_api_client.exec_start(
@@ -643,15 +656,29 @@ class TerminalToolkit(BaseToolkit):
 
                 self._start_output_reader_thread(session_id)
 
-                # time.sleep(0.1)
-                initial_output = self._collect_output_until_idle(session_id)
-
+                # Return immediately with session ID and instructions
                 return (
-                    f"Session started with ID: {session_id}\n\n"
-                    f"[Initial Output]:\n{initial_output}"
+                    f"Session '{session_id}' started.\n\n"
+                    f"You could use:\n"
+                    f"  - shell_view('{session_id}') - get output\n"
+                    f"  - shell_write_to_process('{session_id}', '<input>')"
+                    f" - send input\n"
+                    f"  - shell_kill_process('{session_id}') - terminate"
                 )
 
             except Exception as e:
+                # Clean up resources on failure
+                if process is not None:
+                    try:
+                        process.terminate()
+                    except Exception:
+                        pass
+                if exec_socket is not None:
+                    try:
+                        exec_socket.close()
+                    except Exception:
+                        pass
+
                 with self._session_lock:
                     if session_id in self.shell_sessions:
                         self.shell_sessions[session_id]["running"] = False
@@ -714,18 +741,16 @@ class TerminalToolkit(BaseToolkit):
             return f"Error writing to session '{id}': {e}"
 
     def shell_view(self, id: str) -> str:
-        r"""This function retrieves any new output from a non-blocking session
-        since the last time this function was called. If the process has
-        terminated, it drains the output queue and appends a termination
-        message. If the process is still running, it simply returns any
-        new output.
+        r"""Retrieves new output from a non-blocking session.
+
+        This function returns only NEW output since the last call. It does NOT
+        wait or block - it returns immediately with whatever is available.
 
         Args:
             id (str): The unique session ID of the non-blocking process.
 
         Returns:
-            str: The new output from the process's stdout and stderr. Returns
-                 an empty string if there is no new output.
+            str: New output if available, or a status message.
         """
         with self._session_lock:
             if id not in self.shell_sessions:
@@ -734,7 +759,6 @@ class TerminalToolkit(BaseToolkit):
             is_running = session["running"]
 
         # If session is terminated, drain the queue and return
-        # with a status message.
         if not is_running:
             final_output = []
             try:
@@ -742,9 +766,13 @@ class TerminalToolkit(BaseToolkit):
                     final_output.append(session["output_stream"].get_nowait())
             except Empty:
                 pass
-            return "".join(final_output) + "\n--- SESSION TERMINATED ---"
 
-        # Otherwise, just drain the queue for a live session.
+            if final_output:
+                return "".join(final_output) + "\n\n--- SESSION TERMINATED ---"
+            else:
+                return "--- SESSION TERMINATED (no new output) ---"
+
+        # For running session, check for new output
         output = []
         try:
             while True:
@@ -752,38 +780,18 @@ class TerminalToolkit(BaseToolkit):
         except Empty:
             pass
 
-        return "".join(output)
-
-    def shell_wait(self, id: str, wait_seconds: float = 5.0) -> str:
-        r"""This function waits for a specified duration for a
-        non-blocking process to produce more output or terminate.
-
-        Args:
-            id (str): The unique session ID of the non-blocking process.
-            wait_seconds (float): The maximum number of seconds to wait.
-
-        Returns:
-            str: All output collected during the wait period.
-        """
-        with self._session_lock:
-            if id not in self.shell_sessions:
-                return f"Error: No session found with ID '{id}'."
-            session = self.shell_sessions[id]
-            if not session["running"]:
-                return (
-                    "Session is no longer running. "
-                    "Use shell_view to get final output."
-                )
-
-        output_collected = []
-        end_time = time.time() + wait_seconds
-        while time.time() < end_time and session["running"]:
-            new_output = self.shell_view(id)
-            if new_output:
-                output_collected.append(new_output)
-            time.sleep(0.2)
-
-        return "".join(output_collected)
+        if output:
+            return "".join(output)
+        else:
+            # No new output - guide the agent
+            return (
+                "[No new output]\n"
+                "Session is running but idle. Actions could take:\n"
+                "  - For interactive sessions: Send input "
+                "with shell_write_to_process()\n"
+                "  - For long tasks: Check again later (don't poll "
+                "too frequently)"
+            )
 
     def shell_kill_process(self, id: str) -> str:
         r"""This function forcibly terminates a running non-blocking process.
@@ -894,8 +902,17 @@ class TerminalToolkit(BaseToolkit):
             except EOFError:
                 return f"User input interrupted for session '{id}'."
 
-    def __del__(self):
-        # Clean up any sessions
+    def __enter__(self):
+        r"""Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        r"""Context manager exit - clean up all sessions."""
+        self.cleanup()
+        return False
+
+    def cleanup(self):
+        r"""Clean up all active sessions."""
         with self._session_lock:
             session_ids = list(self.shell_sessions.keys())
         for session_id in session_ids:
@@ -904,7 +921,24 @@ class TerminalToolkit(BaseToolkit):
                     "running", False
                 )
             if is_running:
-                self.shell_kill_process(session_id)
+                try:
+                    self.shell_kill_process(session_id)
+                except Exception as e:
+                    logger.warning(
+                        f"Failed to kill session '{session_id}' "
+                        f"during cleanup: {e}"
+                    )
+
+    cleanup._manual_timeout = True  # type: ignore[attr-defined]
+
+    def __del__(self):
+        r"""Fallback cleanup in destructor."""
+        try:
+            self.cleanup()
+        except Exception:
+            pass
+
+    __del__._manual_timeout = True  # type: ignore[attr-defined]
 
     def get_tools(self) -> List[FunctionTool]:
         r"""Returns a list of FunctionTool objects representing the functions
@@ -917,7 +951,6 @@ class TerminalToolkit(BaseToolkit):
         return [
             FunctionTool(self.shell_exec),
             FunctionTool(self.shell_view),
-            FunctionTool(self.shell_wait),
             FunctionTool(self.shell_write_to_process),
             FunctionTool(self.shell_kill_process),
             FunctionTool(self.shell_ask_user_for_help),

camel/types/enums.py

@@ -24,6 +24,7 @@ logger = get_logger(__name__)
 class RoleType(Enum):
     ASSISTANT = "assistant"
     USER = "user"
+    SYSTEM = "system"
     CRITIC = "critic"
     EMBODIMENT = "embodiment"
     DEFAULT = "default"

camel/utils/context_utils.py

@@ -13,6 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
+import re
 from datetime import datetime
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional
@@ -90,6 +91,17 @@ class WorkflowSummary(BaseModel):
         "mid-task by using the HumanToolkit.",
         default="",
     )
+    tags: List[str] = Field(
+        description="3-10 categorization tags that describe the workflow "
+        "type, domain, and key capabilities. Use lowercase with hyphens. "
+        "Tags should be broad, reusable categories to help with semantic "
+        "matching to similar tasks. "
+        "Examples: 'data-analysis', 'web-scraping', 'api-integration', "
+        "'code-generation', 'file-processing', 'database-query', "
+        "'text-processing', 'image-manipulation', 'email-automation', "
+        "'report-generation'.",
+        default_factory=list,
+    )
 
     @classmethod
     def get_instruction_prompt(cls) -> str:
@@ -111,7 +123,12 @@ class WorkflowSummary(BaseModel):
             'about a simple math problem, the workflow must be short, '
             'e.g. <60 words. By contrast, if the task is complex and '
             'multi-step, such as finding particular job applications based '
-            'on user CV, the workflow must be longer, e.g. about 120 words.'
+            'on user CV, the workflow must be longer, e.g. about 120 words. '
+            'For tags, provide 3-5 broad categorization tags using lowercase '
+            'with hyphens (e.g., "data-analysis", "web-scraping") that '
+            'describe the workflow domain, type, and key capabilities to '
+            'help future agents discover this workflow when working on '
+            'similar tasks.'
         )
 
 
@@ -131,6 +148,10 @@ class ContextUtility:
     - Shared session management for workforce workflows
     """
 
+    # maximum filename length for workflow files (chosen for filesystem
+    # compatibility and readability)
+    MAX_WORKFLOW_FILENAME_LENGTH: ClassVar[int] = 50
+
     # Class variables for shared session management
     _shared_sessions: ClassVar[Dict[str, 'ContextUtility']] = {}
     _default_workforce_session: ClassVar[Optional['ContextUtility']] = None
@@ -191,6 +212,54 @@ class ContextUtility:
         timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')
         return f"session_{timestamp}"
 
+    @staticmethod
+    def sanitize_workflow_filename(
+        name: str,
+        max_length: Optional[int] = None,
+    ) -> str:
+        r"""Sanitize a name string for use as a workflow filename.
+
+        Converts the input string to a safe filename by:
+        - converting to lowercase
+        - replacing spaces with underscores
+        - removing special characters (keeping only alphanumeric and
+          underscores)
+        - truncating to maximum length if specified
+
+        Args:
+            name (str): The name string to sanitize (e.g., role_name or
+                task_title).
+            max_length (Optional[int]): Maximum length for the sanitized
+                filename. If None, uses MAX_WORKFLOW_FILENAME_LENGTH.
+                (default: :obj:`None`)
+
+        Returns:
+            str: Sanitized filename string suitable for filesystem use.
+                Returns "agent" if sanitization results in empty string.
+
+        Example:
+            >>> ContextUtility.sanitize_workflow_filename("Data Analyst!")
+            'data_analyst'
+            >>> ContextUtility.sanitize_workflow_filename("Test@123", 5)
+            'test1'
+        """
+        if max_length is None:
+            max_length = ContextUtility.MAX_WORKFLOW_FILENAME_LENGTH
+
+        # sanitize: lowercase, spaces to underscores, remove special chars
+        clean_name = name.lower().replace(" ", "_")
+        clean_name = re.sub(r'[^a-z0-9_]', '', clean_name)
+
+        # truncate if too long
+        if len(clean_name) > max_length:
+            clean_name = clean_name[:max_length]
+
+        # ensure it's not empty after sanitization
+        if not clean_name:
+            clean_name = "agent"
+
+        return clean_name
+
     # ========= GENERIC FILE MANAGEMENT METHODS =========
 
     def _ensure_directory_exists(self) -> None:
@@ -480,7 +549,6 @@ class ContextUtility:
             'session_id': self.session_id,
             'working_directory': str(self.working_directory),
             'created_at': datetime.now().isoformat(),
-            'base_directory': str(self.working_directory.parent),
         }
 
     def list_sessions(self, base_dir: Optional[str] = None) -> List[str]:
@@ -741,6 +809,137 @@ class ContextUtility:
         result = '\n'.join(filtered_lines).strip()
         return result
 
+    # ========= WORKFLOW INFO METHODS =========
+
+    def extract_workflow_info(self, file_path: str) -> Dict[str, Any]:
+        r"""Extract info from a workflow markdown file.
+
+        This method reads only the essential info from a workflow file
+        (title, description, tags) for use in workflow selection without
+        loading the entire workflow content.
+
+        Args:
+            file_path (str): Full path to the workflow markdown file.
+
+        Returns:
+            Dict[str, Any]: Workflow info including title, description,
+                tags, and file_path. Returns empty dict on error.
+        """
+        import re
+
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                content = f.read()
+
+            metadata: Dict[str, Any] = {'file_path': file_path}
+
+            # extract task title
+            title_match = re.search(
+                r'### Task Title\s*\n(.+?)(?:\n###|\n\n|$)', content, re.DOTALL
+            )
+            if title_match:
+                metadata['title'] = title_match.group(1).strip()
+            else:
+                metadata['title'] = ""
+
+            # extract task description
+            desc_match = re.search(
+                r'### Task Description\s*\n(.+?)(?:\n###|\n\n|$)',
+                content,
+                re.DOTALL,
+            )
+            if desc_match:
+                metadata['description'] = desc_match.group(1).strip()
+            else:
+                metadata['description'] = ""
+
+            # extract tags
+            tags_match = re.search(
+                r'### Tags\s*\n(.+?)(?:\n###|\n\n|$)', content, re.DOTALL
+            )
+            if tags_match:
+                tags_section = tags_match.group(1).strip()
+                # Parse bullet list of tags
+                tags = [
+                    line.strip().lstrip('- ')
+                    for line in tags_section.split('\n')
+                    if line.strip().startswith('-')
+                ]
+                metadata['tags'] = tags
+            else:
+                metadata['tags'] = []
+
+            return metadata
+
+        except Exception as e:
+            logger.warning(
+                f"Error extracting workflow info from {file_path}: {e}"
+            )
+            return {}
+
+    def get_all_workflows_info(
+        self, session_id: Optional[str] = None
+    ) -> List[Dict[str, Any]]:
+        r"""Get info from all workflow files in workforce_workflows.
+
+        This method scans the workforce_workflows directory for workflow
+        markdown files and extracts their info for use in workflow
+        selection.
+
+        Args:
+            session_id (Optional[str]): If provided, only return workflows
+                from this specific session. If None, returns workflows from
+                all sessions.
+
+        Returns:
+            List[Dict[str, Any]]: List of workflow info dicts, sorted
+                by session timestamp (newest first).
+        """
+        import glob
+        import re
+
+        workflows_metadata = []
+
+        # Determine base directory for workforce workflows
+        camel_workdir = os.environ.get("CAMEL_WORKDIR")
+        if camel_workdir:
+            base_dir = os.path.join(camel_workdir, "workforce_workflows")
+        else:
+            base_dir = "workforce_workflows"
+
+        # Build search pattern
+        if session_id:
+            search_pattern = os.path.join(
+                base_dir, session_id, "*_workflow.md"
+            )
+        else:
+            search_pattern = os.path.join(base_dir, "*", "*_workflow.md")
+
+        # Find all workflow files
+        workflow_files = glob.glob(search_pattern)
+
+        if not workflow_files:
+            logger.info(f"No workflow files found in {base_dir}")
+            return []
+
+        # Sort by session timestamp (newest first)
+        def extract_session_timestamp(filepath: str) -> str:
+            match = re.search(r'session_(\d{8}_\d{6}_\d{6})', filepath)
+            return match.group(1) if match else ""
+
+        workflow_files.sort(key=extract_session_timestamp, reverse=True)
+
+        # Extract info from each file
+        for file_path in workflow_files:
+            metadata = self.extract_workflow_info(file_path)
+            if metadata:  # Only add if extraction succeeded
+                workflows_metadata.append(metadata)
+
+        logger.info(
+            f"Found {len(workflows_metadata)} workflow file(s) with info"
+        )
+        return workflows_metadata
+
     # ========= SHARED SESSION MANAGEMENT METHODS =========
 
     @classmethod