camel-ai 0.2.23__py3-none-any.whl → 0.2.24__py3-none-any.whl

camel/toolkits/mcp_toolkit.py

@@ -0,0 +1,251 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import inspect
+from contextlib import AsyncExitStack, asynccontextmanager
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Set,
+    Union,
+)
+from urllib.parse import urlparse
+
+if TYPE_CHECKING:
+    from mcp import ListToolsResult, Tool
+
+from camel.toolkits import BaseToolkit, FunctionTool
+
+
+class MCPToolkit(BaseToolkit):
+    r"""MCPToolkit provides an abstraction layer to interact with external
+    tools using the Model Context Protocol (MCP). It supports two modes of
+    connection:
+
+    1. stdio mode: Connects via standard input/output streams for local
+       command-line interactions.
+
+    2. SSE mode (HTTP Server-Sent Events): Connects via HTTP for persistent,
+       event-based interactions.
+
+    Attributes:
+        command_or_url (str): URL for SSE mode or command executable for stdio
+            mode. (default: :obj:`'None'`)
+        args (List[str]): List of command-line arguments if stdio mode is used.
+            (default: :obj:`'None'`)
+        env (Dict[str, str]): Environment variables for the stdio mode command.
+            (default: :obj:`'None'`)
+        timeout (Optional[float]): Connection timeout. (default: :obj:`'None'`)
+    """
+
+    def __init__(
+        self,
+        command_or_url: str,
+        args: Optional[List[str]] = None,
+        env: Optional[Dict[str, str]] = None,
+        timeout: Optional[float] = None,
+    ):
+        from mcp import Tool
+        from mcp.client.session import ClientSession
+
+        super().__init__(timeout=timeout)
+
+        self.command_or_url = command_or_url
+        self.args = args or []
+        self.env = env or {}
+
+        self._mcp_tools: List[Tool] = []
+        self._session: Optional['ClientSession'] = None
+        self._exit_stack = AsyncExitStack()
+        self._is_connected = False
+
+    @asynccontextmanager
+    async def connection(self):
+        r"""Async context manager for establishing and managing the connection
+        with the MCP server. Automatically selects SSE or stdio mode based
+        on the provided `command_or_url`.
+
+        Yields:
+            MCPToolkit: Instance with active connection ready for tool
+                interaction.
+        """
+        from mcp.client.session import ClientSession
+        from mcp.client.sse import sse_client
+        from mcp.client.stdio import StdioServerParameters, stdio_client
+
+        try:
+            if urlparse(self.command_or_url).scheme in ("http", "https"):
+                (
+                    read_stream,
+                    write_stream,
+                ) = await self._exit_stack.enter_async_context(
+                    sse_client(self.command_or_url)
+                )
+            else:
+                server_parameters = StdioServerParameters(
+                    command=self.command_or_url, args=self.args, env=self.env
+                )
+                (
+                    read_stream,
+                    write_stream,
+                ) = await self._exit_stack.enter_async_context(
+                    stdio_client(server_parameters)
+                )
+
+            self._session = await self._exit_stack.enter_async_context(
+                ClientSession(read_stream, write_stream)
+            )
+            await self._session.initialize()
+            list_tools_result = await self.list_mcp_tools()
+            self._mcp_tools = list_tools_result.tools
+            self._is_connected = True
+            yield self
+
+        finally:
+            self._is_connected = False
+            await self._exit_stack.aclose()
+            self._session = None
+
+    async def list_mcp_tools(self) -> Union[str, "ListToolsResult"]:
+        r"""Retrieves the list of available tools from the connected MCP
+        server.
+
+        Returns:
+            ListToolsResult: Result containing available MCP tools.
+        """
+        if not self._session:
+            return "MCP Client is not connected. Call `connection()` first."
+        try:
+            return await self._session.list_tools()
+        except Exception as e:
+            return f"Failed to list MCP tools: {e!s}"
+
+    def generate_function_from_mcp_tool(self, mcp_tool: "Tool") -> Callable:
+        r"""Dynamically generates a Python callable function corresponding to
+        a given MCP tool.
+
+        Args:
+            mcp_tool (Tool): The MCP tool definition received from the MCP
+                server.
+
+        Returns:
+            Callable: A dynamically created async Python function that wraps
+                the MCP tool.
+        """
+        func_name = mcp_tool.name
+        func_desc = mcp_tool.description or "No description provided."
+        parameters_schema = mcp_tool.inputSchema.get("properties", {})
+        required_params = mcp_tool.inputSchema.get("required", [])
+
+        type_map = {
+            "string": str,
+            "integer": int,
+            "number": float,
+            "boolean": bool,
+            "array": list,
+            "object": dict,
+        }
+        annotations = {}  # used to type hints
+        defaults: Dict[str, Any] = {}  # store default values
+
+        func_params = []
+        for param_name, param_schema in parameters_schema.items():
+            param_type = param_schema.get("type", "Any")
+            param_type = type_map.get(param_type, Any)
+
+            annotations[param_name] = param_type
+            if param_name not in required_params:
+                defaults[param_name] = None
+
+            func_params.append(param_name)
+
+        async def dynamic_function(**kwargs):
+            r"""Auto-generated function for MCP Tool interaction.
+
+            Args:
+                kwargs: Keyword arguments corresponding to MCP tool parameters.
+
+            Returns:
+                str: The textual result returned by the MCP tool.
+            """
+            from mcp.types import CallToolResult
+
+            missing_params: Set[str] = set(required_params) - set(
+                kwargs.keys()
+            )
+            if missing_params:
+                raise ValueError(
+                    f"Missing required parameters: {missing_params}"
+                )
+
+            result: CallToolResult = await self._session.call_tool(
+                func_name, kwargs
+            )
+
+            if not result.content:
+                return "No data available for this request."
+
+            # Handle different content types
+            content = result.content[0]
+            if content.type == "text":
+                return content.text
+            elif content.type == "image":
+                # Return image URL or data URI if available
+                if hasattr(content, "url") and content.url:
+                    return f"Image available at: {content.url}"
+                return "Image content received (data URI not shown)"
+            elif content.type == "embedded_resource":
+                # Return resource information if available
+                if hasattr(content, "name") and content.name:
+                    return f"Embedded resource: {content.name}"
+                return "Embedded resource received"
+            else:
+                msg = f"Received content of type '{content.type}'"
+                return f"{msg} which is not fully supported yet."
+
+        dynamic_function.__name__ = func_name
+        dynamic_function.__doc__ = func_desc
+        dynamic_function.__annotations__ = annotations
+
+        sig = inspect.Signature(
+            parameters=[
+                inspect.Parameter(
+                    name=param,
+                    kind=inspect.Parameter.KEYWORD_ONLY,
+                    default=defaults.get(param, inspect.Parameter.empty),
+                    annotation=annotations[param],
+                )
+                for param in func_params
+            ]
+        )
+        dynamic_function.__signature__ = sig  # type: ignore[attr-defined]
+
+        return dynamic_function
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Returns a list of FunctionTool objects representing the
+        functions in the toolkit. Each function is dynamically generated
+        based on the MCP tool definitions received from the server.
+
+        Returns:
+            List[FunctionTool]: A list of FunctionTool objects
+                representing the functions in the toolkit.
+        """
+        return [
+            FunctionTool(self.generate_function_from_mcp_tool(mcp_tool))
+            for mcp_tool in self._mcp_tools
+        ]

camel/toolkits/terminal_toolkit.py

@@ -0,0 +1,421 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import subprocess
+from typing import Any, Dict, List, Optional
+
+from camel.logger import get_logger
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+logger = get_logger(__name__)
+
+
+class TerminalToolkit(BaseToolkit):
+    r"""A toolkit for terminal operations across multiple operating systems.
+
+    This toolkit provides a set of functions for terminal operations such as
+    searching for files by name or content, executing shell commands, and
+    managing terminal sessions.
+
+    Args:
+        timeout (Optional[float]): The timeout for terminal operations.
+        shell_sessions (Optional[Dict[str, Any]]): A dictionary to store
+            shell session information. If None, an empty dictionary will be
+            used.
+
+    Note:
+        Most functions are compatible with Unix-based systems (macOS, Linux).
+        For Windows compatibility, additional implementation details are
+        needed.
+    """
+
+    def __init__(
+        self,
+        timeout: Optional[float] = None,
+        shell_sessions: Optional[Dict[str, Any]] = None,
+    ):
+        import platform
+
+        super().__init__(timeout=timeout)
+        self.shell_sessions = shell_sessions or {}
+        self.os_type = (
+            platform.system()
+        )  # 'Windows', 'Darwin' (macOS), 'Linux'
+
+    def file_find_in_content(
+        self, file: str, regex: str, sudo: bool = False
+    ) -> str:
+        r"""Search for matching text within file content.
+
+        Args:
+            file (str): Absolute path of the file to search within.
+            regex (str): Regular expression pattern to match.
+            sudo (bool, optional): Whether to use sudo privileges. Defaults to
+                False. Note: Using sudo requires the process to have
+                appropriate permissions.
+
+        Returns:
+            str: Matching content found in the file.
+        """
+        if not os.path.exists(file):
+            return f"File not found: {file}"
+
+        if not os.path.isfile(file):
+            return f"The path provided is not a file: {file}"
+
+        command = []
+        if sudo:
+            command.extend(["sudo"])
+
+        if self.os_type in ['Darwin', 'Linux']:  # macOS or Linux
+            command.extend(["grep", "-E", regex, file])
+        else:  # Windows
+            # For Windows, we could use PowerShell or findstr
+            command.extend(["findstr", "/R", regex, file])
+
+        try:
+            result = subprocess.run(
+                command, check=False, capture_output=True, text=True
+            )
+            return result.stdout.strip()
+        except subprocess.SubprocessError as e:
+            logger.error(f"Error searching in file content: {e}")
+            return f"Error: {e!s}"
+
+    def file_find_by_name(self, path: str, glob: str) -> str:
+        r"""Find files by name pattern in specified directory.
+
+        Args:
+            path (str): Absolute path of directory to search.
+            glob (str): Filename pattern using glob syntax wildcards.
+
+        Returns:
+            str: List of files matching the pattern.
+        """
+        if not os.path.exists(path):
+            return f"Directory not found: {path}"
+
+        if not os.path.isdir(path):
+            return f"The path provided is not a directory: {path}"
+
+        command = []
+        if self.os_type in ['Darwin', 'Linux']:  # macOS or Linux
+            command.extend(["find", path, "-name", glob])
+        else:  # Windows
+            # For Windows, we use dir command with /s for recursive search
+            # and /b for bare format
+
+            pattern = glob
+            command.extend(["dir", "/s", "/b", os.path.join(path, pattern)])
+
+        try:
+            result = subprocess.run(
+                command, check=False, capture_output=True, text=True
+            )
+            return result.stdout.strip()
+        except subprocess.SubprocessError as e:
+            logger.error(f"Error finding files by name: {e}")
+            return f"Error: {e!s}"
+
+    def shell_exec(self, id: str, exec_dir: str, command: str) -> str:
+        r"""Execute commands in a specified shell session.
+
+        Args:
+            id (str): Unique identifier of the target shell session.
+            exec_dir (str): Working directory for command execution (must use
+                absolute path).
+            command (str): Shell command to execute.
+
+        Returns:
+            str: Output of the command execution or error message.
+        """
+        if not os.path.isabs(exec_dir):
+            return f"exec_dir must be an absolute path: {exec_dir}"
+
+        if not os.path.exists(exec_dir):
+            return f"Directory not found: {exec_dir}"
+
+        # If the session doesn't exist, create a new one
+        if id not in self.shell_sessions:
+            self.shell_sessions[id] = {
+                "process": None,
+                "output": "",
+                "running": False,
+            }
+
+        try:
+            # Execute the command in the specified directory
+            process = subprocess.Popen(
+                command,
+                shell=True,
+                cwd=exec_dir,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                stdin=subprocess.PIPE,
+                text=False,
+            )
+
+            # Store the process and mark as running
+            self.shell_sessions[id]["process"] = process
+            self.shell_sessions[id]["running"] = True
+            self.shell_sessions[id]["output"] = ""
+
+            # Get initial output (non-blocking)
+            stdout, stderr = "", ""
+            try:
+                if process.stdout:
+                    stdout = process.stdout.read().decode('utf-8')
+                if process.stderr:
+                    stderr = process.stderr.read().decode('utf-8')
+            except Exception as e:
+                logger.error(f"Error reading initial output: {e}")
+                return f"Error: {e!s}"
+
+            output = stdout
+            if stderr:
+                output += f"\nErrors:\n{stderr}"
+
+            self.shell_sessions[id]["output"] = output
+            return (
+                f"Command started in session '{id}'. Initial output: {output}"
+            )
+
+        except subprocess.SubprocessError as e:
+            self.shell_sessions[id]["running"] = False
+            error_msg = f"Error executing command: {e}"
+            self.shell_sessions[id]["output"] = error_msg
+            logger.error(error_msg)
+            return error_msg
+
+    def shell_view(self, id: str) -> str:
+        r"""View the content of a specified shell session.
+
+        Args:
+            id (str): Unique identifier of the target shell session.
+
+        Returns:
+            str: Current output content of the shell session.
+        """
+        if id not in self.shell_sessions:
+            return f"Shell session not found: {id}"
+
+        session = self.shell_sessions[id]
+        process = session.get("process")
+
+        if process is None:
+            return f"No active process in session '{id}'"
+
+        # Try to get any new output
+        if session["running"] and process.poll() is None:
+            try:
+                # Non-blocking read from stdout/stderr
+                stdout_data, stderr_data = "", ""
+                if process.stdout and process.stdout.readable():
+                    stdout_data = process.stdout.read1().decode('utf-8')
+                if process.stderr and process.stderr.readable():
+                    stderr_data = process.stderr.read1().decode('utf-8')
+
+                if stdout_data:
+                    session["output"] += stdout_data
+                if stderr_data:
+                    session["output"] += f"\nErrors:\n{stderr_data}"
+            except Exception as e:
+                logger.error(f"Error getting process output: {e}")
+                return f"Error: {e!s}"
+
+        # Check if the process has completed
+        if process.poll() is not None and session["running"]:
+            try:
+                # Get remaining output if any
+                stdout_data, stderr_data = "", ""
+                if process.stdout and process.stdout.readable():
+                    stdout_data = process.stdout.read().decode('utf-8')
+                if process.stderr and process.stderr.readable():
+                    stderr_data = process.stderr.read().decode('utf-8')
+
+                if stdout_data:
+                    session["output"] += stdout_data
+                if stderr_data:
+                    session["output"] += f"\nErrors:\n{stderr_data}"
+            except Exception as e:
+                logger.error(f"Error getting final process output: {e}")
+                return f"Error: {e!s}"
+            finally:
+                session["running"] = False
+
+        return session["output"]
+
+    def shell_wait(self, id: str, seconds: Optional[int] = None) -> str:
+        r"""Wait for the running process in a specified shell session to
+        return.
+
+        Args:
+            id (str): Unique identifier of the target shell session.
+            seconds (Optional[int], optional): Wait duration in seconds.
+                If None, wait indefinitely. Defaults to None.
+
+        Returns:
+            str: Final output content after waiting.
+        """
+        if id not in self.shell_sessions:
+            return f"Shell session not found: {id}"
+
+        session = self.shell_sessions[id]
+        process = session.get("process")
+
+        if process is None:
+            return f"No active process in session '{id}'"
+
+        if not session["running"]:
+            return f"Process in session '{id}' is not running"
+
+        try:
+            # Use communicate with timeout
+            stdout, stderr = process.communicate(timeout=seconds)
+
+            if stdout:
+                stdout_str = (
+                    stdout.decode('utf-8')
+                    if isinstance(stdout, bytes)
+                    else stdout
+                )
+                session["output"] += stdout_str
+            if stderr:
+                stderr_str = (
+                    stderr.decode('utf-8')
+                    if isinstance(stderr, bytes)
+                    else stderr
+                )
+                session["output"] += f"\nErrors:\n{stderr_str}"
+
+            session["running"] = False
+            return (
+                f"Process completed in session '{id}'. "
+                f"Output: {session['output']}"
+            )
+
+        except subprocess.TimeoutExpired:
+            return (
+                f"Process in session '{id}' is still running "
+                f"after {seconds} seconds"
+            )
+        except Exception as e:
+            logger.error(f"Error waiting for process: {e}")
+            return f"Error waiting for process: {e!s}"
+
+    def shell_write_to_process(
+        self, id: str, input: str, press_enter: bool
+    ) -> str:
+        r"""Write input to a running process in a specified shell session.
+
+        Args:
+            id (str): Unique identifier of the target shell session.
+            input (str): Input content to write to the process.
+            press_enter (bool): Whether to press Enter key after input.
+
+        Returns:
+            str: Status message indicating whether the input was sent.
+        """
+        if id not in self.shell_sessions:
+            return f"Shell session not found: {id}"
+
+        session = self.shell_sessions[id]
+        process = session.get("process")
+
+        if process is None:
+            return f"No active process in session '{id}'"
+
+        if not session["running"] or process.poll() is not None:
+            return f"Process in session '{id}' is not running"
+
+        try:
+            if not process.stdin or process.stdin.closed:
+                return (
+                    f"Cannot write to process in session '{id}': "
+                    f"stdin is closed"
+                )
+
+            if press_enter:
+                input = input + "\n"
+
+            # Write bytes to stdin
+            process.stdin.write(input.encode('utf-8'))
+            process.stdin.flush()
+
+            return f"Input sent to process in session '{id}'"
+        except Exception as e:
+            logger.error(f"Error writing to process: {e}")
+            return f"Error writing to process: {e!s}"
+
+    def shell_kill_process(self, id: str) -> str:
+        r"""Terminate a running process in a specified shell session.
+
+        Args:
+            id (str): Unique identifier of the target shell session.
+
+        Returns:
+            str: Status message indicating whether the process was terminated.
+        """
+        if id not in self.shell_sessions:
+            return f"Shell session not found: {id}"
+
+        session = self.shell_sessions[id]
+        process = session.get("process")
+
+        if process is None:
+            return f"No active process in session '{id}'"
+
+        if not session["running"] or process.poll() is not None:
+            return f"Process in session '{id}' is not running"
+
+        try:
+            # Clean up process resources before termination
+            if process.stdin and not process.stdin.closed:
+                process.stdin.close()
+
+            process.terminate()
+            try:
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                logger.warning(
+                    f"Process in session '{id}' did not terminate gracefully"
+                    f", forcing kill"
+                )
+                process.kill()
+
+            session["running"] = False
+            return f"Process in session '{id}' has been terminated"
+        except Exception as e:
+            logger.error(f"Error killing process: {e}")
+            return f"Error killing process: {e!s}"
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Returns a list of FunctionTool objects representing the functions
+        in the toolkit.
+
+        Returns:
+            List[FunctionTool]: A list of FunctionTool objects representing the
+                functions in the toolkit.
+        """
+        return [
+            FunctionTool(self.file_find_in_content),
+            FunctionTool(self.file_find_by_name),
+            FunctionTool(self.shell_exec),
+            FunctionTool(self.shell_view),
+            FunctionTool(self.shell_wait),
+            FunctionTool(self.shell_write_to_process),
+            FunctionTool(self.shell_kill_process),
+        ]

camel/toolkits/web_toolkit.py

@@ -157,16 +157,15 @@ def _parse_json_output(text: str) -> Dict[str, Any]:
     if triple_quotes_match:
         text = triple_quotes_match.group(1).strip()
 
-    text = text.replace("`", '"')
-
     try:
         return json.loads(text)
     except json.JSONDecodeError:
         try:
-            fixed_text = re.sub(r'`([^`]*)`', r'"\1"', text)
+            fixed_text = re.sub(
+                r'`([^`]*?)`(?=\s*[:,\[\]{}]|$)', r'"\1"', text
+            )
             return json.loads(fixed_text)
         except json.JSONDecodeError:
-            # Try to extract key fields
             result = {}
             try:
                 bool_pattern = r'"(\w+)"\s*:\s*(true|false)'
@@ -986,7 +985,7 @@ Here are an example of the output:
 {{
     "observation": [IMAGE_DESCRIPTION],
     "reasoning": [YOUR_REASONING],
-    "action_code": `fill_input_id([ID], [TEXT])`
+    "action_code": "fill_input_id([ID], [TEXT])"
 }}
 
 Here are some tips for you: