camel-ai 0.2.34__py3-none-any.whl → 0.2.36__py3-none-any.whl

camel/toolkits/browser_toolkit.py

@@ -438,6 +438,7 @@ class BaseBrowser:
             sync_playwright,
         )
 
+        self._ensure_browser_installed()
         self.history: list = []
         self.headless = headless
         self.playwright = sync_playwright().start()
@@ -914,6 +915,48 @@ class BaseBrowser:
         markdown_content = html2text(html_content)
         return markdown_content
 
+    def _ensure_browser_installed(self) -> None:
+        r"""Ensure the browser is installed."""
+        import platform
+        import subprocess
+        import sys
+
+        try:
+            from playwright.sync_api import sync_playwright
+
+            with sync_playwright() as p:
+                browser = p.chromium.launch()
+                browser.close()
+        except Exception:
+            logger.info("Installing Chromium browser...")
+            try:
+                subprocess.run(
+                    [
+                        sys.executable,
+                        "-m",
+                        "playwright",
+                        "install",
+                        "chromium",
+                    ],
+                    check=True,
+                    capture_output=True,
+                )
+                if platform.system().lower() == "linux":
+                    subprocess.run(
+                        [
+                            sys.executable,
+                            "-m",
+                            "playwright",
+                            "install-deps",
+                            "chromium",
+                        ],
+                        check=True,
+                        capture_output=True,
+                    )
+                logger.info("Chromium browser installation completed")
+            except subprocess.CalledProcessError as e:
+                raise RuntimeError(f"Failed to install browser: {e.stderr}")
+
 
 class BrowserToolkit(BaseToolkit):
     r"""A class for browsing the web and interacting with web pages.

camel/toolkits/code_execution.py

@@ -29,6 +29,7 @@ class CodeExecutionToolkit(BaseToolkit):
 
     Args:
         sandbox (str): The environment type used to execute code.
+            (default: `subprocess`)
         verbose (bool): Whether to print the output of the code execution.
             (default: :obj:`False`)
         unsafe_mode (bool):  If `True`, the interpreter runs the code
@@ -43,7 +44,7 @@ class CodeExecutionToolkit(BaseToolkit):
         self,
         sandbox: Literal[
             "internal_python", "jupyter", "docker", "subprocess", "e2b"
-        ] = "internal_python",
+        ] = "subprocess",
         verbose: bool = False,
         unsafe_mode: bool = False,
         import_white_list: Optional[List[str]] = None,

camel/toolkits/mcp_toolkit.py

@@ -56,6 +56,8 @@ class _MCPServer(BaseToolkit):
         env (Dict[str, str]): Environment variables for the stdio mode command.
             (default: :obj:`'None'`)
         timeout (Optional[float]): Connection timeout. (default: :obj:`'None'`)
+        headers (Dict[str, str]): Headers for the HTTP request.
+            (default: :obj:`'None'`)
     """
 
     def __init__(
@@ -64,6 +66,7 @@ class _MCPServer(BaseToolkit):
         args: Optional[List[str]] = None,
         env: Optional[Dict[str, str]] = None,
         timeout: Optional[float] = None,
+        headers: Optional[Dict[str, str]] = None,
     ):
         from mcp import Tool
         from mcp.client.session import ClientSession
@@ -73,6 +76,7 @@ class _MCPServer(BaseToolkit):
         self.command_or_url = command_or_url
         self.args = args or []
         self.env = env or {}
+        self.headers = headers or {}
 
         self._mcp_tools: List[Tool] = []
         self._session: Optional['ClientSession'] = None
@@ -99,7 +103,10 @@ class _MCPServer(BaseToolkit):
                     read_stream,
                     write_stream,
                 ) = await self._exit_stack.enter_async_context(
-                    sse_client(self.command_or_url)
+                    sse_client(
+                        self.command_or_url,
+                        headers=self.headers,
+                    )
                 )
             else:
                 command = self.command_or_url
@@ -343,6 +350,27 @@ class MCPToolkit(BaseToolkit):
         Either `servers` or `config_path` must be provided. If both are
         provided, servers from both sources will be combined.
 
+        For web servers in the config file, you can specify authorization
+        headers using the "headers" field to connect to protected MCP server
+        endpoints.
+
+        Example configuration:
+
+        .. code-block:: json
+
+            {
+              "mcpWebServers": {
+                "protected-server": {
+                  "url": "https://example.com/mcp",
+                  "timeout": 30,
+                  "headers": {
+                    "Authorization": "Bearer YOUR_TOKEN",
+                    "X-API-Key": "YOUR_API_KEY"
+                  }
+                }
+              }
+            }
+
     Attributes:
         servers (List[_MCPServer]): List of _MCPServer instances being managed.
     """
@@ -442,6 +470,7 @@ class MCPToolkit(BaseToolkit):
             server = _MCPServer(
                 command_or_url=cfg["url"],
                 timeout=cfg.get("timeout", None),
+                headers=cfg.get("headers", {}),
             )
             all_servers.append(server)
 

camel/toolkits/memory_toolkit.py

@@ -0,0 +1,129 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+from typing import TYPE_CHECKING, Optional
+
+from camel.memories import (
+    ChatHistoryMemory,
+    MemoryRecord,
+    ScoreBasedContextCreator,
+)
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+if TYPE_CHECKING:
+    from camel.agents import ChatAgent
+
+
+class MemoryToolkit(BaseToolkit):
+    r"""A toolkit that provides methods for saving, loading, and clearing a
+    ChatAgent's memory.
+    These methods are exposed as FunctionTool objects for
+    function calling. Internally, it calls:
+      - agent.save_memory(path)
+      - agent.load_memory(new_memory_obj)
+      - agent.load_memory_from_path(path)
+      - agent.clear_memory()
+
+    Args:
+        agent (ChatAgent): The chat agent whose memory will be managed.
+        timeout (Optional[float], optional): Maximum execution time allowed for
+            toolkit operations in seconds. If None, no timeout is applied.
+            (default: :obj:`None`)
+    """
+
+    def __init__(self, agent: 'ChatAgent', timeout: Optional[float] = None):
+        super().__init__(timeout=timeout)
+        self.agent = agent
+
+    def save(self, path: str) -> str:
+        r"""Saves the agent's current memory to a JSON file.
+
+        Args:
+            path (str): The file path to save the memory to.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.save_memory(path)
+        return f"Memory saved to {path}"
+
+    def load(self, memory_json: str) -> str:
+        r"""Loads memory into the agent from a JSON string.
+
+        Args:
+            memory_json (str): A JSON string containing memory records.
+
+        Returns:
+            str: Confirmation or error message.
+        """
+        try:
+            data = json.loads(memory_json.strip())
+            if not isinstance(data, list):
+                return "[ERROR] Memory data should be a list of records."
+
+            # Build a fresh ChatHistoryMemory
+            context_creator = ScoreBasedContextCreator(
+                token_counter=self.agent.model_backend.token_counter,
+                token_limit=self.agent.model_backend.token_limit,
+            )
+            new_memory = ChatHistoryMemory(context_creator)
+
+            # Convert each record dict -> MemoryRecord
+            for record_dict in data:
+                record = MemoryRecord.from_dict(record_dict)
+                new_memory.write_record(record)
+
+            # Load into the agent
+            self.agent.load_memory(new_memory)
+            return "Loaded memory from provided JSON string."
+        except json.JSONDecodeError:
+            return "[ERROR] Invalid JSON string provided."
+        except Exception as e:
+            return f"[ERROR] Failed to load memory: {e!s}"
+
+    def load_from_path(self, path: str) -> str:
+        r"""Loads the agent's memory from a JSON file.
+
+        Args:
+            path (str): The file path to load the memory from.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.load_memory_from_path(path)
+        return f"Memory loaded from {path}"
+
+    def clear_memory(self) -> str:
+        r"""Clears the agent's memory.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.clear_memory()
+        return "Memory has been cleared."
+
+    def get_tools(self) -> list[FunctionTool]:
+        r"""Expose the memory management methods as function tools
+        for the ChatAgent.
+
+        Returns:
+            list[FunctionTool]: List of FunctionTool objects.
+        """
+        return [
+            FunctionTool(self.save),
+            FunctionTool(self.load),
+            FunctionTool(self.load_from_path),
+            FunctionTool(self.clear_memory),
+        ]

camel/types/enums.py

@@ -63,6 +63,11 @@ class ModelType(UnifiedModelType, Enum):
     GROQ_MIXTRAL_8_7B = "mixtral-8x7b-32768"
     GROQ_GEMMA_2_9B_IT = "gemma2-9b-it"
 
+    # OpenRouter models
+    OPENROUTER_LLAMA_3_1_405B = "meta-llama/llama-3.3-405b-instruct"
+    OPENROUTER_LLAMA_3_1_70B = "meta-llama/llama-3.3-70b-instruct"
+    OPENROUTER_OLYMPICODER_7B = "open-r1/olympiccoder-7b:free"
+
     # TogetherAI platform models support tool calling
     TOGETHER_LLAMA_3_1_8B = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo"
     TOGETHER_LLAMA_3_1_70B = "meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo"
@@ -253,6 +258,7 @@ class ModelType(UnifiedModelType, Enum):
                 self.is_together,
                 self.is_sambanova,
                 self.is_groq,
+                self.is_openrouter,
                 self.is_sglang,
                 self.is_moonshot,
                 self.is_siliconflow,
@@ -342,6 +348,15 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.GROQ_GEMMA_2_9B_IT,
         }
 
+    @property
+    def is_openrouter(self) -> bool:
+        r"""Returns whether this type of models is served by OpenRouter."""
+        return self in {
+            ModelType.OPENROUTER_LLAMA_3_1_405B,
+            ModelType.OPENROUTER_LLAMA_3_1_70B,
+            ModelType.OPENROUTER_OLYMPICODER_7B,
+        }
+
     @property
     def is_together(self) -> bool:
         r"""Returns whether this type of models is served by Together AI."""
@@ -580,6 +595,7 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.MOONSHOT_V1_8K,
             ModelType.GLM_4V_FLASH,
             ModelType.GLM_4_AIRX,
+            ModelType.OPENROUTER_OLYMPICODER_7B,
         }:
             return 8_192
         elif self in {
@@ -686,6 +702,8 @@ class ModelType(UnifiedModelType, Enum):
             ModelType.SGLANG_QWEN_2_5_7B,
             ModelType.SGLANG_QWEN_2_5_32B,
             ModelType.SGLANG_QWEN_2_5_72B,
+            ModelType.OPENROUTER_LLAMA_3_1_70B,
+            ModelType.OPENROUTER_LLAMA_3_1_405B,
         }:
             return 131_072
         elif self in {
@@ -881,6 +899,7 @@ class ModelPlatformType(Enum):
     AZURE = "azure"
     ANTHROPIC = "anthropic"
     GROQ = "groq"
+    OPENROUTER = "openrouter"
     OLLAMA = "ollama"
     LITELLM = "litellm"
     ZHIPU = "zhipuai"
@@ -931,6 +950,11 @@ class ModelPlatformType(Enum):
         r"""Returns whether this platform is groq."""
         return self is ModelPlatformType.GROQ
 
+    @property
+    def is_openrouter(self) -> bool:
+        r"""Returns whether this platform is openrouter."""
+        return self is ModelPlatformType.OPENROUTER
+
     @property
     def is_ollama(self) -> bool:
         r"""Returns whether this platform is ollama."""

camel/types/unified_model_type.py

@@ -78,6 +78,11 @@ class UnifiedModelType(str):
         r"""Returns whether the model is a Groq served model."""
         return True
 
+    @property
+    def is_openrouter(self) -> bool:
+        r"""Returns whether the model is a OpenRouter served model."""
+        return True
+
     @property
     def is_zhipuai(self) -> bool:
         r"""Returns whether the model is a Zhipuai model."""

camel/utils/chunker/__init__.py

@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseChunker
+from .code_chunker import CodeChunker
+from .uio_chunker import UnstructuredIOChunker
+
+__all__ = [
+    "BaseChunker",
+    "CodeChunker",
+    "UnstructuredIOChunker",
+]

camel/utils/chunker/base.py

@@ -0,0 +1,24 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseChunker(ABC):
+    r"""An abstract base class for all CAMEL chunkers."""
+
+    @abstractmethod
+    def chunk(self, content: Any) -> Any:
+        r"""Chunk the given content"""
+        pass

camel/utils/chunker/code_chunker.py

@@ -0,0 +1,193 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+from typing import List, Optional
+
+from unstructured.documents.elements import Element, ElementMetadata
+
+from camel.messages import OpenAIUserMessage
+from camel.types import ModelType
+from camel.utils import BaseTokenCounter, OpenAITokenCounter
+
+from .base import BaseChunker
+
+
+class CodeChunker(BaseChunker):
+    r"""A class for chunking code or text while respecting structure
+        and token limits.
+
+        This class ensures that structured elements such as functions,
+        classes, and regions are not arbitrarily split across chunks.
+        It also handles oversized lines and Base64-encoded images.
+
+    Attributes:
+        chunk_size (int, optional): The maximum token size per chunk.
+            (default: :obj:`8192`)
+        token_counter (BaseTokenCounter, optional): The tokenizer used for
+            token counting, if `None`, OpenAITokenCounter will be used.
+            (default: :obj:`None`)
+        remove_image: (bool, optional): If the chunker should skip the images.
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 8192,
+        token_counter: Optional[BaseTokenCounter] = None,
+        remove_image: Optional[bool] = True,
+    ):
+        self.chunk_size = chunk_size
+        self.token_counter = (
+            token_counter
+            if token_counter
+            else OpenAITokenCounter(model=ModelType.GPT_4O_MINI)
+        )
+        self.remove_image = remove_image
+        self.struct_pattern = re.compile(
+            r'^\s*(?:(def|class|function)\s+\w+|'
+            r'(public|private|protected)\s+[\w<>]+\s+\w+\s*\(|'
+            r'\b(interface|enum|namespace)\s+\w+|'
+            r'#\s*(region|endregion)\b)'
+        )
+        self.image_pattern = re.compile(
+            r'!\[.*?\]\((?:data:image/[^;]+;base64,[a-zA-Z0-9+/]+=*|[^)]+)\)'
+        )
+
+    def count_tokens(self, text: str):
+        r"""Counts the number of tokens in the given text.
+
+        Args:
+            text (str): The input text to be tokenized.
+
+        Returns:
+            int: The number of tokens in the input text.
+        """
+        return self.token_counter.count_tokens_from_messages(
+            [OpenAIUserMessage(role="user", name="user", content=text)]
+        )
+
+    def _split_oversized(self, line: str) -> List[str]:
+        r"""Splits an oversized line into multiple chunks based on token limits
+
+        Args:
+            line (str): The oversized line to be split.
+
+        Returns:
+            List[str]: A list of smaller chunks after splitting the
+                oversized line.
+        """
+        tokens = self.token_counter.encode(line)
+        chunks = []
+        buffer = []
+        current_count = 0
+
+        for token in tokens:
+            buffer.append(token)
+            current_count += 1
+
+            if current_count >= self.chunk_size:
+                chunks.append(self.token_counter.decode(buffer).strip())
+                buffer = []
+                current_count = 0
+
+        if buffer:
+            chunks.append(self.token_counter.decode(buffer))
+        return chunks
+
+    def chunk(self, content: List[str]) -> List[Element]:
+        r"""Splits the content into smaller chunks while preserving
+        structure and adhering to token constraints.
+
+        Args:
+            content (List[str]): The content to be chunked.
+
+        Returns:
+            List[str]: A list of chunked text segments.
+        """
+        content_str = "\n".join(map(str, content))
+        chunks = []
+        current_chunk: list[str] = []
+        current_tokens = 0
+        struct_buffer: list[str] = []
+        struct_tokens = 0
+
+        for line in content_str.splitlines(keepends=True):
+            if self.remove_image:
+                if self.image_pattern.match(line):
+                    continue
+
+            line_tokens = self.count_tokens(line)
+
+            if line_tokens > self.chunk_size:
+                if current_chunk:
+                    chunks.append("".join(current_chunk))
+                    current_chunk = []
+                    current_tokens = 0
+                chunks.extend(self._split_oversized(line))
+                continue
+
+            if self.struct_pattern.match(line):
+                if struct_buffer:
+                    if current_tokens + struct_tokens <= self.chunk_size:
+                        current_chunk.extend(struct_buffer)
+                        current_tokens += struct_tokens
+                    else:
+                        if current_chunk:
+                            chunks.append("".join(current_chunk))
+                        current_chunk = struct_buffer.copy()
+                        current_tokens = struct_tokens
+                    struct_buffer = []
+                    struct_tokens = 0
+
+                struct_buffer.append(line)
+                struct_tokens += line_tokens
+            else:
+                if struct_buffer:
+                    struct_buffer.append(line)
+                    struct_tokens += line_tokens
+                else:
+                    if current_tokens + line_tokens > self.chunk_size:
+                        chunks.append("".join(current_chunk))
+                        current_chunk = [line]
+                        current_tokens = line_tokens
+                    else:
+                        current_chunk.append(line)
+                        current_tokens += line_tokens
+
+        if struct_buffer:
+            if current_tokens + struct_tokens <= self.chunk_size:
+                current_chunk.extend(struct_buffer)
+            else:
+                if current_chunk:
+                    chunks.append("".join(current_chunk))
+                current_chunk = struct_buffer
+
+        if current_chunk:
+            chunks.append("".join(current_chunk))
+
+        final_chunks = []
+        for chunk in chunks:
+            chunk_token = self.count_tokens(chunk)
+            if chunk_token > self.chunk_size:
+                final_chunks.extend(self._split_oversized(chunk))
+            else:
+                final_chunks.append(chunk)
+
+        # TODO: need to reconsider how to correctly form metadata (maybe need
+        # to decouple the connection with unstructuredIO)
+        chunked_elements = []
+        for chunk in final_chunks:
+            element = Element(metadata=ElementMetadata())
+            element.text = chunk
+            chunked_elements.append(element)
+        return chunked_elements

camel/utils/chunker/uio_chunker.py

@@ -0,0 +1,66 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import List, Optional
+
+from unstructured.documents.elements import Element
+
+from camel.loaders import UnstructuredIO
+from camel.utils.chunker import BaseChunker
+
+
+class UnstructuredIOChunker(BaseChunker):
+    r"""A class for chunking text while respecting structure and
+    character limits.
+
+    This class ensures that structured elements, such as document sections
+    and titles, are not arbitrarily split across chunks. It utilizes the
+    `UnstructuredIO` class to process and segment elements while maintaining
+    readability and coherence. The chunking method can be adjusted based on
+    the provided `chunk_type` parameter.
+
+    Args:
+        chunk_type (str, optional): The method used for chunking text.
+            (default: :obj:`"chunk_by_title"`)
+        max_characters (int, optional): The maximum number of characters
+            allowed per chunk. (default: :obj:`500`)
+        metadata_filename (Optional[str], optional): An optional filename
+            for storing metadata related to chunking. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        chunk_type: str = "chunk_by_title",
+        max_characters: int = 500,
+        metadata_filename: Optional[str] = None,
+    ):
+        self.uio = UnstructuredIO()
+        self.chunk_type = chunk_type
+        self.max_characters = max_characters
+        self.metadata_filename = metadata_filename
+
+    def chunk(self, content: List[Element]) -> List[Element]:
+        r"""Splits the content into smaller chunks while preserving
+        structure and adhering to token constraints.
+
+        Args:
+            content (List[Element]): The content to be chunked.
+
+        Returns:
+            List[Element]: A list of chunked text segments.
+        """
+        return self.uio.chunk_elements(
+            chunk_type=self.chunk_type,
+            elements=content,
+            max_characters=self.max_characters,
+        )