camel-ai 0.2.33__py3-none-any.whl → 0.2.35__py3-none-any.whl

camel/storages/key_value_storages/json.py

@@ -26,7 +26,7 @@ from camel.types import (
 )
 
 
-class _CamelJSONEncoder(json.JSONEncoder):
+class CamelJSONEncoder(json.JSONEncoder):
     r"""A custom JSON encoder for serializing specifically enumerated types.
     Ensures enumerated types can be stored in and retrieved from JSON format.
     """
@@ -62,7 +62,7 @@ class JsonStorage(BaseKeyValueStorage):
     def _json_object_hook(self, d) -> Any:
         if "__enum__" in d:
             name, member = d["__enum__"].split(".")
-            return getattr(_CamelJSONEncoder.CAMEL_ENUMS[name], member)
+            return getattr(CamelJSONEncoder.CAMEL_ENUMS[name], member)
         else:
             return d
 
@@ -75,11 +75,7 @@ class JsonStorage(BaseKeyValueStorage):
         """
         with self.json_path.open("a") as f:
             f.writelines(
-                [
-                    json.dumps(r, cls=_CamelJSONEncoder, ensure_ascii=False)
-                    + "\n"
-                    for r in records
-                ]
+                [json.dumps(r, cls=CamelJSONEncoder) + "\n" for r in records]
             )
 
     def load(self) -> List[Dict[str, Any]]:

camel/storages/vectordb_storages/base.py

@@ -87,7 +87,11 @@ class VectorDBQueryResult(BaseModel):
     ) -> "VectorDBQueryResult":
         r"""A class method to construct a `VectorDBQueryResult` instance."""
         return cls(
-            record=VectorRecord(vector=vector, id=id, payload=payload),
+            record=VectorRecord(
+                vector=vector,
+                id=id,
+                payload=payload,
+            ),
             similarity=similarity,
         )
 

camel/toolkits/__init__.py

@@ -50,6 +50,7 @@ from .semantic_scholar_toolkit import SemanticScholarToolkit
 from .zapier_toolkit import ZapierToolkit
 from .sympy_toolkit import SymPyToolkit
 from .mineru_toolkit import MinerUToolkit
+from .memory_toolkit import MemoryToolkit
 from .audio_analysis_toolkit import AudioAnalysisToolkit
 from .excel_toolkit import ExcelToolkit
 from .video_analysis_toolkit import VideoAnalysisToolkit
@@ -60,7 +61,6 @@ from .file_write_toolkit import FileWriteToolkit
 from .terminal_toolkit import TerminalToolkit
 from .pubmed_toolkit import PubMedToolkit
 
-
 __all__ = [
     'BaseToolkit',
     'FunctionTool',
@@ -97,6 +97,7 @@ __all__ = [
     'ZapierToolkit',
     'SymPyToolkit',
     'MinerUToolkit',
+    'MemoryToolkit',
     'MCPToolkit',
     'MCPToolkitManager',
     'AudioAnalysisToolkit',

camel/toolkits/file_write_toolkit.py

@@ -13,6 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 
+import re
 from datetime import datetime
 from pathlib import Path
 from typing import List, Optional, Union
@@ -69,17 +70,22 @@ class FileWriteToolkit(BaseToolkit):
         r"""Convert the given string path to a Path object.
 
         If the provided path is not absolute, it is made relative to the
-        default output directory.
+        default output directory. The filename part is sanitized to replace
+        spaces and special characters with underscores, ensuring safe usage
+        in downstream processing.
 
         Args:
             file_path (str): The file path to resolve.
 
         Returns:
-            Path: A fully resolved (absolute) Path object.
+            Path: A fully resolved (absolute) and sanitized Path object.
         """
         path_obj = Path(file_path)
         if not path_obj.is_absolute():
             path_obj = self.output_dir / path_obj
+
+        sanitized_filename = self._sanitize_filename(path_obj.name)
+        path_obj = path_obj.parent / sanitized_filename
         return path_obj.resolve()
 
     def _write_text_file(
@@ -369,3 +375,19 @@ class FileWriteToolkit(BaseToolkit):
         return [
             FunctionTool(self.write_to_file),
         ]
+
+    def _sanitize_filename(self, filename: str) -> str:
+        r"""Sanitize a filename by replacing any character that is not
+        alphanumeric, a dot (.), hyphen (-), or underscore (_) with an
+        underscore (_).
+
+        Args:
+            filename (str): The original filename which may contain spaces or
+                special characters.
+
+        Returns:
+            str: The sanitized filename with disallowed characters replaced by
+                underscores.
+        """
+        safe = re.sub(r'[^\w\-.]', '_', filename)
+        return safe

camel/toolkits/github_toolkit.py

@@ -110,9 +110,21 @@ class GithubToolkit(BaseToolkit):
                 successfully or not.
         """
         sb = self.repo.get_branch(self.repo.default_branch)
-        self.repo.create_git_ref(
-            ref=f"refs/heads/{branch_name}", sha=sb.commit.sha
-        )
+        from github import GithubException
+
+        try:
+            self.repo.create_git_ref(
+                ref=f"refs/heads/{branch_name}", sha=sb.commit.sha
+            )
+        except GithubException as e:
+            if e.message == "Reference already exists":
+                # agent might have pushed the branch separately.
+                logger.warning(
+                    f"Branch {branch_name} already exists. "
+                    "Continuing with the existing branch."
+                )
+            else:
+                raise
 
         file = self.repo.get_contents(file_path)
 

camel/toolkits/memory_toolkit.py

@@ -0,0 +1,129 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+from typing import TYPE_CHECKING, Optional
+
+from camel.memories import (
+    ChatHistoryMemory,
+    MemoryRecord,
+    ScoreBasedContextCreator,
+)
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+if TYPE_CHECKING:
+    from camel.agents import ChatAgent
+
+
+class MemoryToolkit(BaseToolkit):
+    r"""A toolkit that provides methods for saving, loading, and clearing a
+    ChatAgent's memory.
+    These methods are exposed as FunctionTool objects for
+    function calling. Internally, it calls:
+      - agent.save_memory(path)
+      - agent.load_memory(new_memory_obj)
+      - agent.load_memory_from_path(path)
+      - agent.clear_memory()
+
+    Args:
+        agent (ChatAgent): The chat agent whose memory will be managed.
+        timeout (Optional[float], optional): Maximum execution time allowed for
+            toolkit operations in seconds. If None, no timeout is applied.
+            (default: :obj:`None`)
+    """
+
+    def __init__(self, agent: 'ChatAgent', timeout: Optional[float] = None):
+        super().__init__(timeout=timeout)
+        self.agent = agent
+
+    def save(self, path: str) -> str:
+        r"""Saves the agent's current memory to a JSON file.
+
+        Args:
+            path (str): The file path to save the memory to.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.save_memory(path)
+        return f"Memory saved to {path}"
+
+    def load(self, memory_json: str) -> str:
+        r"""Loads memory into the agent from a JSON string.
+
+        Args:
+            memory_json (str): A JSON string containing memory records.
+
+        Returns:
+            str: Confirmation or error message.
+        """
+        try:
+            data = json.loads(memory_json.strip())
+            if not isinstance(data, list):
+                return "[ERROR] Memory data should be a list of records."
+
+            # Build a fresh ChatHistoryMemory
+            context_creator = ScoreBasedContextCreator(
+                token_counter=self.agent.model_backend.token_counter,
+                token_limit=self.agent.model_backend.token_limit,
+            )
+            new_memory = ChatHistoryMemory(context_creator)
+
+            # Convert each record dict -> MemoryRecord
+            for record_dict in data:
+                record = MemoryRecord.from_dict(record_dict)
+                new_memory.write_record(record)
+
+            # Load into the agent
+            self.agent.load_memory(new_memory)
+            return "Loaded memory from provided JSON string."
+        except json.JSONDecodeError:
+            return "[ERROR] Invalid JSON string provided."
+        except Exception as e:
+            return f"[ERROR] Failed to load memory: {e!s}"
+
+    def load_from_path(self, path: str) -> str:
+        r"""Loads the agent's memory from a JSON file.
+
+        Args:
+            path (str): The file path to load the memory from.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.load_memory_from_path(path)
+        return f"Memory loaded from {path}"
+
+    def clear_memory(self) -> str:
+        r"""Clears the agent's memory.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.clear_memory()
+        return "Memory has been cleared."
+
+    def get_tools(self) -> list[FunctionTool]:
+        r"""Expose the memory management methods as function tools
+        for the ChatAgent.
+
+        Returns:
+            list[FunctionTool]: List of FunctionTool objects.
+        """
+        return [
+            FunctionTool(self.save),
+            FunctionTool(self.load),
+            FunctionTool(self.load_from_path),
+            FunctionTool(self.clear_memory),
+        ]

camel/utils/chunker/__init__.py

@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseChunker
+from .code_chunker import CodeChunker
+from .uio_chunker import UnstructuredIOChunker
+
+__all__ = [
+    "BaseChunker",
+    "CodeChunker",
+    "UnstructuredIOChunker",
+]

camel/utils/chunker/base.py

@@ -0,0 +1,24 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseChunker(ABC):
+    r"""An abstract base class for all CAMEL chunkers."""
+
+    @abstractmethod
+    def chunk(self, content: Any) -> Any:
+        r"""Chunk the given content"""
+        pass

camel/utils/chunker/code_chunker.py

@@ -0,0 +1,193 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+from typing import List, Optional
+
+from unstructured.documents.elements import Element, ElementMetadata
+
+from camel.messages import OpenAIUserMessage
+from camel.types import ModelType
+from camel.utils import BaseTokenCounter, OpenAITokenCounter
+
+from .base import BaseChunker
+
+
+class CodeChunker(BaseChunker):
+    r"""A class for chunking code or text while respecting structure
+        and token limits.
+
+        This class ensures that structured elements such as functions,
+        classes, and regions are not arbitrarily split across chunks.
+        It also handles oversized lines and Base64-encoded images.
+
+    Attributes:
+        chunk_size (int, optional): The maximum token size per chunk.
+            (default: :obj:`8192`)
+        token_counter (BaseTokenCounter, optional): The tokenizer used for
+            token counting, if `None`, OpenAITokenCounter will be used.
+            (default: :obj:`None`)
+        remove_image: (bool, optional): If the chunker should skip the images.
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 8192,
+        token_counter: Optional[BaseTokenCounter] = None,
+        remove_image: Optional[bool] = True,
+    ):
+        self.chunk_size = chunk_size
+        self.token_counter = (
+            token_counter
+            if token_counter
+            else OpenAITokenCounter(model=ModelType.GPT_4O_MINI)
+        )
+        self.remove_image = remove_image
+        self.struct_pattern = re.compile(
+            r'^\s*(?:(def|class|function)\s+\w+|'
+            r'(public|private|protected)\s+[\w<>]+\s+\w+\s*\(|'
+            r'\b(interface|enum|namespace)\s+\w+|'
+            r'#\s*(region|endregion)\b)'
+        )
+        self.image_pattern = re.compile(
+            r'!\[.*?\]\((?:data:image/[^;]+;base64,[a-zA-Z0-9+/]+=*|[^)]+)\)'
+        )
+
+    def count_tokens(self, text: str):
+        r"""Counts the number of tokens in the given text.
+
+        Args:
+            text (str): The input text to be tokenized.
+
+        Returns:
+            int: The number of tokens in the input text.
+        """
+        return self.token_counter.count_tokens_from_messages(
+            [OpenAIUserMessage(role="user", name="user", content=text)]
+        )
+
+    def _split_oversized(self, line: str) -> List[str]:
+        r"""Splits an oversized line into multiple chunks based on token limits
+
+        Args:
+            line (str): The oversized line to be split.
+
+        Returns:
+            List[str]: A list of smaller chunks after splitting the
+                oversized line.
+        """
+        tokens = self.token_counter.encode(line)
+        chunks = []
+        buffer = []
+        current_count = 0
+
+        for token in tokens:
+            buffer.append(token)
+            current_count += 1
+
+            if current_count >= self.chunk_size:
+                chunks.append(self.token_counter.decode(buffer).strip())
+                buffer = []
+                current_count = 0
+
+        if buffer:
+            chunks.append(self.token_counter.decode(buffer))
+        return chunks
+
+    def chunk(self, content: List[str]) -> List[Element]:
+        r"""Splits the content into smaller chunks while preserving
+        structure and adhering to token constraints.
+
+        Args:
+            content (List[str]): The content to be chunked.
+
+        Returns:
+            List[str]: A list of chunked text segments.
+        """
+        content_str = "\n".join(map(str, content))
+        chunks = []
+        current_chunk: list[str] = []
+        current_tokens = 0
+        struct_buffer: list[str] = []
+        struct_tokens = 0
+
+        for line in content_str.splitlines(keepends=True):
+            if self.remove_image:
+                if self.image_pattern.match(line):
+                    continue
+
+            line_tokens = self.count_tokens(line)
+
+            if line_tokens > self.chunk_size:
+                if current_chunk:
+                    chunks.append("".join(current_chunk))
+                    current_chunk = []
+                    current_tokens = 0
+                chunks.extend(self._split_oversized(line))
+                continue
+
+            if self.struct_pattern.match(line):
+                if struct_buffer:
+                    if current_tokens + struct_tokens <= self.chunk_size:
+                        current_chunk.extend(struct_buffer)
+                        current_tokens += struct_tokens
+                    else:
+                        if current_chunk:
+                            chunks.append("".join(current_chunk))
+                        current_chunk = struct_buffer.copy()
+                        current_tokens = struct_tokens
+                    struct_buffer = []
+                    struct_tokens = 0
+
+                struct_buffer.append(line)
+                struct_tokens += line_tokens
+            else:
+                if struct_buffer:
+                    struct_buffer.append(line)
+                    struct_tokens += line_tokens
+                else:
+                    if current_tokens + line_tokens > self.chunk_size:
+                        chunks.append("".join(current_chunk))
+                        current_chunk = [line]
+                        current_tokens = line_tokens
+                    else:
+                        current_chunk.append(line)
+                        current_tokens += line_tokens
+
+        if struct_buffer:
+            if current_tokens + struct_tokens <= self.chunk_size:
+                current_chunk.extend(struct_buffer)
+            else:
+                if current_chunk:
+                    chunks.append("".join(current_chunk))
+                current_chunk = struct_buffer
+
+        if current_chunk:
+            chunks.append("".join(current_chunk))
+
+        final_chunks = []
+        for chunk in chunks:
+            chunk_token = self.count_tokens(chunk)
+            if chunk_token > self.chunk_size:
+                final_chunks.extend(self._split_oversized(chunk))
+            else:
+                final_chunks.append(chunk)
+
+        # TODO: need to reconsider how to correctly form metadata (maybe need
+        # to decouple the connection with unstructuredIO)
+        chunked_elements = []
+        for chunk in final_chunks:
+            element = Element(metadata=ElementMetadata())
+            element.text = chunk
+            chunked_elements.append(element)
+        return chunked_elements

camel/utils/chunker/uio_chunker.py

@@ -0,0 +1,66 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import List, Optional
+
+from unstructured.documents.elements import Element
+
+from camel.loaders import UnstructuredIO
+from camel.utils.chunker import BaseChunker
+
+
+class UnstructuredIOChunker(BaseChunker):
+    r"""A class for chunking text while respecting structure and
+    character limits.
+
+    This class ensures that structured elements, such as document sections
+    and titles, are not arbitrarily split across chunks. It utilizes the
+    `UnstructuredIO` class to process and segment elements while maintaining
+    readability and coherence. The chunking method can be adjusted based on
+    the provided `chunk_type` parameter.
+
+    Args:
+        chunk_type (str, optional): The method used for chunking text.
+            (default: :obj:`"chunk_by_title"`)
+        max_characters (int, optional): The maximum number of characters
+            allowed per chunk. (default: :obj:`500`)
+        metadata_filename (Optional[str], optional): An optional filename
+            for storing metadata related to chunking. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        chunk_type: str = "chunk_by_title",
+        max_characters: int = 500,
+        metadata_filename: Optional[str] = None,
+    ):
+        self.uio = UnstructuredIO()
+        self.chunk_type = chunk_type
+        self.max_characters = max_characters
+        self.metadata_filename = metadata_filename
+
+    def chunk(self, content: List[Element]) -> List[Element]:
+        r"""Splits the content into smaller chunks while preserving
+        structure and adhering to token constraints.
+
+        Args:
+            content (List[Element]): The content to be chunked.
+
+        Returns:
+            List[Element]: A list of chunked text segments.
+        """
+        return self.uio.chunk_elements(
+            chunk_type=self.chunk_type,
+            elements=content,
+            max_characters=self.max_characters,
+        )