camel-ai 0.2.34__py3-none-any.whl → 0.2.35__py3-none-any.whl

camel/toolkits/memory_toolkit.py

@@ -0,0 +1,129 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+from typing import TYPE_CHECKING, Optional
+
+from camel.memories import (
+    ChatHistoryMemory,
+    MemoryRecord,
+    ScoreBasedContextCreator,
+)
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+
+if TYPE_CHECKING:
+    from camel.agents import ChatAgent
+
+
+class MemoryToolkit(BaseToolkit):
+    r"""A toolkit that provides methods for saving, loading, and clearing a
+    ChatAgent's memory.
+    These methods are exposed as FunctionTool objects for
+    function calling. Internally, it calls:
+      - agent.save_memory(path)
+      - agent.load_memory(new_memory_obj)
+      - agent.load_memory_from_path(path)
+      - agent.clear_memory()
+
+    Args:
+        agent (ChatAgent): The chat agent whose memory will be managed.
+        timeout (Optional[float], optional): Maximum execution time allowed for
+            toolkit operations in seconds. If None, no timeout is applied.
+            (default: :obj:`None`)
+    """
+
+    def __init__(self, agent: 'ChatAgent', timeout: Optional[float] = None):
+        super().__init__(timeout=timeout)
+        self.agent = agent
+
+    def save(self, path: str) -> str:
+        r"""Saves the agent's current memory to a JSON file.
+
+        Args:
+            path (str): The file path to save the memory to.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.save_memory(path)
+        return f"Memory saved to {path}"
+
+    def load(self, memory_json: str) -> str:
+        r"""Loads memory into the agent from a JSON string.
+
+        Args:
+            memory_json (str): A JSON string containing memory records.
+
+        Returns:
+            str: Confirmation or error message.
+        """
+        try:
+            data = json.loads(memory_json.strip())
+            if not isinstance(data, list):
+                return "[ERROR] Memory data should be a list of records."
+
+            # Build a fresh ChatHistoryMemory
+            context_creator = ScoreBasedContextCreator(
+                token_counter=self.agent.model_backend.token_counter,
+                token_limit=self.agent.model_backend.token_limit,
+            )
+            new_memory = ChatHistoryMemory(context_creator)
+
+            # Convert each record dict -> MemoryRecord
+            for record_dict in data:
+                record = MemoryRecord.from_dict(record_dict)
+                new_memory.write_record(record)
+
+            # Load into the agent
+            self.agent.load_memory(new_memory)
+            return "Loaded memory from provided JSON string."
+        except json.JSONDecodeError:
+            return "[ERROR] Invalid JSON string provided."
+        except Exception as e:
+            return f"[ERROR] Failed to load memory: {e!s}"
+
+    def load_from_path(self, path: str) -> str:
+        r"""Loads the agent's memory from a JSON file.
+
+        Args:
+            path (str): The file path to load the memory from.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.load_memory_from_path(path)
+        return f"Memory loaded from {path}"
+
+    def clear_memory(self) -> str:
+        r"""Clears the agent's memory.
+
+        Returns:
+            str: Confirmation message.
+        """
+        self.agent.clear_memory()
+        return "Memory has been cleared."
+
+    def get_tools(self) -> list[FunctionTool]:
+        r"""Expose the memory management methods as function tools
+        for the ChatAgent.
+
+        Returns:
+            list[FunctionTool]: List of FunctionTool objects.
+        """
+        return [
+            FunctionTool(self.save),
+            FunctionTool(self.load),
+            FunctionTool(self.load_from_path),
+            FunctionTool(self.clear_memory),
+        ]

camel/utils/chunker/__init__.py

@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .base import BaseChunker
+from .code_chunker import CodeChunker
+from .uio_chunker import UnstructuredIOChunker
+
+__all__ = [
+    "BaseChunker",
+    "CodeChunker",
+    "UnstructuredIOChunker",
+]

camel/utils/chunker/base.py

@@ -0,0 +1,24 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseChunker(ABC):
+    r"""An abstract base class for all CAMEL chunkers."""
+
+    @abstractmethod
+    def chunk(self, content: Any) -> Any:
+        r"""Chunk the given content"""
+        pass

camel/utils/chunker/code_chunker.py

@@ -0,0 +1,193 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import re
+from typing import List, Optional
+
+from unstructured.documents.elements import Element, ElementMetadata
+
+from camel.messages import OpenAIUserMessage
+from camel.types import ModelType
+from camel.utils import BaseTokenCounter, OpenAITokenCounter
+
+from .base import BaseChunker
+
+
+class CodeChunker(BaseChunker):
+    r"""A class for chunking code or text while respecting structure
+        and token limits.
+
+        This class ensures that structured elements such as functions,
+        classes, and regions are not arbitrarily split across chunks.
+        It also handles oversized lines and Base64-encoded images.
+
+    Attributes:
+        chunk_size (int, optional): The maximum token size per chunk.
+            (default: :obj:`8192`)
+        token_counter (BaseTokenCounter, optional): The tokenizer used for
+            token counting, if `None`, OpenAITokenCounter will be used.
+            (default: :obj:`None`)
+        remove_image: (bool, optional): If the chunker should skip the images.
+    """
+
+    def __init__(
+        self,
+        chunk_size: int = 8192,
+        token_counter: Optional[BaseTokenCounter] = None,
+        remove_image: Optional[bool] = True,
+    ):
+        self.chunk_size = chunk_size
+        self.token_counter = (
+            token_counter
+            if token_counter
+            else OpenAITokenCounter(model=ModelType.GPT_4O_MINI)
+        )
+        self.remove_image = remove_image
+        self.struct_pattern = re.compile(
+            r'^\s*(?:(def|class|function)\s+\w+|'
+            r'(public|private|protected)\s+[\w<>]+\s+\w+\s*\(|'
+            r'\b(interface|enum|namespace)\s+\w+|'
+            r'#\s*(region|endregion)\b)'
+        )
+        self.image_pattern = re.compile(
+            r'!\[.*?\]\((?:data:image/[^;]+;base64,[a-zA-Z0-9+/]+=*|[^)]+)\)'
+        )
+
+    def count_tokens(self, text: str):
+        r"""Counts the number of tokens in the given text.
+
+        Args:
+            text (str): The input text to be tokenized.
+
+        Returns:
+            int: The number of tokens in the input text.
+        """
+        return self.token_counter.count_tokens_from_messages(
+            [OpenAIUserMessage(role="user", name="user", content=text)]
+        )
+
+    def _split_oversized(self, line: str) -> List[str]:
+        r"""Splits an oversized line into multiple chunks based on token limits
+
+        Args:
+            line (str): The oversized line to be split.
+
+        Returns:
+            List[str]: A list of smaller chunks after splitting the
+                oversized line.
+        """
+        tokens = self.token_counter.encode(line)
+        chunks = []
+        buffer = []
+        current_count = 0
+
+        for token in tokens:
+            buffer.append(token)
+            current_count += 1
+
+            if current_count >= self.chunk_size:
+                chunks.append(self.token_counter.decode(buffer).strip())
+                buffer = []
+                current_count = 0
+
+        if buffer:
+            chunks.append(self.token_counter.decode(buffer))
+        return chunks
+
+    def chunk(self, content: List[str]) -> List[Element]:
+        r"""Splits the content into smaller chunks while preserving
+        structure and adhering to token constraints.
+
+        Args:
+            content (List[str]): The content to be chunked.
+
+        Returns:
+            List[str]: A list of chunked text segments.
+        """
+        content_str = "\n".join(map(str, content))
+        chunks = []
+        current_chunk: list[str] = []
+        current_tokens = 0
+        struct_buffer: list[str] = []
+        struct_tokens = 0
+
+        for line in content_str.splitlines(keepends=True):
+            if self.remove_image:
+                if self.image_pattern.match(line):
+                    continue
+
+            line_tokens = self.count_tokens(line)
+
+            if line_tokens > self.chunk_size:
+                if current_chunk:
+                    chunks.append("".join(current_chunk))
+                    current_chunk = []
+                    current_tokens = 0
+                chunks.extend(self._split_oversized(line))
+                continue
+
+            if self.struct_pattern.match(line):
+                if struct_buffer:
+                    if current_tokens + struct_tokens <= self.chunk_size:
+                        current_chunk.extend(struct_buffer)
+                        current_tokens += struct_tokens
+                    else:
+                        if current_chunk:
+                            chunks.append("".join(current_chunk))
+                        current_chunk = struct_buffer.copy()
+                        current_tokens = struct_tokens
+                    struct_buffer = []
+                    struct_tokens = 0
+
+                struct_buffer.append(line)
+                struct_tokens += line_tokens
+            else:
+                if struct_buffer:
+                    struct_buffer.append(line)
+                    struct_tokens += line_tokens
+                else:
+                    if current_tokens + line_tokens > self.chunk_size:
+                        chunks.append("".join(current_chunk))
+                        current_chunk = [line]
+                        current_tokens = line_tokens
+                    else:
+                        current_chunk.append(line)
+                        current_tokens += line_tokens
+
+        if struct_buffer:
+            if current_tokens + struct_tokens <= self.chunk_size:
+                current_chunk.extend(struct_buffer)
+            else:
+                if current_chunk:
+                    chunks.append("".join(current_chunk))
+                current_chunk = struct_buffer
+
+        if current_chunk:
+            chunks.append("".join(current_chunk))
+
+        final_chunks = []
+        for chunk in chunks:
+            chunk_token = self.count_tokens(chunk)
+            if chunk_token > self.chunk_size:
+                final_chunks.extend(self._split_oversized(chunk))
+            else:
+                final_chunks.append(chunk)
+
+        # TODO: need to reconsider how to correctly form metadata (maybe need
+        # to decouple the connection with unstructuredIO)
+        chunked_elements = []
+        for chunk in final_chunks:
+            element = Element(metadata=ElementMetadata())
+            element.text = chunk
+            chunked_elements.append(element)
+        return chunked_elements

camel/utils/chunker/uio_chunker.py

@@ -0,0 +1,66 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import List, Optional
+
+from unstructured.documents.elements import Element
+
+from camel.loaders import UnstructuredIO
+from camel.utils.chunker import BaseChunker
+
+
+class UnstructuredIOChunker(BaseChunker):
+    r"""A class for chunking text while respecting structure and
+    character limits.
+
+    This class ensures that structured elements, such as document sections
+    and titles, are not arbitrarily split across chunks. It utilizes the
+    `UnstructuredIO` class to process and segment elements while maintaining
+    readability and coherence. The chunking method can be adjusted based on
+    the provided `chunk_type` parameter.
+
+    Args:
+        chunk_type (str, optional): The method used for chunking text.
+            (default: :obj:`"chunk_by_title"`)
+        max_characters (int, optional): The maximum number of characters
+            allowed per chunk. (default: :obj:`500`)
+        metadata_filename (Optional[str], optional): An optional filename
+            for storing metadata related to chunking. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        chunk_type: str = "chunk_by_title",
+        max_characters: int = 500,
+        metadata_filename: Optional[str] = None,
+    ):
+        self.uio = UnstructuredIO()
+        self.chunk_type = chunk_type
+        self.max_characters = max_characters
+        self.metadata_filename = metadata_filename
+
+    def chunk(self, content: List[Element]) -> List[Element]:
+        r"""Splits the content into smaller chunks while preserving
+        structure and adhering to token constraints.
+
+        Args:
+            content (List[Element]): The content to be chunked.
+
+        Returns:
+            List[Element]: A list of chunked text segments.
+        """
+        return self.uio.chunk_elements(
+            chunk_type=self.chunk_type,
+            elements=content,
+            max_characters=self.max_characters,
+        )

camel/utils/token_counting.py

@@ -90,6 +90,30 @@ class BaseTokenCounter(ABC):
         """
         pass
 
+    @abstractmethod
+    def encode(self, text: str) -> List[int]:
+        r"""Encode text into token IDs.
+
+        Args:
+            text (str): The text to encode.
+
+        Returns:
+            List[int]: List of token IDs.
+        """
+        pass
+
+    @abstractmethod
+    def decode(self, token_ids: List[int]) -> str:
+        r"""Decode token IDs back to text.
+
+        Args:
+            token_ids (List[int]): List of token IDs to decode.
+
+        Returns:
+            str: Decoded text.
+        """
+        pass
+
 
 class OpenAITokenCounter(BaseTokenCounter):
     def __init__(self, model: UnifiedModelType):
@@ -227,6 +251,28 @@ class OpenAITokenCounter(BaseTokenCounter):
         total = EXTRA_TOKENS + SQUARE_TOKENS * h * w
         return total
 
+    def encode(self, text: str) -> List[int]:
+        r"""Encode text into token IDs.
+
+        Args:
+            text (str): The text to encode.
+
+        Returns:
+            List[int]: List of token IDs.
+        """
+        return self.encoding.encode(text, disallowed_special=())
+
+    def decode(self, token_ids: List[int]) -> str:
+        r"""Decode token IDs back to text.
+
+        Args:
+            token_ids (List[int]): List of token IDs to decode.
+
+        Returns:
+            str: Decoded text.
+        """
+        return self.encoding.decode(token_ids)
+
 
 class AnthropicTokenCounter(BaseTokenCounter):
     @dependencies_required('anthropic')
@@ -266,6 +312,33 @@ class AnthropicTokenCounter(BaseTokenCounter):
             model=self.model,
         ).input_tokens
 
+    def encode(self, text: str) -> List[int]:
+        r"""Encode text into token IDs.
+
+        Args:
+            text (str): The text to encode.
+
+        Returns:
+            List[int]: List of token IDs.
+        """
+        raise NotImplementedError(
+            "The Anthropic API does not provide direct access to token IDs. "
+            "Use count_tokens_from_messages() for token counting instead."
+        )
+
+    def decode(self, token_ids: List[int]) -> str:
+        r"""Decode token IDs back to text.
+
+        Args:
+            token_ids (List[int]): List of token IDs to decode.
+
+        Returns:
+            str: Decoded text.
+        """
+        raise NotImplementedError(
+            "The Anthropic API does not provide functionality to decode token IDs."
+        )
+
 
 class LiteLLMTokenCounter(BaseTokenCounter):
     def __init__(self, model_type: UnifiedModelType):
@@ -319,6 +392,32 @@ class LiteLLMTokenCounter(BaseTokenCounter):
         """
         return self.completion_cost(completion_response=response)
 
+    def encode(self, text: str) -> List[int]:
+        r"""Encode text into token IDs.
+
+        Args:
+            text (str): The text to encode.
+
+        Returns:
+            List[int]: List of token IDs.
+        """
+        from litellm import encoding
+
+        return encoding.encode(text, disallowed_special=())
+
+    def decode(self, token_ids: List[int]) -> str:
+        r"""Decode token IDs back to text.
+
+        Args:
+            token_ids (List[int]): List of token IDs to decode.
+
+        Returns:
+            str: Decoded text.
+        """
+        from litellm import encoding
+
+        return encoding.decode(token_ids)
+
 
 class MistralTokenCounter(BaseTokenCounter):
     def __init__(self, model_type: ModelType):
@@ -390,3 +489,37 @@ class MistralTokenCounter(BaseTokenCounter):
         )
 
         return mistral_request
+
+    def encode(self, text: str) -> List[int]:
+        r"""Encode text into token IDs.
+
+        Args:
+            text (str): The text to encode.
+
+        Returns:
+            List[int]: List of token IDs.
+        """
+        # Use the Mistral tokenizer to encode the text
+        return self.tokenizer.encode_chat_completion(
+            ChatCompletionRequest(
+                model=self.model_type,
+                messages=[
+                    {
+                        "role": "user",
+                        "content": text,
+                    }
+                ],
+            )
+        )
+
+    def decode(self, token_ids: List[int]) -> str:
+        r"""Decode token IDs back to text.
+
+        Args:
+            token_ids (List[int]): List of token IDs to decode.
+
+        Returns:
+            str: Decoded text.
+        """
+        # Use the Mistral tokenizer to decode the tokens
+        return self.tokenizer.decode(token_ids)

{camel_ai-0.2.34.dist-info → camel_ai-0.2.35.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: camel-ai
-Version: 0.2.34
+Version: 0.2.35
 Summary: Communicative Agents for AI Society Study
 Project-URL: Homepage, https://www.camel-ai.org/
 Project-URL: Repository, https://github.com/camel-ai/camel