camel-ai 0.2.67__py3-none-any.whl → 0.2.80a2__py3-none-any.whl

camel/loaders/base_loader.py

@@ -0,0 +1,85 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+
+class BaseLoader(ABC):
+    r"""Abstract base class for all data loaders in CAMEL."""
+
+    @abstractmethod
+    def _load_single(self, source: Union[str, Path]) -> Dict[str, Any]:
+        r"""Load data from a single source.
+
+        Args:
+            source (Union[str, Path]): The data source to load from.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the loaded data. It is
+                recommended that the dictionary includes a "content" key with
+                the primary data and optional metadata keys.
+        """
+        pass
+
+    def load(
+        self,
+        source: Union[str, Path, List[Union[str, Path]]],
+    ) -> Dict[str, List[Dict[str, Any]]]:
+        r"""Load data from one or multiple sources.
+
+        Args:
+            source (Union[str, Path, List[Union[str, Path]]]): The data source
+                (s) to load from. Can be:
+                - A single path/URL (str or Path)
+                - A list of paths/URLs
+
+        Returns:
+            Dict[str, List[Dict[str, Any]]]: A dictionary with a single key
+                "contents" containing a list of loaded data. If a single source
+                is provided, the list will contain a single item.
+
+        Raises:
+            ValueError: If no sources are provided
+            Exception: If loading fails for any source
+        """
+        if not source:
+            raise ValueError("At least one source must be provided")
+
+        # Convert single source to list for uniform processing
+        sources = [source] if isinstance(source, (str, Path)) else list(source)
+
+        # Process all sources
+        results = []
+        for i, src in enumerate(sources, 1):
+            try:
+                content = self._load_single(src)
+                results.append(content)
+            except Exception as e:
+                raise RuntimeError(
+                    f"Error loading source {i}/{len(sources)}: {src}"
+                ) from e
+
+        return {"contents": results}
+
+    @property
+    @abstractmethod
+    def supported_formats(self) -> set[str]:
+        r"""Get the set of supported file formats or data sources.
+
+        Returns:
+            set[str]: A set of strings representing the supported formats/
+            sources.
+        """
+        pass

camel/loaders/chunkr_reader.py

@@ -34,6 +34,12 @@ class ChunkrReaderConfig:
         high_resolution (bool, optional): Whether to use high resolution OCR.
             (default: :obj:`True`)
         ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+        **kwargs: Additional keyword arguments to pass to the Chunkr
+            Configuration. This accepts all other Configuration parameters
+            such as expires_in, pipeline, segment_processing,
+            segmentation_strategy, etc.
+            See: https://github.com/lumina-ai-inc/chunkr/blob/main/core/src/
+            models/task.rs#L749
     """
 
     def __init__(
@@ -41,10 +47,12 @@ class ChunkrReaderConfig:
         chunk_processing: int = 512,
         high_resolution: bool = True,
         ocr_strategy: str = "Auto",
+        **kwargs,
     ):
         self.chunk_processing = chunk_processing
         self.high_resolution = high_resolution
         self.ocr_strategy = ocr_strategy
+        self.kwargs = kwargs
 
 
 class ChunkrReader:
@@ -190,4 +198,5 @@ class ChunkrReader:
                 "Auto": OcrStrategy.AUTO,
                 "All": OcrStrategy.ALL,
             }.get(chunkr_config.ocr_strategy, OcrStrategy.ALL),
+            **chunkr_config.kwargs,
         )

camel/loaders/firecrawl_reader.py

@@ -98,8 +98,8 @@ class Firecrawl:
     def scrape(
         self,
         url: str,
-        params: Optional[Dict[str, Any]] = None,
-    ) -> Dict:
+        params: Optional[Dict[str, str]] = None,
+    ) -> Dict[str, str]:
         r"""To scrape a single URL. This function supports advanced scraping
         by setting different parameters and returns the full scraped data as a
         dictionary.
@@ -108,11 +108,11 @@ class Firecrawl:
 
         Args:
             url (str): The URL to read.
-            params (Optional[Dict[str, Any]]): Additional parameters for the
+            params (Optional[Dict[str, str]]): Additional parameters for the
                 scrape request.
 
         Returns:
-            Dict: The scraped data.
+            Dict[str, str]: The scraped data.
 
         Raises:
             RuntimeError: If the scrape process fails.

camel/logger.py

@@ -155,7 +155,7 @@ def set_log_level(level):
     _logger.debug(f"Logging level set to: {logging.getLevelName(level)}")
 
 
-def get_logger(name):
+def get_logger(name: str):
     r"""Get a logger with the specified name, prefixed with 'camel.'.
 
 

camel/memories/agent_memories.py

@@ -51,7 +51,9 @@ class ChatHistoryMemory(AgentMemory):
             raise ValueError("`window_size` must be non-negative.")
         self._context_creator = context_creator
         self._window_size = window_size
-        self._chat_history_block = ChatHistoryBlock(storage=storage)
+        self._chat_history_block = ChatHistoryBlock(
+            storage=storage,
+        )
         self._agent_id = agent_id
 
     @property
@@ -88,6 +90,63 @@ class ChatHistoryMemory(AgentMemory):
     def clear(self) -> None:
         self._chat_history_block.clear()
 
+    def clean_tool_calls(self) -> None:
+        r"""Removes tool call messages from memory.
+        This method removes all FUNCTION/TOOL role messages and any ASSISTANT
+        messages that contain tool_calls in their meta_dict to save token
+        usage.
+        """
+        from camel.types import OpenAIBackendRole
+
+        # Get all messages from storage
+        record_dicts = self._chat_history_block.storage.load()
+        if not record_dicts:
+            return
+
+        # Track indices to remove (reverse order for efficient deletion)
+        indices_to_remove = []
+
+        # Identify indices of tool-related messages
+        for i, record in enumerate(record_dicts):
+            role = record.get('role_at_backend')
+
+            # Mark FUNCTION messages for removal
+            if role == OpenAIBackendRole.FUNCTION.value:
+                indices_to_remove.append(i)
+            # Mark TOOL messages for removal
+            elif role == OpenAIBackendRole.TOOL.value:
+                indices_to_remove.append(i)
+            # Mark ASSISTANT messages with tool_calls for removal
+            elif role == OpenAIBackendRole.ASSISTANT.value:
+                message_dict = record.get('message', {})
+                # Check for tool_calls in message
+                has_tool_calls = 'tool_calls' in message_dict
+                is_func_calling = (
+                    message_dict.get('__class__') == 'FunctionCallingMessage'
+                    and 'args' in message_dict
+                )
+
+                if has_tool_calls or is_func_calling:
+                    indices_to_remove.append(i)
+
+        # Remove records in-place
+        for i in reversed(indices_to_remove):
+            del record_dicts[i]
+
+        # Clear storage and save the modified records back
+        self._chat_history_block.storage.clear()
+        self._chat_history_block.storage.save(record_dicts)
+
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes the most recent records from chat history memory."""
+        return self._chat_history_block.pop_records(count)
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from chat history memory."""
+        return self._chat_history_block.remove_records_by_indices(indices)
+
 
 class VectorDBMemory(AgentMemory):
     r"""An agent memory wrapper of :obj:`VectorDBBlock`. This memory queries
@@ -152,6 +211,20 @@ class VectorDBMemory(AgentMemory):
         r"""Removes all records from the vector database memory."""
         self._vectordb_block.clear()
 
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Rolling back is unsupported for vector database memory."""
+        raise NotImplementedError(
+            "VectorDBMemory does not support removing historical records."
+        )
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removing by indices is unsupported for vector database memory."""
+        raise NotImplementedError(
+            "VectorDBMemory does not support removing records by indices."
+        )
+
 
 class LongtermAgentMemory(AgentMemory):
     r"""An implementation of the :obj:`AgentMemory` abstract base class for
@@ -236,3 +309,13 @@ class LongtermAgentMemory(AgentMemory):
         r"""Removes all records from the memory."""
         self.chat_history_block.clear()
         self.vector_db_block.clear()
+
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes recent chat history records while leaving vector memory."""
+        return self.chat_history_block.pop_records(count)
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from chat history."""
+        return self.chat_history_block.remove_records_by_indices(indices)

camel/memories/base.py

@@ -45,6 +45,32 @@ class MemoryBlock(ABC):
         """
         self.write_records([record])
 
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes records from the memory and returns the removed records.
+
+        Args:
+            count (int): Number of records to remove.
+
+        Returns:
+            List[MemoryRecord]: The records that were removed from the memory
+                in their original order.
+        """
+        raise NotImplementedError
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from the memory.
+
+        Args:
+            indices (List[int]): List of indices to remove. Indices should be
+                valid positions in the current record list.
+
+        Returns:
+            List[MemoryRecord]: The removed records in their original order.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def clear(self) -> None:
         r"""Clears all messages from the memory."""
@@ -149,6 +175,14 @@ class AgentMemory(MemoryBlock, ABC):
         """
         return self.get_context_creator().create_context(self.retrieve())
 
+    def clean_tool_calls(self) -> None:
+        r"""Removes tool call messages from memory.
+        This is an optional method that can be overridden by subclasses
+        to implement cleaning of tool-related messages. By default, it
+        does nothing, maintaining backward compatibility.
+        """
+        pass
+
     def __repr__(self) -> str:
         r"""Returns a string representation of the AgentMemory.
 

camel/memories/blocks/chat_history_block.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-import warnings
 from typing import List, Optional
 
 from camel.memories.base import MemoryBlock
@@ -39,7 +38,7 @@ class ChatHistoryBlock(MemoryBlock):
             last message is 1.0, and with each step taken backward, the score
             of the message is multiplied by the `keep_rate`. Higher `keep_rate`
             leads to high possibility to keep history messages during context
-            creation.
+            creation. (default: :obj:`0.9`)
     """
 
     def __init__(
@@ -70,7 +69,8 @@ class ChatHistoryBlock(MemoryBlock):
         """
         record_dicts = self.storage.load()
         if len(record_dicts) == 0:
-            warnings.warn("The `ChatHistoryMemory` is empty.")
+            # Empty memory is a valid state (e.g., during initialization).
+            # Users can check if memory is empty by checking the returned list.
             return list()
 
         if window_size is not None and window_size >= 0:
@@ -81,7 +81,10 @@ class ChatHistoryBlock(MemoryBlock):
                 if (
                     record_dicts
                     and record_dicts[0]['role_at_backend']
-                    in {OpenAIBackendRole.SYSTEM, OpenAIBackendRole.DEVELOPER}
+                    in {
+                        OpenAIBackendRole.SYSTEM.value,
+                        OpenAIBackendRole.DEVELOPER.value,
+                    }
                 )
                 else 0
             )
@@ -164,3 +167,118 @@ class ChatHistoryBlock(MemoryBlock):
     def clear(self) -> None:
         r"""Clears all chat messages from the memory."""
         self.storage.clear()
+
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes the most recent records from the memory.
+
+        Args:
+            count (int): Number of records to remove from the end of the
+                conversation history. A value of 0 results in no changes.
+
+        Returns:
+            List[MemoryRecord]: The removed records in chronological order.
+        """
+        if not isinstance(count, int):
+            raise TypeError("`count` must be an integer.")
+        if count < 0:
+            raise ValueError("`count` must be non-negative.")
+        if count == 0:
+            return []
+
+        record_dicts = self.storage.load()
+        if not record_dicts:
+            return []
+
+        # Preserve initial system/developer instruction if present.
+        protected_prefix = (
+            1
+            if (
+                record_dicts
+                and record_dicts[0]['role_at_backend']
+                in {
+                    OpenAIBackendRole.SYSTEM.value,
+                    OpenAIBackendRole.DEVELOPER.value,
+                }
+            )
+            else 0
+        )
+
+        removable_count = max(len(record_dicts) - protected_prefix, 0)
+        if removable_count == 0:
+            return []
+
+        pop_count = min(count, removable_count)
+        split_index = len(record_dicts) - pop_count
+
+        popped_dicts = record_dicts[split_index:]
+        remaining_dicts = record_dicts[:split_index]
+
+        self.storage.clear()
+        if remaining_dicts:
+            self.storage.save(remaining_dicts)
+
+        return [MemoryRecord.from_dict(record) for record in popped_dicts]
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from the memory.
+
+        Args:
+            indices (List[int]): List of indices to remove. Indices are
+                positions in the current record list (0-based).
+                System/developer messages at index 0 are protected and will
+                not be removed.
+
+        Returns:
+            List[MemoryRecord]: The removed records in their original order.
+        """
+        if not indices:
+            return []
+
+        record_dicts = self.storage.load()
+        if not record_dicts:
+            return []
+
+        # Preserve initial system/developer instruction if present.
+        protected_prefix = (
+            1
+            if (
+                record_dicts
+                and record_dicts[0]['role_at_backend']
+                in {
+                    OpenAIBackendRole.SYSTEM.value,
+                    OpenAIBackendRole.DEVELOPER.value,
+                }
+            )
+            else 0
+        )
+
+        # Filter out protected indices and invalid ones
+        valid_indices = sorted(
+            {
+                idx
+                for idx in indices
+                if idx >= protected_prefix and idx < len(record_dicts)
+            }
+        )
+
+        if not valid_indices:
+            return []
+
+        # Extract records to remove (in original order)
+        removed_records = [record_dicts[idx] for idx in valid_indices]
+
+        # Build remaining records by excluding removed indices
+        remaining_dicts = [
+            record
+            for idx, record in enumerate(record_dicts)
+            if idx not in valid_indices
+        ]
+
+        # Save back to storage
+        self.storage.clear()
+        if remaining_dicts:
+            self.storage.save(remaining_dicts)
+
+        return [MemoryRecord.from_dict(record) for record in removed_records]

camel/memories/blocks/vectordb_block.py

@@ -89,13 +89,20 @@ class VectorDBBlock(MemoryBlock):
             records (List[MemoryRecord]): Memory records to be added to the
                 memory.
         """
+        # Filter out records with empty message content
+        valid_records = [
+            record
+            for record in records
+            if record.message.content and record.message.content.strip()
+        ]
+
         v_records = [
             VectorRecord(
                 vector=self.embedding.embed(record.message.content),
                 payload=record.to_dict(),
                 id=str(record.uuid),
             )
-            for record in records
+            for record in valid_records
         ]
         self.storage.add(v_records)