camel-ai 0.2.34__py3-none-any.whl → 0.2.35__py3-none-any.whl

camel/datasets/base_generator.py

@@ -13,10 +13,14 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import abc
+import asyncio
 import json
 import random
 from pathlib import Path
-from typing import List, Union
+from typing import Any, Dict, List, Union
+
+from pydantic import ValidationError
+from torch.utils.data import IterableDataset
 
 from camel.logger import get_logger
 
@@ -25,23 +29,47 @@ from .models import DataPoint
 logger = get_logger(__name__)
 
 
-class BaseGenerator(abc.ABC):
+class BaseGenerator(abc.ABC, IterableDataset):
     r"""Abstract base class for data generators.
 
     This class defines the interface for generating synthetic datapoints.
     Concrete implementations should provide specific generation strategies.
     """
 
-    def __init__(self, seed: int = 42, **kwargs):
+    def __init__(
+        self,
+        seed: int = 42,
+        cache: Union[str, Path, None] = None,
+        data_path: Union[str, Path, None] = None,
+        **kwargs,
+    ):
         r"""Initialize the base generator.
 
         Args:
             seed (int): Random seed for reproducibility. (default: :obj:`42`)
+            cache (Union[str, Path, None]): Optional path to save generated
+                datapoints during iteration. If None is provided, datapoints
+                will be discarded every 100 generations.
+            data_path (Union[str, Path, None]): Optional path to a JSONL file
+                to initialize the dataset from.
             **kwargs: Additional generator parameters.
         """
         self._rng = random.Random(seed)
+        self.cache = Path(cache) if cache else None
 
         self._data: List[DataPoint] = []
+        self._batch_to_save: List[DataPoint] = []
+
+        if data_path:
+            file_path = Path(data_path)
+            raw_data = self._init_from_jsonl(file_path)
+            try:
+                data_points = [DataPoint(**item) for item in raw_data]
+                self._data.extend(data_points)
+            except ValidationError as e:
+                raise ValueError(
+                    f"Failed to create DataPoint from JSONL data: {e}"
+                )
 
     @abc.abstractmethod
     async def generate_new(self, n: int, **kwargs) -> List[DataPoint]:
@@ -56,34 +84,112 @@ class BaseGenerator(abc.ABC):
         """
         pass
 
-    def __len__(self) -> int:
-        r"""Return the size of the generated dataset."""
-        return len(self._data)
+    def __aiter__(self):
+        r"""Async iterator that yields datapoints dynamically.
 
-    def __getitem__(self, idx: int) -> DataPoint:
-        r"""Retrieve a datapoint by index.
+        If a `data_path` was provided during initialization, those datapoints
+        are yielded first. When self._data is empty, 20 new datapoints
+        are generated. Every 100 yields, the batch is appended to the
+        JSONL file or discarded if `cache` is None.
 
-        Args:
-            idx (int): Index of the datapoint.
+        Yields:
+            DataPoint: A single datapoint.
+        """
 
-        Returns:
-            DataPoint: The datapoint corresponding to the given index.
+        async def generator():
+            while True:
+                if not self._data:
+                    new_datapoints = await self.generate_new(20)
+                    self._data.extend(new_datapoints)
+                datapoint = self._data.pop(0)
+                yield datapoint
+                self._batch_to_save.append(datapoint)
+                if len(self._batch_to_save) == 100:
+                    if self.cache:
+                        with self.cache.open("a", encoding="utf-8") as f:
+                            for dp in self._batch_to_save:
+                                json.dump(dp.to_dict(), f, ensure_ascii=False)
+                                f.write("\n")
+                    self._batch_to_save = []
 
-        Raises:
-            IndexError: If idx is out of bounds.
+        return generator()
+
+    def __iter__(self):
+        r"""Synchronous iterator for PyTorch IterableDataset compatibility.
+
+        If a `data_path` was provided during initialization, those datapoints
+        are yielded first. When self._data is empty, 20 new datapoints
+        are generated. Every 100 yields, the batch is appended to the
+        JSONL file or discarded if `cache` is None.
+
+        Yields:
+            DataPoint: A single datapoint.
         """
-        if idx < 0 or idx >= len(self._data):
-            raise IndexError(
-                f"Index {idx} out of bounds for dataset of "
-                f"size {len(self._data)}"
-            )
-        return self._data[idx]
+        try:
+            if asyncio.get_event_loop().is_running():
+                raise RuntimeError(
+                    "Cannot use synchronous iteration (__iter__) in an async "
+                    "context; use 'async for' with __aiter__ instead"
+                )
+        except RuntimeError as e:
+            if "no running event loop" not in str(e):
+                raise
+
+        while True:
+            if not self._data:
+                new_datapoints = asyncio.run(self.generate_new(20))
+                self._data.extend(new_datapoints)
+            datapoint = self._data.pop(0)
+            yield datapoint
+            self._batch_to_save.append(datapoint)
+            if len(self._batch_to_save) == 100:
+                if self.cache:
+                    with self.cache.open("a", encoding="utf-8") as f:
+                        for dp in self._batch_to_save:
+                            json.dump(dp.to_dict(), f, ensure_ascii=False)
+                            f.write("\n")
+                self._batch_to_save = []
 
     def sample(self) -> DataPoint:
-        if len(self._data) == 0:
-            raise RuntimeError("Dataset is empty, cannot sample.")
-        idx = self._rng.randint(0, len(self._data) - 1)
-        return self[idx]
+        r"""Returns the next datapoint from the current dataset
+        synchronously.
+
+        Raises:
+            RuntimeError: If called in an async context.
+
+        Returns:
+            DataPoint: The next DataPoint.
+
+        Note:
+            This method is intended for synchronous contexts.
+            Use 'async_sample' in asynchronous contexts to
+            avoid blocking or runtime errors.
+        """
+        try:
+            if asyncio.get_event_loop().is_running():
+                raise RuntimeError(
+                    "Cannot use synchronous sampling (sample) "
+                    "in an async context; use async_sample instead"
+                )
+        except RuntimeError as e:
+            if "no running event loop" not in str(e):
+                raise
+
+        return next(iter(self))
+
+    async def async_sample(self) -> DataPoint:
+        r"""Returns the next datapoint from the current dataset asynchronously.
+
+        Returns:
+            DataPoint: The next datapoint.
+
+        Note:
+            This method is intended for asynchronous contexts. Use 'sample'
+            in synchronous contexts.
+        """
+
+        async_iter = self.__aiter__()
+        return await async_iter.__anext__()
 
     def save_to_jsonl(self, file_path: Union[str, Path]) -> None:
         r"""Saves the generated datapoints to a JSONL (JSON Lines) file.
@@ -99,7 +205,7 @@ class BaseGenerator(abc.ABC):
 
         Notes:
             - Uses `self._data`, which contains the generated datapoints.
-            - Overwrites the file if it already exists.
+            - Appends to the file if it already exists.
             - Ensures compatibility with large datasets by using JSONL format.
         """
         if not self._data:
@@ -108,11 +214,66 @@ class BaseGenerator(abc.ABC):
         file_path = Path(file_path)
 
         try:
-            with file_path.open("w", encoding="utf-8") as f:
+            with file_path.open("a", encoding="utf-8") as f:
                 for datapoint in self._data:
                     json.dump(datapoint.to_dict(), f, ensure_ascii=False)
-                    f.write("\n")  # Ensure each entry is on a new line
+                    f.write("\n")
             logger.info(f"Dataset saved successfully to {file_path}")
         except IOError as e:
             logger.error(f"Error writing to file {file_path}: {e}")
             raise
+
+    def flush(self, file_path: Union[str, Path]) -> None:
+        r"""Flush the current data to a JSONL file and clear the data.
+
+        Args:
+            file_path (Union[str, Path]): Path to save the JSONL file.
+
+        Notes:
+            - Uses `save_to_jsonl` to save `self._data`.
+        """
+
+        self.save_to_jsonl(file_path)
+        self._data = []
+        logger.info(f"Data flushed to {file_path} and cleared from the memory")
+
+    def _init_from_jsonl(self, file_path: Path) -> List[Dict[str, Any]]:
+        r"""Load and parse a dataset from a JSONL file.
+
+        Args:
+            file_path (Path): Path to the JSONL file.
+
+        Returns:
+            List[Dict[str, Any]]: A list of datapoint dictionaries.
+
+        Raises:
+            FileNotFoundError: If the specified JSONL file does not exist.
+            ValueError: If a line contains invalid JSON or is not a dictionary.
+        """
+        if not file_path.exists():
+            raise FileNotFoundError(f"JSONL file not found: {file_path}")
+
+        raw_data = []
+        logger.debug(f"Loading JSONL from {file_path}")
+        with file_path.open('r', encoding='utf-8') as f:
+            for line_number, line in enumerate(f, start=1):
+                line = line.strip()
+                if not line:
+                    continue  # Skip blank lines
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError as e:
+                    raise ValueError(
+                        f"Invalid JSON on line {line_number} "
+                        f"in file {file_path}: {e}"
+                    )
+                if not isinstance(record, dict):
+                    raise ValueError(
+                        f"Expected a dictionary at line {line_number}, "
+                        f"got {type(record).__name__}"
+                    )
+                raw_data.append(record)
+        logger.info(
+            f"Successfully loaded {len(raw_data)} items from {file_path}"
+        )
+        return raw_data

camel/memories/agent_memories.py

@@ -33,6 +33,8 @@ class ChatHistoryMemory(AgentMemory):
         window_size (int, optional): The number of recent chat messages to
             retrieve. If not provided, the entire chat history will be
             retrieved.  (default: :obj:`None`)
+        agent_id (str, optional): The ID of the agent associated with the chat
+            history.
     """
 
     def __init__(
@@ -40,6 +42,7 @@ class ChatHistoryMemory(AgentMemory):
         context_creator: BaseContextCreator,
         storage: Optional[BaseKeyValueStorage] = None,
         window_size: Optional[int] = None,
+        agent_id: Optional[str] = None,
     ) -> None:
         if window_size is not None and not isinstance(window_size, int):
             raise TypeError("`window_size` must be an integer or None.")
@@ -48,6 +51,15 @@ class ChatHistoryMemory(AgentMemory):
         self._context_creator = context_creator
         self._window_size = window_size
         self._chat_history_block = ChatHistoryBlock(storage=storage)
+        self._agent_id = agent_id
+
+    @property
+    def agent_id(self) -> Optional[str]:
+        return self._agent_id
+
+    @agent_id.setter
+    def agent_id(self, val: Optional[str]) -> None:
+        self._agent_id = val
 
     def retrieve(self) -> List[ContextRecord]:
         records = self._chat_history_block.retrieve(self._window_size)
@@ -63,6 +75,10 @@ class ChatHistoryMemory(AgentMemory):
         return records
 
     def write_records(self, records: List[MemoryRecord]) -> None:
+        for record in records:
+            # assign the agent_id to the record
+            if record.agent_id == "" and self.agent_id is not None:
+                record.agent_id = self.agent_id
         self._chat_history_block.write_records(records)
 
     def get_context_creator(self) -> BaseContextCreator:
@@ -84,6 +100,8 @@ class VectorDBMemory(AgentMemory):
             (default: :obj:`None`)
         retrieve_limit (int, optional): The maximum number of messages
             to be added into the context.  (default: :obj:`3`)
+        agent_id (str, optional): The ID of the agent associated with
+            the messages stored in the vector database.
     """
 
     def __init__(
@@ -91,13 +109,23 @@ class VectorDBMemory(AgentMemory):
         context_creator: BaseContextCreator,
         storage: Optional[BaseVectorStorage] = None,
         retrieve_limit: int = 3,
+        agent_id: Optional[str] = None,
     ) -> None:
         self._context_creator = context_creator
         self._retrieve_limit = retrieve_limit
         self._vectordb_block = VectorDBBlock(storage=storage)
+        self._agent_id = agent_id
 
         self._current_topic: str = ""
 
+    @property
+    def agent_id(self) -> Optional[str]:
+        return self._agent_id
+
+    @agent_id.setter
+    def agent_id(self, val: Optional[str]) -> None:
+        self._agent_id = val
+
     def retrieve(self) -> List[ContextRecord]:
         return self._vectordb_block.retrieve(
             self._current_topic,
@@ -109,6 +137,11 @@ class VectorDBMemory(AgentMemory):
         for record in records:
             if record.role_at_backend == OpenAIBackendRole.USER:
                 self._current_topic = record.message.content
+
+            # assign the agent_id to the record
+            if record.agent_id == "" and self.agent_id is not None:
+                record.agent_id = self.agent_id
+
         self._vectordb_block.write_records(records)
 
     def get_context_creator(self) -> BaseContextCreator:
@@ -133,6 +166,8 @@ class LongtermAgentMemory(AgentMemory):
             (default: :obj:`None`)
         retrieve_limit (int, optional): The maximum number of messages
             to be added into the context.  (default: :obj:`3`)
+        agent_id (str, optional): The ID of the agent associated with the chat
+            history and the messages stored in the vector database.
     """
 
     def __init__(
@@ -141,12 +176,22 @@ class LongtermAgentMemory(AgentMemory):
         chat_history_block: Optional[ChatHistoryBlock] = None,
         vector_db_block: Optional[VectorDBBlock] = None,
         retrieve_limit: int = 3,
+        agent_id: Optional[str] = None,
     ) -> None:
         self.chat_history_block = chat_history_block or ChatHistoryBlock()
         self.vector_db_block = vector_db_block or VectorDBBlock()
         self.retrieve_limit = retrieve_limit
         self._context_creator = context_creator
         self._current_topic: str = ""
+        self._agent_id = agent_id
+
+    @property
+    def agent_id(self) -> Optional[str]:
+        return self._agent_id
+
+    @agent_id.setter
+    def agent_id(self, val: Optional[str]) -> None:
+        self._agent_id = val
 
     def get_context_creator(self) -> BaseContextCreator:
         r"""Returns the context creator used by the memory.
@@ -166,7 +211,8 @@ class LongtermAgentMemory(AgentMemory):
         """
         chat_history = self.chat_history_block.retrieve()
         vector_db_retrieve = self.vector_db_block.retrieve(
-            self._current_topic, self.retrieve_limit
+            self._current_topic,
+            self.retrieve_limit,
         )
         return chat_history[:1] + vector_db_retrieve + chat_history[1:]
 

camel/memories/base.py

@@ -13,7 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 from abc import ABC, abstractmethod
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 from camel.memories.records import ContextRecord, MemoryRecord
 from camel.messages import OpenAIMessage
@@ -112,6 +112,16 @@ class AgentMemory(MemoryBlock, ABC):
     the memory records stored within the AgentMemory.
     """
 
+    @property
+    @abstractmethod
+    def agent_id(self) -> Optional[str]:
+        pass
+
+    @agent_id.setter
+    @abstractmethod
+    def agent_id(self, val: Optional[str]) -> None:
+        pass
+
     @abstractmethod
     def retrieve(self) -> List[ContextRecord]:
         r"""Get a record list from the memory for creating model context.
@@ -138,3 +148,15 @@ class AgentMemory(MemoryBlock, ABC):
                 context in OpenAIMessage format and the total token count.
         """
         return self.get_context_creator().create_context(self.retrieve())
+
+    def __repr__(self) -> str:
+        r"""Returns a string representation of the AgentMemory.
+
+        Returns:
+            str: A string in the format 'ClassName(agent_id=<id>)'
+                if agent_id exists, otherwise just 'ClassName()'.
+        """
+        agent_id = getattr(self, '_agent_id', None)
+        if agent_id:
+            return f"{self.__class__.__name__}(agent_id='{agent_id}')"
+        return f"{self.__class__.__name__}()"

camel/memories/records.py

@@ -39,6 +39,8 @@ class MemoryRecord(BaseModel):
             key-value pairs that provide more information. If not given, it
             will be an empty `Dict`.
         timestamp (float, optional): The timestamp when the record was created.
+        agent_id (str): The identifier of the agent associated with this
+            memory.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -50,6 +52,7 @@ class MemoryRecord(BaseModel):
     timestamp: float = Field(
         default_factory=lambda: datetime.now(timezone.utc).timestamp()
     )
+    agent_id: str = Field(default="")
 
     _MESSAGE_TYPES: ClassVar[dict] = {
         "BaseMessage": BaseMessage,
@@ -73,6 +76,7 @@ class MemoryRecord(BaseModel):
             role_at_backend=record_dict["role_at_backend"],
             extra_info=record_dict["extra_info"],
             timestamp=record_dict["timestamp"],
+            agent_id=record_dict["agent_id"],
         )
 
     def to_dict(self) -> Dict[str, Any]:
@@ -88,6 +92,7 @@ class MemoryRecord(BaseModel):
             "role_at_backend": self.role_at_backend,
             "extra_info": self.extra_info,
             "timestamp": self.timestamp,
+            "agent_id": self.agent_id,
         }
 
     def to_openai_message(self) -> OpenAIMessage:

camel/models/stub_model.py

@@ -44,6 +44,31 @@ class StubTokenCounter(BaseTokenCounter):
         """
         return 10
 
+    def encode(self, text: str) -> List[int]:
+        r"""Encode text into token IDs for STUB models.
+
+        Args:
+            text (str): The text to encode.
+
+        Returns:
+            List[int]: List of token IDs.
+        """
+        # For stub models, just return a list of 0s with length proportional
+        # to text length
+        return [0] * (len(text) // 4 + 1)  # Simple approximation
+
+    def decode(self, token_ids: List[int]) -> str:
+        r"""Decode token IDs back to text for STUB models.
+
+        Args:
+            token_ids (List[int]): List of token IDs to decode.
+
+        Returns:
+            str: Decoded text.
+        """
+        # For stub models, return a placeholder string
+        return "[Stub decoded text]"
+
 
 class StubModel(BaseModelBackend):
     r"""A dummy model used for unit tests."""

camel/retrievers/vector_retriever.py

@@ -27,6 +27,7 @@ from camel.storages import (
     VectorRecord,
 )
 from camel.utils import Constants
+from camel.utils.chunker import BaseChunker, UnstructuredIOChunker
 
 if TYPE_CHECKING:
     from unstructured.documents.elements import Element
@@ -78,6 +79,7 @@ class VectorRetriever(BaseRetriever):
         should_chunk: bool = True,
         extra_info: Optional[dict] = None,
         metadata_filename: Optional[str] = None,
+        chunker: Optional[BaseChunker] = None,
         **kwargs: Any,
     ) -> None:
         r"""Processes content from local file path, remote URL, string
@@ -101,6 +103,12 @@ class VectorRetriever(BaseRetriever):
                 used for storing metadata. Defaults to None.
             **kwargs (Any): Additional keyword arguments for content parsing.
         """
+        if chunker is None:
+            chunker = UnstructuredIOChunker(
+                chunk_type=chunk_type,
+                max_characters=max_characters,
+                metadata_filename=metadata_filename,
+            )
         from unstructured.documents.elements import Element
 
         if isinstance(content, Element):
@@ -140,13 +148,7 @@ class VectorRetriever(BaseRetriever):
         else:
             # Chunk the content if required
             chunks = (
-                self.uio.chunk_elements(
-                    chunk_type=chunk_type,
-                    elements=elements,
-                    max_characters=max_characters,
-                )
-                if should_chunk
-                else elements
+                chunker.chunk(content=elements) if should_chunk else (elements)
             )
 
             # Process chunks in batches and store embeddings
@@ -157,6 +159,7 @@ class VectorRetriever(BaseRetriever):
                 )
 
                 records = []
+                offset = 0
                 # Prepare the payload for each vector record, includes the
                 # content path, chunk metadata, and chunk text
                 for vector, chunk in zip(batch_vectors, batch_chunks):
@@ -178,6 +181,7 @@ class VectorRetriever(BaseRetriever):
                     chunk_metadata["metadata"].pop("orig_elements", "")
                     chunk_metadata["extra_info"] = extra_info or {}
                     chunk_text = {"text": str(chunk)}
+                    chunk_metadata["metadata"]["piece_num"] = i + offset + 1
                     combined_dict = {
                         **content_path_info,
                         **chunk_metadata,
@@ -187,6 +191,7 @@ class VectorRetriever(BaseRetriever):
                     records.append(
                         VectorRecord(vector=vector, payload=combined_dict)
                     )
+                    offset += 1
 
                 self.storage.add(records=records)
 

camel/storages/key_value_storages/__init__.py

@@ -14,7 +14,7 @@
 
 from .base import BaseKeyValueStorage
 from .in_memory import InMemoryKeyValueStorage
-from .json import JsonStorage
+from .json import CamelJSONEncoder, JsonStorage
 from .redis import RedisStorage
 
 __all__ = [
@@ -22,4 +22,5 @@ __all__ = [
     'InMemoryKeyValueStorage',
     'JsonStorage',
     'RedisStorage',
+    'CamelJSONEncoder',
 ]

camel/storages/key_value_storages/json.py

@@ -26,7 +26,7 @@ from camel.types import (
 )
 
 
-class _CamelJSONEncoder(json.JSONEncoder):
+class CamelJSONEncoder(json.JSONEncoder):
     r"""A custom JSON encoder for serializing specifically enumerated types.
     Ensures enumerated types can be stored in and retrieved from JSON format.
     """
@@ -62,7 +62,7 @@ class JsonStorage(BaseKeyValueStorage):
     def _json_object_hook(self, d) -> Any:
         if "__enum__" in d:
             name, member = d["__enum__"].split(".")
-            return getattr(_CamelJSONEncoder.CAMEL_ENUMS[name], member)
+            return getattr(CamelJSONEncoder.CAMEL_ENUMS[name], member)
         else:
             return d
 
@@ -75,11 +75,7 @@ class JsonStorage(BaseKeyValueStorage):
         """
         with self.json_path.open("a") as f:
             f.writelines(
-                [
-                    json.dumps(r, cls=_CamelJSONEncoder, ensure_ascii=False)
-                    + "\n"
-                    for r in records
-                ]
+                [json.dumps(r, cls=CamelJSONEncoder) + "\n" for r in records]
             )
 
     def load(self) -> List[Dict[str, Any]]:

camel/storages/vectordb_storages/base.py

@@ -87,7 +87,11 @@ class VectorDBQueryResult(BaseModel):
     ) -> "VectorDBQueryResult":
         r"""A class method to construct a `VectorDBQueryResult` instance."""
         return cls(
-            record=VectorRecord(vector=vector, id=id, payload=payload),
+            record=VectorRecord(
+                vector=vector,
+                id=id,
+                payload=payload,
+            ),
             similarity=similarity,
         )
 

camel/toolkits/__init__.py

@@ -50,6 +50,7 @@ from .semantic_scholar_toolkit import SemanticScholarToolkit
 from .zapier_toolkit import ZapierToolkit
 from .sympy_toolkit import SymPyToolkit
 from .mineru_toolkit import MinerUToolkit
+from .memory_toolkit import MemoryToolkit
 from .audio_analysis_toolkit import AudioAnalysisToolkit
 from .excel_toolkit import ExcelToolkit
 from .video_analysis_toolkit import VideoAnalysisToolkit
@@ -60,7 +61,6 @@ from .file_write_toolkit import FileWriteToolkit
 from .terminal_toolkit import TerminalToolkit
 from .pubmed_toolkit import PubMedToolkit
 
-
 __all__ = [
     'BaseToolkit',
     'FunctionTool',
@@ -97,6 +97,7 @@ __all__ = [
     'ZapierToolkit',
     'SymPyToolkit',
     'MinerUToolkit',
+    'MemoryToolkit',
     'MCPToolkit',
     'MCPToolkitManager',
     'AudioAnalysisToolkit',