camel-ai 0.2.34__py3-none-any.whl → 0.2.36__py3-none-any.whl

camel/datasets/base_generator.py

@@ -13,10 +13,14 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import abc
+import asyncio
 import json
 import random
 from pathlib import Path
-from typing import List, Union
+from typing import Any, Dict, List, Union
+
+from pydantic import ValidationError
+from torch.utils.data import IterableDataset
 
 from camel.logger import get_logger
 
@@ -25,23 +29,47 @@ from .models import DataPoint
 logger = get_logger(__name__)
 
 
-class BaseGenerator(abc.ABC):
+class BaseGenerator(abc.ABC, IterableDataset):
     r"""Abstract base class for data generators.
 
     This class defines the interface for generating synthetic datapoints.
     Concrete implementations should provide specific generation strategies.
     """
 
-    def __init__(self, seed: int = 42, **kwargs):
+    def __init__(
+        self,
+        seed: int = 42,
+        cache: Union[str, Path, None] = None,
+        data_path: Union[str, Path, None] = None,
+        **kwargs,
+    ):
         r"""Initialize the base generator.
 
         Args:
             seed (int): Random seed for reproducibility. (default: :obj:`42`)
+            cache (Union[str, Path, None]): Optional path to save generated
+                datapoints during iteration. If None is provided, datapoints
+                will be discarded every 100 generations.
+            data_path (Union[str, Path, None]): Optional path to a JSONL file
+                to initialize the dataset from.
             **kwargs: Additional generator parameters.
         """
         self._rng = random.Random(seed)
+        self.cache = Path(cache) if cache else None
 
         self._data: List[DataPoint] = []
+        self._batch_to_save: List[DataPoint] = []
+
+        if data_path:
+            file_path = Path(data_path)
+            raw_data = self._init_from_jsonl(file_path)
+            try:
+                data_points = [DataPoint(**item) for item in raw_data]
+                self._data.extend(data_points)
+            except ValidationError as e:
+                raise ValueError(
+                    f"Failed to create DataPoint from JSONL data: {e}"
+                )
 
     @abc.abstractmethod
     async def generate_new(self, n: int, **kwargs) -> List[DataPoint]:
@@ -56,34 +84,112 @@ class BaseGenerator(abc.ABC):
         """
         pass
 
-    def __len__(self) -> int:
-        r"""Return the size of the generated dataset."""
-        return len(self._data)
+    def __aiter__(self):
+        r"""Async iterator that yields datapoints dynamically.
 
-    def __getitem__(self, idx: int) -> DataPoint:
-        r"""Retrieve a datapoint by index.
+        If a `data_path` was provided during initialization, those datapoints
+        are yielded first. When self._data is empty, 20 new datapoints
+        are generated. Every 100 yields, the batch is appended to the
+        JSONL file or discarded if `cache` is None.
 
-        Args:
-            idx (int): Index of the datapoint.
+        Yields:
+            DataPoint: A single datapoint.
+        """
 
-        Returns:
-            DataPoint: The datapoint corresponding to the given index.
+        async def generator():
+            while True:
+                if not self._data:
+                    new_datapoints = await self.generate_new(20)
+                    self._data.extend(new_datapoints)
+                datapoint = self._data.pop(0)
+                yield datapoint
+                self._batch_to_save.append(datapoint)
+                if len(self._batch_to_save) == 100:
+                    if self.cache:
+                        with self.cache.open("a", encoding="utf-8") as f:
+                            for dp in self._batch_to_save:
+                                json.dump(dp.to_dict(), f, ensure_ascii=False)
+                                f.write("\n")
+                    self._batch_to_save = []
 
-        Raises:
-            IndexError: If idx is out of bounds.
+        return generator()
+
+    def __iter__(self):
+        r"""Synchronous iterator for PyTorch IterableDataset compatibility.
+
+        If a `data_path` was provided during initialization, those datapoints
+        are yielded first. When self._data is empty, 20 new datapoints
+        are generated. Every 100 yields, the batch is appended to the
+        JSONL file or discarded if `cache` is None.
+
+        Yields:
+            DataPoint: A single datapoint.
         """
-        if idx < 0 or idx >= len(self._data):
-            raise IndexError(
-                f"Index {idx} out of bounds for dataset of "
-                f"size {len(self._data)}"
-            )
-        return self._data[idx]
+        try:
+            if asyncio.get_event_loop().is_running():
+                raise RuntimeError(
+                    "Cannot use synchronous iteration (__iter__) in an async "
+                    "context; use 'async for' with __aiter__ instead"
+                )
+        except RuntimeError as e:
+            if "no running event loop" not in str(e):
+                raise
+
+        while True:
+            if not self._data:
+                new_datapoints = asyncio.run(self.generate_new(20))
+                self._data.extend(new_datapoints)
+            datapoint = self._data.pop(0)
+            yield datapoint
+            self._batch_to_save.append(datapoint)
+            if len(self._batch_to_save) == 100:
+                if self.cache:
+                    with self.cache.open("a", encoding="utf-8") as f:
+                        for dp in self._batch_to_save:
+                            json.dump(dp.to_dict(), f, ensure_ascii=False)
+                            f.write("\n")
+                self._batch_to_save = []
 
     def sample(self) -> DataPoint:
-        if len(self._data) == 0:
-            raise RuntimeError("Dataset is empty, cannot sample.")
-        idx = self._rng.randint(0, len(self._data) - 1)
-        return self[idx]
+        r"""Returns the next datapoint from the current dataset
+        synchronously.
+
+        Raises:
+            RuntimeError: If called in an async context.
+
+        Returns:
+            DataPoint: The next DataPoint.
+
+        Note:
+            This method is intended for synchronous contexts.
+            Use 'async_sample' in asynchronous contexts to
+            avoid blocking or runtime errors.
+        """
+        try:
+            if asyncio.get_event_loop().is_running():
+                raise RuntimeError(
+                    "Cannot use synchronous sampling (sample) "
+                    "in an async context; use async_sample instead"
+                )
+        except RuntimeError as e:
+            if "no running event loop" not in str(e):
+                raise
+
+        return next(iter(self))
+
+    async def async_sample(self) -> DataPoint:
+        r"""Returns the next datapoint from the current dataset asynchronously.
+
+        Returns:
+            DataPoint: The next datapoint.
+
+        Note:
+            This method is intended for asynchronous contexts. Use 'sample'
+            in synchronous contexts.
+        """
+
+        async_iter = self.__aiter__()
+        return await async_iter.__anext__()
 
     def save_to_jsonl(self, file_path: Union[str, Path]) -> None:
         r"""Saves the generated datapoints to a JSONL (JSON Lines) file.
@@ -99,7 +205,7 @@ class BaseGenerator(abc.ABC):
 
         Notes:
             - Uses `self._data`, which contains the generated datapoints.
-            - Overwrites the file if it already exists.
+            - Appends to the file if it already exists.
             - Ensures compatibility with large datasets by using JSONL format.
         """
         if not self._data:
@@ -108,11 +214,66 @@ class BaseGenerator(abc.ABC):
         file_path = Path(file_path)
 
         try:
-            with file_path.open("w", encoding="utf-8") as f:
+            with file_path.open("a", encoding="utf-8") as f:
                 for datapoint in self._data:
                     json.dump(datapoint.to_dict(), f, ensure_ascii=False)
-                    f.write("\n")  # Ensure each entry is on a new line
+                    f.write("\n")
             logger.info(f"Dataset saved successfully to {file_path}")
         except IOError as e:
             logger.error(f"Error writing to file {file_path}: {e}")
             raise
+
+    def flush(self, file_path: Union[str, Path]) -> None:
+        r"""Flush the current data to a JSONL file and clear the data.
+
+        Args:
+            file_path (Union[str, Path]): Path to save the JSONL file.
+
+        Notes:
+            - Uses `save_to_jsonl` to save `self._data`.
+        """
+
+        self.save_to_jsonl(file_path)
+        self._data = []
+        logger.info(f"Data flushed to {file_path} and cleared from the memory")
+
+    def _init_from_jsonl(self, file_path: Path) -> List[Dict[str, Any]]:
+        r"""Load and parse a dataset from a JSONL file.
+
+        Args:
+            file_path (Path): Path to the JSONL file.
+
+        Returns:
+            List[Dict[str, Any]]: A list of datapoint dictionaries.
+
+        Raises:
+            FileNotFoundError: If the specified JSONL file does not exist.
+            ValueError: If a line contains invalid JSON or is not a dictionary.
+        """
+        if not file_path.exists():
+            raise FileNotFoundError(f"JSONL file not found: {file_path}")
+
+        raw_data = []
+        logger.debug(f"Loading JSONL from {file_path}")
+        with file_path.open('r', encoding='utf-8') as f:
+            for line_number, line in enumerate(f, start=1):
+                line = line.strip()
+                if not line:
+                    continue  # Skip blank lines
+                try:
+                    record = json.loads(line)
+                except json.JSONDecodeError as e:
+                    raise ValueError(
+                        f"Invalid JSON on line {line_number} "
+                        f"in file {file_path}: {e}"
+                    )
+                if not isinstance(record, dict):
+                    raise ValueError(
+                        f"Expected a dictionary at line {line_number}, "
+                        f"got {type(record).__name__}"
+                    )
+                raw_data.append(record)
+        logger.info(
+            f"Successfully loaded {len(raw_data)} items from {file_path}"
+        )
+        return raw_data

camel/datasets/few_shot_generator.py

@@ -22,7 +22,6 @@ from camel.agents import ChatAgent
 from camel.logger import get_logger
 from camel.models.base_model import BaseModelBackend
 from camel.verifiers import BaseVerifier
-from camel.verifiers.models import VerifierInput
 
 from .base_generator import BaseGenerator
 from .models import DataPoint
@@ -203,10 +202,8 @@ class FewShotGenerator(BaseGenerator):
 
                 try:
                     verifier_response = await self.verifier.verify(
-                        VerifierInput(
-                            llm_response=rationale,
-                            ground_truth=None,
-                        )
+                        solution=rationale,
+                        ground_truth=None,
                     )
                     if not verifier_response or not verifier_response.result:
                         raise ValueError(

camel/environments/single_step.py

@@ -23,9 +23,6 @@ from camel.verifiers.base import (
     BaseVerifier,
     VerificationResult,
 )
-from camel.verifiers.models import (
-    VerifierInput,
-)
 
 from .models import Action, Observation, StepResult
 
@@ -189,10 +186,7 @@ class SingleStepEnv:
 
         # verify the extracted
         verification_result = await self.verifier.verify(
-            VerifierInput(
-                llm_response=extraction_result,
-                ground_truth=self._state.final_answer,
-            )
+            solution=extraction_result, ground_truth=self._state.final_answer
         )
 
         # compute rewards

camel/memories/agent_memories.py

@@ -18,7 +18,8 @@ from typing import List, Optional
 from camel.memories.base import AgentMemory, BaseContextCreator
 from camel.memories.blocks import ChatHistoryBlock, VectorDBBlock
 from camel.memories.records import ContextRecord, MemoryRecord
-from camel.storages import BaseKeyValueStorage, BaseVectorStorage
+from camel.storages.key_value_storages.base import BaseKeyValueStorage
+from camel.storages.vectordb_storages.base import BaseVectorStorage
 from camel.types import OpenAIBackendRole
 
 
@@ -33,6 +34,8 @@ class ChatHistoryMemory(AgentMemory):
         window_size (int, optional): The number of recent chat messages to
             retrieve. If not provided, the entire chat history will be
             retrieved.  (default: :obj:`None`)
+        agent_id (str, optional): The ID of the agent associated with the chat
+            history.
     """
 
     def __init__(
@@ -40,6 +43,7 @@ class ChatHistoryMemory(AgentMemory):
         context_creator: BaseContextCreator,
         storage: Optional[BaseKeyValueStorage] = None,
         window_size: Optional[int] = None,
+        agent_id: Optional[str] = None,
     ) -> None:
         if window_size is not None and not isinstance(window_size, int):
             raise TypeError("`window_size` must be an integer or None.")
@@ -48,6 +52,15 @@ class ChatHistoryMemory(AgentMemory):
         self._context_creator = context_creator
         self._window_size = window_size
         self._chat_history_block = ChatHistoryBlock(storage=storage)
+        self._agent_id = agent_id
+
+    @property
+    def agent_id(self) -> Optional[str]:
+        return self._agent_id
+
+    @agent_id.setter
+    def agent_id(self, val: Optional[str]) -> None:
+        self._agent_id = val
 
     def retrieve(self) -> List[ContextRecord]:
         records = self._chat_history_block.retrieve(self._window_size)
@@ -63,6 +76,10 @@ class ChatHistoryMemory(AgentMemory):
         return records
 
     def write_records(self, records: List[MemoryRecord]) -> None:
+        for record in records:
+            # assign the agent_id to the record
+            if record.agent_id == "" and self.agent_id is not None:
+                record.agent_id = self.agent_id
         self._chat_history_block.write_records(records)
 
     def get_context_creator(self) -> BaseContextCreator:
@@ -84,6 +101,8 @@ class VectorDBMemory(AgentMemory):
             (default: :obj:`None`)
         retrieve_limit (int, optional): The maximum number of messages
             to be added into the context.  (default: :obj:`3`)
+        agent_id (str, optional): The ID of the agent associated with
+            the messages stored in the vector database.
     """
 
     def __init__(
@@ -91,13 +110,23 @@ class VectorDBMemory(AgentMemory):
         context_creator: BaseContextCreator,
         storage: Optional[BaseVectorStorage] = None,
         retrieve_limit: int = 3,
+        agent_id: Optional[str] = None,
     ) -> None:
         self._context_creator = context_creator
         self._retrieve_limit = retrieve_limit
         self._vectordb_block = VectorDBBlock(storage=storage)
+        self._agent_id = agent_id
 
         self._current_topic: str = ""
 
+    @property
+    def agent_id(self) -> Optional[str]:
+        return self._agent_id
+
+    @agent_id.setter
+    def agent_id(self, val: Optional[str]) -> None:
+        self._agent_id = val
+
     def retrieve(self) -> List[ContextRecord]:
         return self._vectordb_block.retrieve(
             self._current_topic,
@@ -109,6 +138,11 @@ class VectorDBMemory(AgentMemory):
         for record in records:
             if record.role_at_backend == OpenAIBackendRole.USER:
                 self._current_topic = record.message.content
+
+            # assign the agent_id to the record
+            if record.agent_id == "" and self.agent_id is not None:
+                record.agent_id = self.agent_id
+
         self._vectordb_block.write_records(records)
 
     def get_context_creator(self) -> BaseContextCreator:
@@ -133,6 +167,8 @@ class LongtermAgentMemory(AgentMemory):
             (default: :obj:`None`)
         retrieve_limit (int, optional): The maximum number of messages
             to be added into the context.  (default: :obj:`3`)
+        agent_id (str, optional): The ID of the agent associated with the chat
+            history and the messages stored in the vector database.
     """
 
     def __init__(
@@ -141,12 +177,22 @@ class LongtermAgentMemory(AgentMemory):
         chat_history_block: Optional[ChatHistoryBlock] = None,
         vector_db_block: Optional[VectorDBBlock] = None,
         retrieve_limit: int = 3,
+        agent_id: Optional[str] = None,
     ) -> None:
         self.chat_history_block = chat_history_block or ChatHistoryBlock()
         self.vector_db_block = vector_db_block or VectorDBBlock()
         self.retrieve_limit = retrieve_limit
         self._context_creator = context_creator
         self._current_topic: str = ""
+        self._agent_id = agent_id
+
+    @property
+    def agent_id(self) -> Optional[str]:
+        return self._agent_id
+
+    @agent_id.setter
+    def agent_id(self, val: Optional[str]) -> None:
+        self._agent_id = val
 
     def get_context_creator(self) -> BaseContextCreator:
         r"""Returns the context creator used by the memory.
@@ -166,7 +212,8 @@ class LongtermAgentMemory(AgentMemory):
         """
         chat_history = self.chat_history_block.retrieve()
         vector_db_retrieve = self.vector_db_block.retrieve(
-            self._current_topic, self.retrieve_limit
+            self._current_topic,
+            self.retrieve_limit,
         )
         return chat_history[:1] + vector_db_retrieve + chat_history[1:]
 

camel/memories/base.py

@@ -13,7 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 from abc import ABC, abstractmethod
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 from camel.memories.records import ContextRecord, MemoryRecord
 from camel.messages import OpenAIMessage
@@ -112,6 +112,16 @@ class AgentMemory(MemoryBlock, ABC):
     the memory records stored within the AgentMemory.
     """
 
+    @property
+    @abstractmethod
+    def agent_id(self) -> Optional[str]:
+        pass
+
+    @agent_id.setter
+    @abstractmethod
+    def agent_id(self, val: Optional[str]) -> None:
+        pass
+
     @abstractmethod
     def retrieve(self) -> List[ContextRecord]:
         r"""Get a record list from the memory for creating model context.
@@ -138,3 +148,15 @@ class AgentMemory(MemoryBlock, ABC):
                 context in OpenAIMessage format and the total token count.
         """
         return self.get_context_creator().create_context(self.retrieve())
+
+    def __repr__(self) -> str:
+        r"""Returns a string representation of the AgentMemory.
+
+        Returns:
+            str: A string in the format 'ClassName(agent_id=<id>)'
+                if agent_id exists, otherwise just 'ClassName()'.
+        """
+        agent_id = getattr(self, '_agent_id', None)
+        if agent_id:
+            return f"{self.__class__.__name__}(agent_id='{agent_id}')"
+        return f"{self.__class__.__name__}()"

camel/memories/blocks/chat_history_block.py

@@ -16,7 +16,8 @@ from typing import List, Optional
 
 from camel.memories.base import MemoryBlock
 from camel.memories.records import ContextRecord, MemoryRecord
-from camel.storages import BaseKeyValueStorage, InMemoryKeyValueStorage
+from camel.storages.key_value_storages.base import BaseKeyValueStorage
+from camel.storages.key_value_storages.in_memory import InMemoryKeyValueStorage
 from camel.types import OpenAIBackendRole
 
 

camel/memories/records.py

@@ -39,6 +39,8 @@ class MemoryRecord(BaseModel):
             key-value pairs that provide more information. If not given, it
             will be an empty `Dict`.
         timestamp (float, optional): The timestamp when the record was created.
+        agent_id (str): The identifier of the agent associated with this
+            memory.
     """
 
     model_config = ConfigDict(arbitrary_types_allowed=True)
@@ -50,6 +52,7 @@ class MemoryRecord(BaseModel):
     timestamp: float = Field(
         default_factory=lambda: datetime.now(timezone.utc).timestamp()
     )
+    agent_id: str = Field(default="")
 
     _MESSAGE_TYPES: ClassVar[dict] = {
         "BaseMessage": BaseMessage,
@@ -73,6 +76,7 @@ class MemoryRecord(BaseModel):
             role_at_backend=record_dict["role_at_backend"],
             extra_info=record_dict["extra_info"],
             timestamp=record_dict["timestamp"],
+            agent_id=record_dict["agent_id"],
         )
 
     def to_dict(self) -> Dict[str, Any]:
@@ -88,6 +92,7 @@ class MemoryRecord(BaseModel):
             "role_at_backend": self.role_at_backend,
             "extra_info": self.extra_info,
             "timestamp": self.timestamp,
+            "agent_id": self.agent_id,
         }
 
     def to_openai_message(self) -> OpenAIMessage:

camel/models/__init__.py

@@ -33,6 +33,7 @@ from .ollama_model import OllamaModel
 from .openai_audio_models import OpenAIAudioModels
 from .openai_compatible_model import OpenAICompatibleModel
 from .openai_model import OpenAIModel
+from .openrouter_model import OpenRouterModel
 from .qwen_model import QwenModel
 from .reka_model import RekaModel
 from .samba_model import SambaModel
@@ -48,6 +49,7 @@ from .zhipuai_model import ZhipuAIModel
 __all__ = [
     'BaseModelBackend',
     'OpenAIModel',
+    'OpenRouterModel',
     'AzureOpenAIModel',
     'AnthropicModel',
     'MistralModel',

camel/models/gemini_model.py

@@ -172,6 +172,24 @@ class GeminiModel(BaseModelBackend):
             for tool in tools:
                 function_dict = tool.get('function', {})
                 function_dict.pop("strict", None)
+
+                # Process parameters to remove anyOf
+                if 'parameters' in function_dict:
+                    params = function_dict['parameters']
+                    if 'properties' in params:
+                        for prop_name, prop_value in params[
+                            'properties'
+                        ].items():
+                            if 'anyOf' in prop_value:
+                                # Replace anyOf with the first type in the list
+                                first_type = prop_value['anyOf'][0]
+                                params['properties'][prop_name] = first_type
+                                # Preserve description if it exists
+                                if 'description' in prop_value:
+                                    params['properties'][prop_name][
+                                        'description'
+                                    ] = prop_value['description']
+
             request_config["tools"] = tools
 
         return self._client.chat.completions.create(
@@ -191,6 +209,24 @@ class GeminiModel(BaseModelBackend):
             for tool in tools:
                 function_dict = tool.get('function', {})
                 function_dict.pop("strict", None)
+
+                # Process parameters to remove anyOf
+                if 'parameters' in function_dict:
+                    params = function_dict['parameters']
+                    if 'properties' in params:
+                        for prop_name, prop_value in params[
+                            'properties'
+                        ].items():
+                            if 'anyOf' in prop_value:
+                                # Replace anyOf with the first type in the list
+                                first_type = prop_value['anyOf'][0]
+                                params['properties'][prop_name] = first_type
+                                # Preserve description if it exists
+                                if 'description' in prop_value:
+                                    params['properties'][prop_name][
+                                        'description'
+                                    ] = prop_value['description']
+
             request_config["tools"] = tools
 
         return await self._async_client.chat.completions.create(

camel/models/groq_model.py

@@ -195,7 +195,10 @@ class GroqModel(BaseModelBackend):
 
     @property
     def stream(self) -> bool:
-        r"""Returns whether the model supports streaming. But Groq API does
-        not support streaming.
+        r"""Returns whether the model is in stream mode, which sends partial
+        results each time.
+
+        Returns:
+            bool: Whether the model is in stream mode.
         """
-        return False
+        return self.model_config_dict.get("stream", False)

camel/models/model_factory.py

@@ -29,6 +29,7 @@ from camel.models.nvidia_model import NvidiaModel
 from camel.models.ollama_model import OllamaModel
 from camel.models.openai_compatible_model import OpenAICompatibleModel
 from camel.models.openai_model import OpenAIModel
+from camel.models.openrouter_model import OpenRouterModel
 from camel.models.qwen_model import QwenModel
 from camel.models.reka_model import RekaModel
 from camel.models.samba_model import SambaModel
@@ -119,6 +120,8 @@ class ModelFactory:
             model_class = AnthropicModel
         elif model_platform.is_groq and model_type.is_groq:
             model_class = GroqModel
+        elif model_platform.is_openrouter and model_type.is_openrouter:
+            model_class = OpenRouterModel
         elif model_platform.is_zhipuai and model_type.is_zhipuai:
             model_class = ZhipuAIModel
         elif model_platform.is_gemini and model_type.is_gemini: