camel-ai 0.2.78__py3-none-any.whl → 0.2.79a1__py3-none-any.whl

camel/datasets/base_generator.py

@@ -62,6 +62,7 @@ class BaseGenerator(abc.ABC, IterableDataset):
         self._buffer = buffer
         self._data: List[DataPoint] = []
         self._batch_to_save: List[DataPoint] = []
+        self._iter_position: int = 0
 
         if data_path:
             file_path = Path(data_path)
@@ -103,9 +104,9 @@ class BaseGenerator(abc.ABC, IterableDataset):
         r"""Async iterator that yields datapoints dynamically.
 
         If a `data_path` was provided during initialization, those datapoints
-        are yielded first. When self._data is empty, 20 new datapoints
-        are generated. Every 100 yields, the batch is appended to the
-        JSONL file or discarded if `cache` is None.
+        are yielded first. When self._iter_position reaches the end of _data,
+        new datapoints are generated. Every 100 yields, the batch is appended
+        to the JSONL file or discarded if `cache` is None.
 
         Yields:
             DataPoint: A single datapoint.
@@ -113,9 +114,10 @@ class BaseGenerator(abc.ABC, IterableDataset):
 
         async def generator():
             while True:
-                if not self._data:
+                if self._iter_position >= len(self._data):
                     await self.generate_new(self._buffer)
-                datapoint = self._data.pop(0)
+                datapoint = self._data[self._iter_position]
+                self._iter_position += 1
                 yield datapoint
                 self._batch_to_save.append(datapoint)
                 if len(self._batch_to_save) == 100:
@@ -132,9 +134,9 @@ class BaseGenerator(abc.ABC, IterableDataset):
         r"""Synchronous iterator for PyTorch IterableDataset compatibility.
 
         If a `data_path` was provided during initialization, those datapoints
-        are yielded first. When self._data is empty, 20 new datapoints
-        are generated. Every 100 yields, the batch is appended to the
-        JSONL file or discarded if `cache` is None.
+        are yielded first. When self._iter_position reaches the end of _data,
+        new datapoints are generated. Every 100 yields, the batch is appended
+        to the JSONL file or discarded if `cache` is None.
 
         Yields:
             DataPoint: A single datapoint.
@@ -150,9 +152,10 @@ class BaseGenerator(abc.ABC, IterableDataset):
                 raise
 
         while True:
-            if not self._data:
+            if self._iter_position >= len(self._data):
                 asyncio.run(self.generate_new(self._buffer))
-            datapoint = self._data.pop(0)
+            datapoint = self._data[self._iter_position]
+            self._iter_position += 1
             yield datapoint
             self._batch_to_save.append(datapoint)
             if len(self._batch_to_save) == 100:
@@ -248,6 +251,7 @@ class BaseGenerator(abc.ABC, IterableDataset):
 
         self.save_to_jsonl(file_path)
         self._data = []
+        self._iter_position = 0
         logger.info(f"Data flushed to {file_path} and cleared from the memory")
 
     def _init_from_jsonl(self, file_path: Path) -> List[Dict[str, Any]]:
@@ -290,3 +294,28 @@ class BaseGenerator(abc.ABC, IterableDataset):
             f"Successfully loaded {len(raw_data)} items from {file_path}"
         )
         return raw_data
+
+    def __getitem__(self, index: int) -> DataPoint:
+        r"""Get a datapoint by index without removing the datapoint from _data.
+
+        Args:
+            index (int): Index of the datapoint to retrieve.
+
+        Returns:
+            DataPoint: The datapoint at the specified index.
+
+        Raises:
+            IndexError: If the index is out of range.
+        """
+        if index < 0 or index >= len(self._data):
+            raise IndexError(f"Index {index} is out of range")
+
+        return self._data[index]
+
+    def __len__(self) -> int:
+        r"""Get the number of datapoints in the dataset.
+
+        Returns:
+            int: The number of datapoints.
+        """
+        return len(self._data)

camel/environments/single_step.py

@@ -218,9 +218,34 @@ class SingleStepEnv:
             return observations[0] if batch_size == 1 else observations
 
         elif isinstance(self.dataset, BaseGenerator):
-            self._states = [
-                await self.dataset.async_sample() for _ in range(batch_size)
-            ]
+            # Generate more data if needed
+            if batch_size > len(self.dataset):
+                new_datapoints_needed = batch_size - len(self.dataset)
+                await self.dataset.generate_new(n=new_datapoints_needed)
+
+                # Verify that enough data was generated
+                if len(self.dataset) < batch_size:
+                    raise RuntimeError(
+                        f"Failed to generate enough datapoints. "
+                        f"Requested {batch_size}, but only "
+                        f"{len(self.dataset)} available after generation."
+                    )
+
+            # Choose sampling strategy based on whether seed is provided
+            if seed is not None:
+                # Deterministic random sampling when seed is provided
+                random_indices = rng.sample(
+                    range(len(self.dataset)), batch_size
+                )
+                self._states = [self.dataset[ind] for ind in random_indices]
+            else:
+                # Sequential sampling when no seed (backward compatible)
+                # Use async_sample to maintain sequential behavior
+                self._states = [
+                    await self.dataset.async_sample()
+                    for _ in range(batch_size)
+                ]
+
             self.current_batch_size = batch_size
             self._states_done = [False] * batch_size
 

camel/memories/__init__.py

@@ -18,7 +18,7 @@ from .agent_memories import (
     VectorDBMemory,
 )
 from .base import AgentMemory, BaseContextCreator, MemoryBlock
-from .blocks.chat_history_block import ChatHistoryBlock, EmptyMemoryWarning
+from .blocks.chat_history_block import ChatHistoryBlock
 from .blocks.vectordb_block import VectorDBBlock
 from .context_creators.score_based import ScoreBasedContextCreator
 from .records import ContextRecord, MemoryRecord
@@ -35,5 +35,4 @@ __all__ = [
     'ChatHistoryBlock',
     'VectorDBBlock',
     'LongtermAgentMemory',
-    'EmptyMemoryWarning',
 ]

camel/memories/agent_memories.py

@@ -129,6 +129,16 @@ class ChatHistoryMemory(AgentMemory):
         # Save the modified records back to storage
         self._chat_history_block.storage.save(record_dicts)
 
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes the most recent records from chat history memory."""
+        return self._chat_history_block.pop_records(count)
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from chat history memory."""
+        return self._chat_history_block.remove_records_by_indices(indices)
+
 
 class VectorDBMemory(AgentMemory):
     r"""An agent memory wrapper of :obj:`VectorDBBlock`. This memory queries
@@ -193,6 +203,20 @@ class VectorDBMemory(AgentMemory):
         r"""Removes all records from the vector database memory."""
         self._vectordb_block.clear()
 
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Rolling back is unsupported for vector database memory."""
+        raise NotImplementedError(
+            "VectorDBMemory does not support removing historical records."
+        )
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removing by indices is unsupported for vector database memory."""
+        raise NotImplementedError(
+            "VectorDBMemory does not support removing records by indices."
+        )
+
 
 class LongtermAgentMemory(AgentMemory):
     r"""An implementation of the :obj:`AgentMemory` abstract base class for
@@ -277,3 +301,13 @@ class LongtermAgentMemory(AgentMemory):
         r"""Removes all records from the memory."""
         self.chat_history_block.clear()
         self.vector_db_block.clear()
+
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes recent chat history records while leaving vector memory."""
+        return self.chat_history_block.pop_records(count)
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from chat history."""
+        return self.chat_history_block.remove_records_by_indices(indices)

camel/memories/base.py

@@ -45,6 +45,32 @@ class MemoryBlock(ABC):
         """
         self.write_records([record])
 
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes records from the memory and returns the removed records.
+
+        Args:
+            count (int): Number of records to remove.
+
+        Returns:
+            List[MemoryRecord]: The records that were removed from the memory
+                in their original order.
+        """
+        raise NotImplementedError
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from the memory.
+
+        Args:
+            indices (List[int]): List of indices to remove. Indices should be
+                valid positions in the current record list.
+
+        Returns:
+            List[MemoryRecord]: The removed records in their original order.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def clear(self) -> None:
         r"""Clears all messages from the memory."""

camel/memories/blocks/chat_history_block.py

@@ -11,7 +11,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
-import warnings
 from typing import List, Optional
 
 from camel.memories.base import MemoryBlock
@@ -21,17 +20,6 @@ from camel.storages.key_value_storages.in_memory import InMemoryKeyValueStorage
 from camel.types import OpenAIBackendRole
 
 
-class EmptyMemoryWarning(UserWarning):
-    """Warning raised when attempting to access an empty memory.
-
-    This warning is raised when operations are performed on memory
-    that contains no records. It can be safely caught and suppressed
-    in contexts where empty memory is expected.
-    """
-
-    pass
-
-
 class ChatHistoryBlock(MemoryBlock):
     r"""An implementation of the :obj:`MemoryBlock` abstract base class for
     maintaining a record of chat histories.
@@ -81,11 +69,8 @@ class ChatHistoryBlock(MemoryBlock):
         """
         record_dicts = self.storage.load()
         if len(record_dicts) == 0:
-            warnings.warn(
-                "The `ChatHistoryMemory` is empty.",
-                EmptyMemoryWarning,
-                stacklevel=1,
-            )
+            # Empty memory is a valid state (e.g., during initialization).
+            # Users can check if memory is empty by checking the returned list.
             return list()
 
         if window_size is not None and window_size >= 0:
@@ -182,3 +167,118 @@ class ChatHistoryBlock(MemoryBlock):
     def clear(self) -> None:
         r"""Clears all chat messages from the memory."""
         self.storage.clear()
+
+    def pop_records(self, count: int) -> List[MemoryRecord]:
+        r"""Removes the most recent records from the memory.
+
+        Args:
+            count (int): Number of records to remove from the end of the
+                conversation history. A value of 0 results in no changes.
+
+        Returns:
+            List[MemoryRecord]: The removed records in chronological order.
+        """
+        if not isinstance(count, int):
+            raise TypeError("`count` must be an integer.")
+        if count < 0:
+            raise ValueError("`count` must be non-negative.")
+        if count == 0:
+            return []
+
+        record_dicts = self.storage.load()
+        if not record_dicts:
+            return []
+
+        # Preserve initial system/developer instruction if present.
+        protected_prefix = (
+            1
+            if (
+                record_dicts
+                and record_dicts[0]['role_at_backend']
+                in {
+                    OpenAIBackendRole.SYSTEM.value,
+                    OpenAIBackendRole.DEVELOPER.value,
+                }
+            )
+            else 0
+        )
+
+        removable_count = max(len(record_dicts) - protected_prefix, 0)
+        if removable_count == 0:
+            return []
+
+        pop_count = min(count, removable_count)
+        split_index = len(record_dicts) - pop_count
+
+        popped_dicts = record_dicts[split_index:]
+        remaining_dicts = record_dicts[:split_index]
+
+        self.storage.clear()
+        if remaining_dicts:
+            self.storage.save(remaining_dicts)
+
+        return [MemoryRecord.from_dict(record) for record in popped_dicts]
+
+    def remove_records_by_indices(
+        self, indices: List[int]
+    ) -> List[MemoryRecord]:
+        r"""Removes records at specified indices from the memory.
+
+        Args:
+            indices (List[int]): List of indices to remove. Indices are
+                positions in the current record list (0-based).
+                System/developer messages at index 0 are protected and will
+                not be removed.
+
+        Returns:
+            List[MemoryRecord]: The removed records in their original order.
+        """
+        if not indices:
+            return []
+
+        record_dicts = self.storage.load()
+        if not record_dicts:
+            return []
+
+        # Preserve initial system/developer instruction if present.
+        protected_prefix = (
+            1
+            if (
+                record_dicts
+                and record_dicts[0]['role_at_backend']
+                in {
+                    OpenAIBackendRole.SYSTEM.value,
+                    OpenAIBackendRole.DEVELOPER.value,
+                }
+            )
+            else 0
+        )
+
+        # Filter out protected indices and invalid ones
+        valid_indices = sorted(
+            {
+                idx
+                for idx in indices
+                if idx >= protected_prefix and idx < len(record_dicts)
+            }
+        )
+
+        if not valid_indices:
+            return []
+
+        # Extract records to remove (in original order)
+        removed_records = [record_dicts[idx] for idx in valid_indices]
+
+        # Build remaining records by excluding removed indices
+        remaining_dicts = [
+            record
+            for idx, record in enumerate(record_dicts)
+            if idx not in valid_indices
+        ]
+
+        # Save back to storage
+        self.storage.clear()
+        if remaining_dicts:
+            self.storage.save(remaining_dicts)
+
+        return [MemoryRecord.from_dict(record) for record in removed_records]