camel-ai 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

camel/memories/base.py

@@ -15,42 +15,19 @@
 from abc import ABC, abstractmethod
 from typing import List, Tuple
 
-from camel.memories import MemoryRecord
+from camel.memories.records import ContextRecord, MemoryRecord
 from camel.messages import OpenAIMessage
+from camel.utils import BaseTokenCounter
 
 
-class BaseMemory(ABC):
-    r"""An abstract base class that defines the foundational operations for a
-    memory component within an agent's memory system.
-
-    The memory component is tasked with functions like saving chat histories,
-    fetching or storing information in vector databases, and other related
-    operations. Every memory system should incorporate at least one instance of
-    a subclass derived from :obj:`BaseMemory`.
-
-    These instances, known as "memories", typically communicate using the
-    :obj:`MemoryRecord` object. Usually, a memory has at least one "storage"
-    mechanism, allowing it to interface with various storage systems, such as
-    disks or vector databases. Additionally, some memories might embed other
-    memory instances, enabling them to function as a high-level controller
-    within the broader memory system.
-
-    By default, when executing the :obj:`step()` method, an agent retrieves
-    messages from its designated memory and combines them with an incoming
-    message for input to the agent. Subsequently, both the response message and
-    the incoming messages are archived back into the memory.
+class MemoryBlock(ABC):
+    r"""An abstract class serves as the fundamental component within the agent
+    memory system. This class is equipped with "write" and "clear" functions.
+    However, it intentionally does not define a retrieval interface, as the
+    structure of the data to be retrieved may vary in different types of
+    memory blocks.
     """
 
-    @abstractmethod
-    def get_context(self) -> Tuple[List[OpenAIMessage], int]:
-        r"""Gets chat context with a proper size for the agent.
-
-        Returns:
-            (List[OpenAIMessage], int): A tuple containing the constructed
-                context in OpenAIMessage format and the total token count.
-        """
-        pass
-
     @abstractmethod
     def write_records(self, records: List[MemoryRecord]) -> None:
         r"""Writes records to the memory, appending them to existing ones.
@@ -70,6 +47,94 @@ class BaseMemory(ABC):
 
     @abstractmethod
     def clear(self) -> None:
-        r"""Clears all messages from the memory.
+        r"""Clears all messages from the memory."""
+        pass
+
+
+class BaseContextCreator(ABC):
+    r"""An abstract base class defining the interface for context creation
+    strategies.
+
+    This class provides a foundational structure for different strategies to
+    generate conversational context from a list of context records. The
+    primary goal is to create a context that is aligned with a specified token
+    count limit, allowing subclasses to define their specific approach.
+
+    Subclasses should implement the :obj:`token_counter`,:obj: `token_limit`,
+    and :obj:`create_context` methods to provide specific context creation
+    logic.
+
+    Attributes:
+        token_counter (BaseTokenCounter): A token counter instance responsible
+            for counting tokens in a message.
+        token_limit (int): The maximum number of tokens allowed in the
+            generated context.
+    """
+
+    @property
+    @abstractmethod
+    def token_counter(self) -> BaseTokenCounter:
+        pass
+
+    @property
+    @abstractmethod
+    def token_limit(self) -> int:
+        pass
+
+    @abstractmethod
+    def create_context(
+        self,
+        records: List[ContextRecord],
+    ) -> Tuple[List[OpenAIMessage], int]:
+        r"""An abstract method to create conversational context from the chat
+        history.
+
+        Constructs the context from provided records. The specifics of how this
+        is done and how the token count is managed should be provided by
+        subclasses implementing this method. The output messages order
+        should keep same as the input order.
+
+        Args:
+            records (List[ContextRecord]): A list of context records from
+                which to generate the context.
+
+        Returns:
+            Tuple[List[OpenAIMessage], int]: A tuple containing the constructed
+                context in OpenAIMessage format and the total token count.
+        """
+        pass
+
+
+class AgentMemory(MemoryBlock, ABC):
+    r"""Represents a specialized form of `MemoryBlock`, uniquely designed for
+    direct integration with an agent. Two key abstract functions, "retrieve"
+    and "get_context_creator", are used for generating model context based on
+    the memory records stored within the AgentMemory.
+    """
+
+    @abstractmethod
+    def retrieve(self) -> List[ContextRecord]:
+        r"""Get a record list from the memory for creating model context.
+
+        Returns:
+            List[ContextRecord]: A record list for creating model context.
         """
         pass
+
+    @abstractmethod
+    def get_context_creator(self) -> BaseContextCreator:
+        r"""Gets context creator.
+
+        Returns:
+            BaseContextCreator: A model context creator.
+        """
+        pass
+
+    def get_context(self) -> Tuple[List[OpenAIMessage], int]:
+        r"""Gets chat context with a proper size for the agent from the memory.
+
+        Returns:
+            (List[OpenAIMessage], int): A tuple containing the constructed
+                context in OpenAIMessage format and the total token count.
+        """
+        return self.get_context_creator().create_context(self.retrieve())

camel/memories/blocks/__init__.py

@@ -0,0 +1,21 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from .chat_history_block import ChatHistoryBlock
+from .vectordb_block import VectorDBBlock
+
+__all__ = [
+    'ChatHistoryBlock',
+    'VectorDBBlock',
+]

camel/memories/{chat_history_memory.py → blocks/chat_history_block.py}

@@ -11,68 +11,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from typing import List, Optional, Tuple
+import warnings
+from typing import List, Optional
 
-from camel.memories import BaseMemory, ContextRecord, MemoryRecord
-from camel.memories.context_creators import BaseContextCreator
-from camel.messages import OpenAIMessage
+from camel.memories.base import MemoryBlock
+from camel.memories.records import ContextRecord, MemoryRecord
 from camel.storages import BaseKeyValueStorage, InMemoryKeyValueStorage
 from camel.types import OpenAIBackendRole
 
 
-class ChatHistoryMemory(BaseMemory):
-    r"""An implementation of the :obj:`BaseMemory` abstract base class for
+class ChatHistoryBlock(MemoryBlock):
+    r"""An implementation of the :obj:`MemoryBlock` abstract base class for
     maintaining a record of chat histories.
 
-    This memory class helps manage conversation histories with a designated
-    storage mechanism, either provided by the user or using a default
+    This memory block helps manage conversation histories with a key-value
+    storage backend, either provided by the user or using a default
     in-memory storage. It offers a windowed approach to retrieving chat
     histories, allowing users to specify how many recent messages they'd
     like to fetch.
 
-    `ChatHistoryMemory` requires messages to be stored with certain
-    metadata (e.g., `role_at_backend`) to maintain consistency and validate
-    the chat history.
-
     Args:
-        context_creator (BaseContextCreator): A context creator contianing
-            the context limit and the message pruning strategy.
         storage (BaseKeyValueStorage, optional): A storage mechanism for
             storing chat history. If `None`, an :obj:`InMemoryKeyValueStorage`
             will be used. (default: :obj:`None`)
-        window_size (int, optional): Specifies the number of recent chat
-            messages to retrieve. If not provided, the entire chat history
-            will be retrieved. (default: :obj:`None`)
+        keep_rate (float, optional): In historical messages, the score of the
+            last message is 1.0, and with each step taken backward, the score
+            of the message is multiplied by the `keep_rate`. Higher `keep_rate`
+            leads to high possiblity to keep history messages during context
+            creation.
     """
 
     def __init__(
         self,
-        context_creator: BaseContextCreator,
         storage: Optional[BaseKeyValueStorage] = None,
-        window_size: Optional[int] = None,
+        keep_rate: float = 0.9,
     ) -> None:
-        self.context_creator = context_creator
+        if keep_rate > 1 or keep_rate < 0:
+            raise ValueError("`keep_rate` should be in [0,1]")
         self.storage = storage or InMemoryKeyValueStorage()
-        self.window_size = window_size
+        self.keep_rate = keep_rate
 
-    def get_context(self) -> Tuple[List[OpenAIMessage], int]:
-        r"""Gets chat context with a proper size for the agent from the memory
+    def retrieve(
+        self,
+        window_size: Optional[int] = None,
+    ) -> List[ContextRecord]:
+        r"""Retrieves records with a proper size for the agent from the memory
         based on the window size or fetches the entire chat history if no
         window size is specified.
 
+        Args:
+            window_size (int, optional): Specifies the number of recent chat
+                messages to retrieve. If not provided, the entire chat history
+                will be retrieved. (default: :obj:`None`)
+
         Returns:
-            (List[OpenAIMessage], int): A tuple containing the constructed
-                context in OpenAIMessage format and the total token count.
-        Raises:
-            ValueError: If the memory is empty or if the first message in the
-                memory is not a system message.
+            List[ContextRecord]: A list of retrieved records.
         """
         record_dicts = self.storage.load()
         if len(record_dicts) == 0:
-            raise ValueError("The `ChatHistoryMemory` is empty.")
+            warnings.warn("The `ChatHistoryMemory` is empty.")
+            return list()
 
         chat_records: List[MemoryRecord] = []
-        truncate_idx = -self.window_size if self.window_size is not None else 0
+        truncate_idx = -window_size if window_size is not None else 0
         for record_dict in record_dicts[truncate_idx:]:
             chat_records.append(MemoryRecord.from_dict(record_dict))
 
@@ -86,11 +87,11 @@ class ChatHistoryMemory(BaseMemory):
                 output_records.append(ContextRecord(record, 1.0))
             else:
                 # Other messages' score drops down gradually
-                score *= 0.99
+                score *= self.keep_rate
                 output_records.append(ContextRecord(record, score))
 
         output_records.reverse()
-        return self.context_creator.create_context(output_records)
+        return output_records
 
     def write_records(self, records: List[MemoryRecord]) -> None:
         r"""Writes memory records to the memory. Additionally, performs
@@ -106,6 +107,5 @@ class ChatHistoryMemory(BaseMemory):
         self.storage.save(stored_records)
 
     def clear(self) -> None:
-        r"""Clears all chat messages from the memory.
-        """
+        r"""Clears all chat messages from the memory."""
         self.storage.clear()

camel/memories/blocks/vectordb_block.py

@@ -0,0 +1,101 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from typing import List, Optional
+
+from camel.embeddings import BaseEmbedding, OpenAIEmbedding
+from camel.memories.base import MemoryBlock
+from camel.memories.records import ContextRecord, MemoryRecord
+from camel.storages.vectordb_storages import (
+    BaseVectorStorage,
+    QdrantStorage,
+    VectorDBQuery,
+    VectorRecord,
+)
+
+
+class VectorDBBlock(MemoryBlock):
+    r"""An implementation of the :obj:`MemoryBlock` abstract base class for
+    maintaining and retrieving information using vector embeddings within a
+    vector database.
+
+    Args:
+        storage (Optional[BaseVectorStorage], optional): The storage mechanism
+            for the vector database. Defaults to in-memory :obj:`Qdrant` if not
+            provided. (default: :obj:`None`)
+        embedding (Optional[BaseEmbedding], optional): Embedding mechanism to
+            convert chat messages into vector representations. Defaults to
+            :obj:`OpenAiEmbedding` if not provided. (default: :obj:`None`)
+    """
+
+    def __init__(
+        self,
+        storage: Optional[BaseVectorStorage] = None,
+        embedding: Optional[BaseEmbedding] = None,
+    ) -> None:
+        self.embedding = embedding or OpenAIEmbedding()
+        self.vector_dim = self.embedding.get_output_dim()
+        self.storage = storage or QdrantStorage(vector_dim=self.vector_dim)
+
+    def retrieve(
+        self,
+        keyword: str,
+        limit: int = 3,
+    ) -> List[ContextRecord]:
+        r"""Retrieves similar records from the vector database based on the
+        content of the keyword.
+
+        Args:
+            keyword (str): This string will be converted into a vector
+                representation to query the database.
+            limit (int, optional): The maximum number of similar messages to
+                retrieve. (default: :obj:`3`).
+
+        Returns:
+            List[ContextRecord]: A list of memory records retrieved from the
+                vector database based on similarity to :obj:`current_state`.
+        """
+        query_vector = self.embedding.embed(keyword)
+        results = self.storage.query(VectorDBQuery(query_vector, top_k=limit))
+        return [
+            ContextRecord(
+                memory_record=MemoryRecord.from_dict(result.record.payload),
+                score=result.similarity,
+            )
+            for result in results
+            if result.record.payload is not None
+        ]
+
+    def write_records(self, records: List[MemoryRecord]) -> None:
+        """
+        Converts the provided chat messages into vector representations and
+        writes them to the vector database.
+
+        Args:
+            records (List[MemoryRecord]): Memory records to be added to the
+                memory.
+        """
+        v_records = [
+            VectorRecord(
+                vector=self.embedding.embed(record.message.content),
+                payload=record.to_dict(),
+                id=str(record.uuid),
+            )
+            for record in records
+        ]
+        self.storage.add(v_records)
+
+    def clear(self) -> None:
+        r"""Removes all records from the vector database memory."""
+        self.storage.clear()

camel/memories/context_creators/__init__.py

@@ -12,7 +12,8 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 
-from .base import BaseContextCreator
 from .score_based import ScoreBasedContextCreator
 
-__all__ = ['BaseContextCreator', 'ScoreBasedContextCreator']
+__all__ = [
+    'ScoreBasedContextCreator',
+]

camel/memories/context_creators/score_based.py

@@ -14,8 +14,8 @@
 from dataclasses import dataclass
 from typing import List, Tuple
 
-from camel.memories import ContextRecord
-from camel.memories.context_creators import BaseContextCreator
+from camel.memories.base import BaseContextCreator
+from camel.memories.records import ContextRecord
 from camel.messages import OpenAIMessage
 from camel.utils import BaseTokenCounter
 
@@ -43,8 +43,9 @@ class ScoreBasedContextCreator(BaseContextCreator):
             generated context.
     """
 
-    def __init__(self, token_counter: BaseTokenCounter,
-                 token_limit: int) -> None:
+    def __init__(
+        self, token_counter: BaseTokenCounter, token_limit: int
+    ) -> None:
         self._token_counter = token_counter
         self._token_limit = token_limit
 
@@ -79,14 +80,22 @@ class ScoreBasedContextCreator(BaseContextCreator):
             RuntimeError: If it's impossible to create a valid context without
                 exceeding the token limit.
         """
-        context_units = [
-            _ContextUnit(
-                idx,
-                record,
-                self.token_counter.count_tokens_from_messages(
-                    [record.memory_record.to_openai_message()]),
-            ) for idx, record in enumerate(records)
-        ]
+        # Create unique context units list
+        uuid_set = set()
+        context_units = []
+        for idx, record in enumerate(records):
+            if record.memory_record.uuid not in uuid_set:
+                uuid_set.add(record.memory_record.uuid)
+                context_units.append(
+                    _ContextUnit(
+                        idx,
+                        record,
+                        self.token_counter.count_tokens_from_messages(
+                            [record.memory_record.to_openai_message()]
+                        ),
+                    )
+                )
+
         # TODO: optimize the process, may give information back to memory
 
         # If not exceed token limit, simply return
@@ -95,27 +104,30 @@ class ScoreBasedContextCreator(BaseContextCreator):
             return self._create_output(context_units)
 
         # Sort by score
-        context_units = sorted(context_units,
-                               key=lambda unit: unit.record.score)
+        context_units = sorted(
+            context_units, key=lambda unit: unit.record.score
+        )
 
-        # Remove least score messages until total token number is smaller
+        # Remove the least score messages until total token number is smaller
         # than token limit
         truncate_idx = None
         for i, unit in enumerate(context_units):
             if unit.record.score == 1:
                 raise RuntimeError(
-                    "Cannot create context: exceed token limit.", total_tokens)
+                    "Cannot create context: exceed token limit.", total_tokens
+                )
             total_tokens -= unit.num_tokens
             if total_tokens <= self.token_limit:
                 truncate_idx = i
                 break
         if truncate_idx is None:
-            raise RuntimeError("Cannot create context: exceed token limit.",
-                               total_tokens)
-        return self._create_output(context_units[truncate_idx + 1:])
+            raise RuntimeError(
+                "Cannot create context: exceed token limit.", total_tokens
+            )
+        return self._create_output(context_units[truncate_idx + 1 :])
 
     def _create_output(
-            self, context_units: List[_ContextUnit]
+        self, context_units: List[_ContextUnit]
     ) -> Tuple[List[OpenAIMessage], int]:
         r"""Helper method to generate output from context units.
 

camel/memories/records.py

@@ -37,6 +37,7 @@ class MemoryRecord:
             key-value pairs that provide more information. If not given, it
             will be an empty `Dict`.
     """
+
     message: BaseMessage
     role_at_backend: OpenAIBackendRole
     uuid: UUID = field(default_factory=uuid4)
@@ -44,7 +45,7 @@ class MemoryRecord:
 
     _MESSAGE_TYPES: ClassVar[dict] = {
         "BaseMessage": BaseMessage,
-        "FunctionCallingMessage": FunctionCallingMessage
+        "FunctionCallingMessage": FunctionCallingMessage,
     }
 
     @classmethod
@@ -73,10 +74,10 @@ class MemoryRecord:
             "uuid": str(self.uuid),
             "message": {
                 "__class__": self.message.__class__.__name__,
-                **asdict(self.message)
+                **asdict(self.message),
             },
             "role_at_backend": self.role_at_backend,
-            "extra_info": self.extra_info
+            "extra_info": self.extra_info,
         }
 
     def to_openai_message(self) -> OpenAIMessage:
@@ -86,7 +87,7 @@ class MemoryRecord:
 
 @dataclass(frozen=True)
 class ContextRecord:
-    r"""The result of memory retrieving.
-    """
+    r"""The result of memory retrieving."""
+
     memory_record: MemoryRecord
     score: float

camel/messages/__init__.py

@@ -12,11 +12,11 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from camel.types import (
-    ChatCompletionSystemMessageParam,
     ChatCompletionAssistantMessageParam,
-    ChatCompletionUserMessageParam,
     ChatCompletionFunctionMessageParam,
     ChatCompletionMessageParam,
+    ChatCompletionSystemMessageParam,
+    ChatCompletionUserMessageParam,
 )
 
 OpenAISystemMessage = ChatCompletionSystemMessageParam