camel-ai 0.1.1__py3-none-any.whl → 0.1.3__py3-none-any.whl

camel/responses/agent_responses.py

@@ -30,6 +30,7 @@ class ChatAgentResponse:
             to terminate the chat session.
         info (Dict[str, Any]): Extra information about the chat message.
     """
+
     msgs: List[BaseMessage]
     terminated: bool
     info: Dict[str, Any]
@@ -37,6 +38,8 @@ class ChatAgentResponse:
     @property
     def msg(self):
         if len(self.msgs) != 1:
-            raise RuntimeError("Property msg is only available "
-                               "for a single message in msgs.")
+            raise RuntimeError(
+                "Property msg is only available "
+                "for a single message in msgs."
+            )
         return self.msgs[0]

camel/retrievers/__init__.py

@@ -0,0 +1,24 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from .auto_retriever import AutoRetriever
+from .base import BaseRetriever
+from .bm25_retriever import BM25Retriever
+from .vector_retriever import VectorRetriever
+
+__all__ = [
+    'BaseRetriever',
+    'VectorRetriever',
+    'AutoRetriever',
+    'BM25Retriever',
+]

camel/retrievers/auto_retriever.py

@@ -0,0 +1,319 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import datetime
+import os
+import re
+from pathlib import Path
+from typing import List, Optional, Tuple, Union
+from urllib.parse import urlparse
+
+from camel.embeddings import BaseEmbedding, OpenAIEmbedding
+from camel.retrievers.vector_retriever import VectorRetriever
+from camel.storages import (
+    BaseVectorStorage,
+    MilvusStorage,
+    QdrantStorage,
+    VectorDBQuery,
+)
+from camel.types import StorageType
+
+DEFAULT_TOP_K_RESULTS = 1
+DEFAULT_SIMILARITY_THRESHOLD = 0.75
+
+
+class AutoRetriever:
+    r"""Facilitates the automatic retrieval of information using a
+    query-based approach with pre-defined elements.
+
+    Attributes:
+        url_and_api_key (Optional[Tuple[str, str]]): URL and API key for
+            accessing the vector storage remotely.
+        vector_storage_local_path (Optional[str]): Local path for vector
+            storage, if applicable.
+        storage_type (Optional[StorageType]): The type of vector storage to
+            use. Defaults to `StorageType.MILVUS`.
+        embedding_model (Optional[BaseEmbedding]): Model used for embedding
+            queries and documents. Defaults to `OpenAIEmbedding()`.
+    """
+
+    def __init__(
+        self,
+        url_and_api_key: Optional[Tuple[str, str]] = None,
+        vector_storage_local_path: Optional[str] = None,
+        storage_type: Optional[StorageType] = None,
+        embedding_model: Optional[BaseEmbedding] = None,
+    ):
+        self.storage_type = storage_type or StorageType.MILVUS
+        self.embedding_model = embedding_model or OpenAIEmbedding()
+        self.vector_storage_local_path = vector_storage_local_path
+        self.url_and_api_key = url_and_api_key
+
+    def _initialize_vector_storage(
+        self,
+        collection_name: Optional[str] = None,
+    ) -> BaseVectorStorage:
+        r"""Sets up and returns a vector storage instance with specified parameters.
+
+        Args:
+            collection_name (Optional[str]): Name of the collection in the
+                vector storage.
+
+        Returns:
+            BaseVectorStorage: Configured vector storage instance.
+        """
+        if self.storage_type == StorageType.MILVUS:
+            if self.url_and_api_key is None:
+                raise ValueError(
+                    "URL and API key required for Milvus storage are not"
+                    "provided."
+                )
+            return MilvusStorage(
+                vector_dim=self.embedding_model.get_output_dim(),
+                collection_name=collection_name,
+                url_and_api_key=self.url_and_api_key,
+            )
+
+        if self.storage_type == StorageType.QDRANT:
+            return QdrantStorage(
+                vector_dim=self.embedding_model.get_output_dim(),
+                collection_name=collection_name,
+                path=self.vector_storage_local_path,
+                url_and_api_key=self.url_and_api_key,
+            )
+
+        raise ValueError(
+            f"Unsupported vector storage type: {self.storage_type}"
+        )
+
+    def _collection_name_generator(self, content_input_path: str) -> str:
+        r"""Generates a valid collection name from a given file path or URL.
+
+        Args:
+        content_input_path: str. The input URL or file path from which to
+            generate the collection name.
+
+        Returns:
+            str: A sanitized, valid collection name suitable for use.
+        """
+        # Check path type
+        parsed_url = urlparse(content_input_path)
+        self.is_url = all([parsed_url.scheme, parsed_url.netloc])
+
+        # Convert given path into a collection name, ensuring it only
+        # contains numbers, letters, and underscores
+        if self.is_url:
+            # For URLs, remove https://, replace /, and any characters not
+            # allowed by Milvus with _
+            collection_name = re.sub(
+                r'[^0-9a-zA-Z]+',
+                '_',
+                content_input_path.replace("https://", ""),
+            )
+        else:
+            # For file paths, get the stem and replace spaces with _, also
+            # ensuring only allowed characters are present
+            collection_name = re.sub(
+                r'[^0-9a-zA-Z]+', '_', Path(content_input_path).stem
+            )
+
+        # Ensure the collection name does not start or end with underscore
+        collection_name = collection_name.strip("_")
+        # Limit the maximum length of the collection name to 30 characters
+        collection_name = collection_name[:30]
+        return collection_name
+
+    def _get_file_modified_date_from_file(self, content_input_path: str) -> str:
+        r"""Retrieves the last modified date and time of a given file. This
+        function takes a file path as input and returns the last modified date
+        and time of that file.
+
+        Args:
+            content_input_path (str): The file path of the content whose
+                modified date is to be retrieved.
+
+        Returns:
+            str: The last modified time from file.
+        """
+        mod_time = os.path.getmtime(content_input_path)
+        readable_mod_time = datetime.datetime.fromtimestamp(mod_time).isoformat(
+            timespec='seconds'
+        )
+        return readable_mod_time
+
+    def _get_file_modified_date_from_storage(
+        self, vector_storage_instance: BaseVectorStorage
+    ) -> str:
+        r"""Retrieves the last modified date and time of a given file. This
+        function takes vector storage instance as input and returns the last
+        modified date from the meta data.
+
+        Args:
+            vector_storage_instance (BaseVectorStorage): The vector storage
+                where modified date is to be retrieved from meta data.
+
+        Returns:
+            str: The last modified date from vector storage.
+        """
+
+        # Insert any query to get modified date from vector db
+        # NOTE: Can be optimized when CAMEL vector storage support
+        # direct chunk payload extraction
+        query_vector_any = self.embedding_model.embed(obj="any_query")
+        query_any = VectorDBQuery(query_vector_any, top_k=1)
+        result_any = vector_storage_instance.query(query_any)
+
+        # Extract the file's last modified date from the metadata
+        # in the query result
+        if result_any[0].record.payload is not None:
+            file_modified_date_from_meta = result_any[0].record.payload[
+                "metadata"
+            ]['last_modified']
+        else:
+            raise ValueError(
+                "The vector storage exits but the payload is None,"
+                "please check the collection"
+            )
+
+        return file_modified_date_from_meta
+
+    def run_vector_retriever(
+        self,
+        query: str,
+        content_input_paths: Union[str, List[str]],
+        top_k: int = DEFAULT_TOP_K_RESULTS,
+        similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+        return_detailed_info: bool = False,
+    ) -> str:
+        r"""Executes the automatic vector retriever process using vector storage.
+
+        Args:
+            query (str): Query string for information retriever.
+            content_input_paths (Union[str, List[str]]): Paths to local
+                files or remote URLs.
+            top_k (int, optional): The number of top results to return during
+                retrieve. Must be a positive integer. Defaults to
+                `DEFAULT_TOP_K_RESULTS`.
+            similarity_threshold (float, optional): The similarity threshold
+                for filtering results. Defaults to
+                `DEFAULT_SIMILARITY_THRESHOLD`.
+            return_detailed_info (bool, optional): Whether to return detailed
+                information including similarity score, content path and
+                metadata. Defaults to False.
+
+        Returns:
+            string: By default, returns only the text information. If
+                `return_detailed_info` is `True`, return detailed information
+                including similarity score, content path and metadata.
+
+        Raises:
+            ValueError: If there's an vector storage existing with content
+                name in the vector path but the payload is None. If
+                `content_input_paths` is empty.
+            RuntimeError: If any errors occur during the retrieve process.
+        """
+        if not content_input_paths:
+            raise ValueError("content_input_paths cannot be empty.")
+
+        content_input_paths = (
+            [content_input_paths]
+            if isinstance(content_input_paths, str)
+            else content_input_paths
+        )
+
+        vr = VectorRetriever()
+
+        retrieved_infos = ""
+        retrieved_infos_text = ""
+
+        for content_input_path in content_input_paths:
+            # Generate a valid collection name
+            collection_name = self._collection_name_generator(
+                content_input_path
+            )
+            try:
+                vector_storage_instance = self._initialize_vector_storage(
+                    collection_name
+                )
+
+                # Check the modified time of the input file path, only works
+                # for local path since no standard way for remote url
+                file_is_modified = False  # initialize with a default value
+                if (
+                    vector_storage_instance.status().vector_count != 0
+                    and not self.is_url
+                ):
+                    # Get original modified date from file
+                    modified_date_from_file = (
+                        self._get_file_modified_date_from_file(
+                            content_input_path
+                        )
+                    )
+                    # Get modified date from vector storage
+                    modified_date_from_storage = (
+                        self._get_file_modified_date_from_storage(
+                            vector_storage_instance
+                        )
+                    )
+                    # Determine if the file has been modified since the last
+                    # check
+                    file_is_modified = (
+                        modified_date_from_file != modified_date_from_storage
+                    )
+
+                if (
+                    vector_storage_instance.status().vector_count == 0
+                    or file_is_modified
+                ):
+                    # Clear the vector storage
+                    vector_storage_instance.clear()
+                    # Process and store the content to the vector storage
+                    vr.process(content_input_path, vector_storage_instance)
+                # Retrieve info by given query from the vector storage
+                retrieved_info = vr.query(
+                    query, vector_storage_instance, top_k, similarity_threshold
+                )
+                # Reorganize the retrieved info with original query
+                for info in retrieved_info:
+                    retrieved_infos += "\n" + str(info)
+                    retrieved_infos_text += "\n" + str(info['text'])
+                output = (
+                    "Original Query:"
+                    + "\n"
+                    + "{"
+                    + query
+                    + "}"
+                    + "\n"
+                    + "Retrieved Context:"
+                    + retrieved_infos
+                )
+                output_text = (
+                    "Original Query:"
+                    + "\n"
+                    + "{"
+                    + query
+                    + "}"
+                    + "\n"
+                    + "Retrieved Context:"
+                    + retrieved_infos_text
+                )
+
+            except Exception as e:
+                raise RuntimeError(
+                    f"Error in auto vector retriever processing: {e!s}"
+                ) from e
+
+        if return_detailed_info:
+            return output
+        else:
+            return output_text

camel/retrievers/base.py

@@ -0,0 +1,64 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from abc import ABC, abstractmethod
+from typing import Any, Dict, List
+
+DEFAULT_TOP_K_RESULTS = 1
+
+
+class BaseRetriever(ABC):
+    r"""Abstract base class for implementing various types of information
+    retrievers.
+    """
+
+    @abstractmethod
+    def __init__(self) -> None:
+        pass
+
+    @abstractmethod
+    def process(
+        self,
+        content_input_path: str,
+        chunk_type: str = "chunk_by_title",
+        **kwargs: Any,
+    ) -> None:
+        r"""Processes content from a file or URL, divides it into chunks by
+        using `Unstructured IO`,then stored internally. This method must be
+        called before executing queries with the retriever.
+
+        Args:
+            content_input_path (str): File path or URL of the content to be
+                processed.
+            chunk_type (str): Type of chunking going to apply. Defaults to
+                "chunk_by_title".
+            **kwargs (Any): Additional keyword arguments for content parsing.
+        """
+        pass
+
+    @abstractmethod
+    def query(
+        self, query: str, top_k: int = DEFAULT_TOP_K_RESULTS, **kwargs: Any
+    ) -> List[Dict[str, Any]]:
+        r"""Query the results. Subclasses should implement this
+        method according to their specific needs.
+
+        Args:
+            query (str): Query string for information retriever.
+            top_k (int, optional): The number of top results to return during
+                retriever. Must be a positive integer. Defaults to
+                `DEFAULT_TOP_K_RESULTS`.
+            **kwargs (Any): Flexible keyword arguments for additional
+                parameters, like `similarity_threshold`.
+        """
+        pass

camel/retrievers/bm25_retriever.py

@@ -0,0 +1,149 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Any, Dict, List
+
+import numpy as np
+
+from camel.loaders import UnstructuredIO
+from camel.retrievers import BaseRetriever
+
+DEFAULT_TOP_K_RESULTS = 1
+
+
+class BM25Retriever(BaseRetriever):
+    r"""An implementation of the `BaseRetriever` using the `BM25` model.
+
+    This class facilitates the retriever of relevant information using a
+    query-based approach, it ranks documents based on the occurrence and
+    frequency of the query terms.
+
+    Attributes:
+        bm25 (BM25Okapi): An instance of the BM25Okapi class used for
+            calculating document scores.
+        content_input_path (str): The path to the content that has been
+            processed and stored.
+        chunks (List[Any]): A list of document chunks processed from the
+            input content.
+
+    References:
+        https://github.com/dorianbrown/rank_bm25
+    """
+
+    def __init__(self) -> None:
+        r"""Initializes the BM25Retriever."""
+
+        try:
+            from rank_bm25 import BM25Okapi
+        except ImportError as e:
+            raise ImportError(
+                "Package `rank_bm25` not installed, install by running"
+                " 'pip install rank_bm25'"
+            ) from e
+
+        self.bm25: BM25Okapi = None
+        self.content_input_path: str = ""
+        self.chunks: List[Any] = []
+
+    def process(
+        self,
+        content_input_path: str,
+        chunk_type: str = "chunk_by_title",
+        **kwargs: Any,
+    ) -> None:
+        r"""Processes content from a file or URL, divides it into chunks by
+        using `Unstructured IO`,then stored internally. This method must be
+        called before executing queries with the retriever.
+
+        Args:
+            content_input_path (str): File path or URL of the content to be
+                processed.
+            chunk_type (str): Type of chunking going to apply. Defaults to
+                "chunk_by_title".
+            **kwargs (Any): Additional keyword arguments for content parsing.
+        """
+        from rank_bm25 import BM25Okapi
+
+        # Load and preprocess documents
+        self.content_input_path = content_input_path
+        unstructured_modules = UnstructuredIO()
+        elements = unstructured_modules.parse_file_or_url(
+            content_input_path, **kwargs
+        )
+        self.chunks = unstructured_modules.chunk_elements(
+            chunk_type=chunk_type, elements=elements
+        )
+
+        # Convert chunks to a list of strings for tokenization
+        tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]
+        self.bm25 = BM25Okapi(tokenized_corpus)
+
+    def query(  # type: ignore[override]
+        self,
+        query: str,
+        top_k: int = DEFAULT_TOP_K_RESULTS,
+    ) -> List[Dict[str, Any]]:
+        r"""Executes a query and compiles the results.
+
+        Args:
+            query (str): Query string for information retriever.
+            top_k (int, optional): The number of top results to return during
+                retriever. Must be a positive integer. Defaults to
+                `DEFAULT_TOP_K_RESULTS`.
+
+        Returns:
+            List[Dict[str]]: Concatenated list of the query results.
+
+        Raises:
+            ValueError: If `top_k` is less than or equal to 0, if the BM25
+                model has not been initialized by calling `process_and_store`
+                first.
+
+        Note:
+            `storage` and `kwargs` parameters are included to maintain
+            compatibility with the `BaseRetriever` interface but are not used
+            in this implementation.
+        """
+
+        if top_k <= 0:
+            raise ValueError("top_k must be a positive integer.")
+
+        if self.bm25 is None:
+            raise ValueError(
+                "BM25 model is not initialized. Call `process_and_store`"
+                " first."
+            )
+
+        # Preprocess query similarly to how documents were processed
+        processed_query = query.split(" ")
+        # Retrieve documents based on BM25 scores
+        scores = self.bm25.get_scores(processed_query)
+
+        top_k_indices = np.argpartition(scores, -top_k)[-top_k:]
+
+        formatted_results = []
+        for i in top_k_indices:
+            result_dict = {
+                'similarity score': scores[i],
+                'content path': self.content_input_path,
+                'metadata': self.chunks[i].metadata.to_dict(),
+                'text': str(self.chunks[i]),
+            }
+            formatted_results.append(result_dict)
+
+        # Sort the list of dictionaries by 'similarity score' from high to low
+        formatted_results.sort(
+            key=lambda x: x['similarity score'], reverse=True
+        )
+
+        return formatted_results