camel-ai 0.1.3__py3-none-any.whl → 0.1.4__py3-none-any.whl

camel/retrievers/bm25_retriever.py

@@ -33,8 +33,8 @@ class BM25Retriever(BaseRetriever):
             calculating document scores.
         content_input_path (str): The path to the content that has been
             processed and stored.
-        chunks (List[Any]): A list of document chunks processed from the
-            input content.
+        unstructured_modules (UnstructuredIO): A module for parsing files and
+            URLs and chunking content based on specified parameters.
 
     References:
         https://github.com/dorianbrown/rank_bm25
@@ -47,13 +47,12 @@ class BM25Retriever(BaseRetriever):
             from rank_bm25 import BM25Okapi
         except ImportError as e:
             raise ImportError(
-                "Package `rank_bm25` not installed, install by running"
-                " 'pip install rank_bm25'"
+                "Package `rank_bm25` not installed, install by running 'pip install rank_bm25'"
             ) from e
 
         self.bm25: BM25Okapi = None
         self.content_input_path: str = ""
-        self.chunks: List[Any] = []
+        self.unstructured_modules: UnstructuredIO = UnstructuredIO()
 
     def process(
         self,
@@ -76,11 +75,10 @@ class BM25Retriever(BaseRetriever):
 
         # Load and preprocess documents
         self.content_input_path = content_input_path
-        unstructured_modules = UnstructuredIO()
-        elements = unstructured_modules.parse_file_or_url(
+        elements = self.unstructured_modules.parse_file_or_url(
             content_input_path, **kwargs
         )
-        self.chunks = unstructured_modules.chunk_elements(
+        self.chunks = self.unstructured_modules.chunk_elements(
             chunk_type=chunk_type, elements=elements
         )
 
@@ -88,7 +86,7 @@ class BM25Retriever(BaseRetriever):
         tokenized_corpus = [str(chunk).split(" ") for chunk in self.chunks]
         self.bm25 = BM25Okapi(tokenized_corpus)
 
-    def query(  # type: ignore[override]
+    def query(
         self,
         query: str,
         top_k: int = DEFAULT_TOP_K_RESULTS,
@@ -106,22 +104,15 @@ class BM25Retriever(BaseRetriever):
 
         Raises:
             ValueError: If `top_k` is less than or equal to 0, if the BM25
-                model has not been initialized by calling `process_and_store`
+                model has not been initialized by calling `process`
                 first.
-
-        Note:
-            `storage` and `kwargs` parameters are included to maintain
-            compatibility with the `BaseRetriever` interface but are not used
-            in this implementation.
         """
 
         if top_k <= 0:
             raise ValueError("top_k must be a positive integer.")
-
-        if self.bm25 is None:
+        if self.bm25 is None or not self.chunks:
             raise ValueError(
-                "BM25 model is not initialized. Call `process_and_store`"
-                " first."
+                "BM25 model is not initialized. Call `process` first."
             )
 
         # Preprocess query similarly to how documents were processed

camel/retrievers/cohere_rerank_retriever.py

@@ -0,0 +1,108 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from typing import Any, Dict, List, Optional
+
+from camel.retrievers import BaseRetriever
+
+DEFAULT_TOP_K_RESULTS = 1
+
+
+class CohereRerankRetriever(BaseRetriever):
+    r"""An implementation of the `BaseRetriever` using the `Cohere Re-ranking`
+    model.
+
+    Attributes:
+        model_name (str): The model name to use for re-ranking.
+        api_key (Optional[str]): The API key for authenticating with the
+            Cohere service.
+
+    References:
+        https://txt.cohere.com/rerank/
+    """
+
+    def __init__(
+        self,
+        model_name: str = "rerank-multilingual-v2.0",
+        api_key: Optional[str] = None,
+    ) -> None:
+        r"""Initializes an instance of the CohereRerankRetriever. This
+        constructor sets up a client for interacting with the Cohere API using
+        the specified model name and API key. If the API key is not provided,
+        it attempts to retrieve it from the COHERE_API_KEY environment
+        variable.
+
+        Args:
+            model_name (str): The name of the model to be used for re-ranking.
+                Defaults to 'rerank-multilingual-v2.0'.
+            api_key (Optional[str]): The API key for authenticating requests
+                to the Cohere API. If not provided, the method will attempt to
+                retrieve the key from the environment variable
+                'COHERE_API_KEY'.
+
+        Raises:
+            ImportError: If the 'cohere' package is not installed.
+            ValueError: If the API key is neither passed as an argument nor
+                set in the environment variable.
+        """
+
+        try:
+            import cohere
+        except ImportError as e:
+            raise ImportError("Package 'cohere' is not installed") from e
+
+        try:
+            self.api_key = api_key or os.environ["COHERE_API_KEY"]
+        except ValueError as e:
+            raise ValueError(
+                "Must pass in cohere api key or specify via COHERE_API_KEY environment variable."
+            ) from e
+
+        self.co = cohere.Client(self.api_key)
+        self.model_name = model_name
+
+    def query(
+        self,
+        query: str,
+        retrieved_result: List[Dict[str, Any]],
+        top_k: int = DEFAULT_TOP_K_RESULTS,
+    ) -> List[Dict[str, Any]]:
+        r"""Queries and compiles results using the Cohere re-ranking model.
+
+        Args:
+            query (str): Query string for information retriever.
+            retrieved_result (List[Dict[str, Any]]): The content to be
+                re-ranked, should be the output from `BaseRetriever` like
+                `VectorRetriever`.
+            top_k (int, optional): The number of top results to return during
+                retriever. Must be a positive integer. Defaults to
+                `DEFAULT_TOP_K_RESULTS`.
+
+        Returns:
+            List[Dict[str, Any]]: Concatenated list of the query results.
+        """
+        rerank_results = self.co.rerank(
+            query=query,
+            documents=retrieved_result,
+            top_n=top_k,
+            model=self.model_name,
+        )
+        formatted_results = []
+        for i in range(0, len(rerank_results.results)):
+            selected_chunk = retrieved_result[rerank_results[i].index]
+            selected_chunk['similarity score'] = rerank_results[
+                i
+            ].relevance_score
+            formatted_results.append(selected_chunk)
+        return formatted_results

camel/retrievers/vector_retriever.py

@@ -14,9 +14,14 @@
 from typing import Any, Dict, List, Optional
 
 from camel.embeddings import BaseEmbedding, OpenAIEmbedding
-from camel.functions import UnstructuredIO
+from camel.loaders import UnstructuredIO
 from camel.retrievers.base import BaseRetriever
-from camel.storages import BaseVectorStorage, VectorDBQuery, VectorRecord
+from camel.storages import (
+    BaseVectorStorage,
+    QdrantStorage,
+    VectorDBQuery,
+    VectorRecord,
+)
 
 DEFAULT_TOP_K_RESULTS = 1
 DEFAULT_SIMILARITY_THRESHOLD = 0.75
@@ -32,21 +37,41 @@ class VectorRetriever(BaseRetriever):
     Attributes:
         embedding_model (BaseEmbedding): Embedding model used to generate
             vector embeddings.
+        storage (BaseVectorStorage): Vector storage to query.
+        similarity_threshold (float, optional): The similarity threshold
+            for filtering results. Defaults to `DEFAULT_SIMILARITY_THRESHOLD`.
+        unstructured_modules (UnstructuredIO): A module for parsing files and
+            URLs and chunking content based on specified parameters.
     """
 
-    def __init__(self, embedding_model: Optional[BaseEmbedding] = None) -> None:
+    def __init__(
+        self,
+        similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
+        embedding_model: Optional[BaseEmbedding] = None,
+        storage: Optional[BaseVectorStorage] = None,
+    ) -> None:
         r"""Initializes the retriever class with an optional embedding model.
 
         Args:
+            similarity_threshold (float, optional): The similarity threshold
+                for filtering results. Defaults to
+                `DEFAULT_SIMILARITY_THRESHOLD`.
             embedding_model (Optional[BaseEmbedding]): The embedding model
                 instance. Defaults to `OpenAIEmbedding` if not provided.
+            storage (BaseVectorStorage): Vector storage to query.
         """
         self.embedding_model = embedding_model or OpenAIEmbedding()
+        self.storage = (
+            storage
+            if storage is not None
+            else QdrantStorage(vector_dim=self.embedding_model.get_output_dim())
+        )
+        self.similarity_threshold = similarity_threshold
+        self.unstructured_modules: UnstructuredIO = UnstructuredIO()
 
-    def process(  # type: ignore[override]
+    def process(
         self,
         content_input_path: str,
-        storage: BaseVectorStorage,
         chunk_type: str = "chunk_by_title",
         **kwargs: Any,
     ) -> None:
@@ -59,12 +84,13 @@ class VectorRetriever(BaseRetriever):
                 processed.
             chunk_type (str): Type of chunking going to apply. Defaults to
                 "chunk_by_title".
-            **kwargs (Any): Additional keyword arguments for elements chunking.
+            **kwargs (Any): Additional keyword arguments for content parsing.
         """
-        unstructured_modules = UnstructuredIO()
-        elements = unstructured_modules.parse_file_or_url(content_input_path)
-        chunks = unstructured_modules.chunk_elements(
-            chunk_type=chunk_type, elements=elements, **kwargs
+        elements = self.unstructured_modules.parse_file_or_url(
+            content_input_path, **kwargs
+        )
+        chunks = self.unstructured_modules.chunk_elements(
+            chunk_type=chunk_type, elements=elements
         )
         # Iterate to process and store embeddings, set batch of 50
         for i in range(0, len(chunks), 50):
@@ -90,28 +116,20 @@ class VectorRetriever(BaseRetriever):
                     VectorRecord(vector=vector, payload=combined_dict)
                 )
 
-            storage.add(records=records)
+            self.storage.add(records=records)
 
-    def query(  # type: ignore[override]
+    def query(
         self,
         query: str,
-        storage: BaseVectorStorage,
         top_k: int = DEFAULT_TOP_K_RESULTS,
-        similarity_threshold: float = DEFAULT_SIMILARITY_THRESHOLD,
-        **kwargs: Any,
     ) -> List[Dict[str, Any]]:
         r"""Executes a query in vector storage and compiles the retrieved
         results into a dictionary.
 
         Args:
             query (str): Query string for information retriever.
-            storage (BaseVectorStorage): Vector storage to query.
             top_k (int, optional): The number of top results to return during
                 retriever. Must be a positive integer. Defaults to 1.
-            similarity_threshold (float, optional): The similarity threshold
-                for filtering results. Defaults to 0.75.
-            **kwargs (Any): Additional keyword arguments for vector storage
-                query.
 
         Returns:
             List[Dict[str, Any]]: Concatenated list of the query results.
@@ -125,23 +143,22 @@ class VectorRetriever(BaseRetriever):
             raise ValueError("top_k must be a positive integer.")
 
         # Load the storage incase it's hosted remote
-        storage.load()
+        self.storage.load()
 
         query_vector = self.embedding_model.embed(obj=query)
         db_query = VectorDBQuery(query_vector=query_vector, top_k=top_k)
-        query_results = storage.query(query=db_query, **kwargs)
+        query_results = self.storage.query(query=db_query)
 
         if query_results[0].record.payload is None:
             raise ValueError(
-                "Payload of vector storage is None, please check"
-                " the collection."
+                "Payload of vector storage is None, please check the collection."
             )
 
         # format the results
         formatted_results = []
         for result in query_results:
             if (
-                result.similarity >= similarity_threshold
+                result.similarity >= self.similarity_threshold
                 and result.record.payload is not None
             ):
                 result_dict = {
@@ -160,7 +177,7 @@ class VectorRetriever(BaseRetriever):
             return [
                 {
                     'text': f"""No suitable information retrieved from {content_path} \
-                with similarity_threshold = {similarity_threshold}."""
+                with similarity_threshold = {self.similarity_threshold}."""
                 }
             ]
         return formatted_results

camel/storages/vectordb_storages/qdrant.py

@@ -185,7 +185,9 @@ class QdrantStorage(BaseVectorStorage):
             VectorDistance.COSINE: Distance.COSINE,
             VectorDistance.EUCLIDEAN: Distance.EUCLID,
         }
-        self._client.recreate_collection(
+        # Since `recreate_collection` method will be removed in the future
+        # by Qdrant, `create_collection` is recommended instead.
+        self._client.create_collection(
             collection_name=collection_name,
             vectors_config=VectorParams(
                 size=size,

camel/toolkits/__init__.py

@@ -0,0 +1,21 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from .base import BaseToolkit
+from .github_toolkit import GithubToolkit
+
+__all__ = [
+    'BaseToolkit',
+    'GithubToolkit',
+]

camel/toolkits/base.py

@@ -0,0 +1,22 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from typing import List
+
+from camel.functions import OpenAIFunction
+
+
+class BaseToolkit:
+    def get_tools(self) -> List[OpenAIFunction]:
+        raise NotImplementedError("Subclasses must implement this method.")

camel/toolkits/github_toolkit.py

@@ -0,0 +1,245 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from dataclasses import dataclass
+from typing import List, Optional
+
+from camel.functions import OpenAIFunction
+
+from .base import BaseToolkit
+
+
+@dataclass
+class GithubIssue:
+    r"""Represents a GitHub issue.
+
+    Attributes:
+        title (str): The title of the issue.
+        body (str): The body/content of the issue.
+        number (int): The issue number.
+        file_path (str): The path of the file associated with the issue.
+        file_content (str): The content of the file associated with the issue.
+    """
+
+    def __init__(
+        self,
+        title: str,
+        body: str,
+        number: int,
+        file_path: str,
+        file_content: str,
+    ) -> None:
+        r"""Initialize a GithubIssue object.
+
+        Args:
+            title (str): The title of the GitHub issue.
+            body (str): The body/content of the GitHub issue.
+            number (int): The issue number.
+            file_path (str): The path of the file associated with the issue.
+            file_content (str): The content of the file associated with the issue.
+        """
+        self.title = title
+        self.body = body
+        self.number = number
+        self.file_path = file_path
+        self.file_content = file_content
+
+    def summary(self) -> str:
+        r"""Returns a summary of the issue.
+
+        Returns:
+            str: A string containing the title, body, number, file path, and file content of the issue.
+        """
+        return (
+            f"Title: {self.title}\n"
+            f"Body: {self.body}\n"
+            f"Number: {self.number}\n"
+            f"File Path: {self.file_path}\n"
+            f"File Content: {self.file_content}"
+        )
+
+
+class GithubToolkit(BaseToolkit):
+    r"""A class representing a toolkit for interacting with GitHub repositories.
+
+    This class provides methods for retrieving open issues, retrieving specific issues,
+    and creating pull requests in a GitHub repository.
+
+    Args:
+        repo_name (str): The name of the GitHub repository.
+        access_token (str, optional): The access token to authenticate with GitHub.
+            If not provided, it will be obtained using the `get_github_access_token` method.
+    """
+
+    def __init__(
+        self, repo_name: str, access_token: Optional[str] = None
+    ) -> None:
+        r"""Initializes a new instance of the GitHubToolkit class.
+
+        Args:
+            repo_name (str): The name of the GitHub repository.
+            access_token (str, optional): The access token to authenticate with GitHub.
+                If not provided, it will be obtained using the `get_github_access_token` method.
+        """
+        if access_token is None:
+            access_token = self.get_github_access_token()
+
+        try:
+            from github import Auth, Github
+        except ImportError:
+            raise ImportError(
+                "Please install `github` first. You can install it by running "
+                "`pip install wikipedia`."
+            )
+        self.github = Github(auth=Auth.Token(access_token))
+        self.repo = self.github.get_repo(repo_name)
+
+    def get_tools(self) -> List[OpenAIFunction]:
+        r"""Returns a list of OpenAIFunction objects representing the functions in the toolkit.
+
+        Returns:
+            List[OpenAIFunction]: A list of OpenAIFunction objects representing the functions in the toolkit.
+        """
+        return [
+            OpenAIFunction(self.retrieve_issue_list),
+            OpenAIFunction(self.retrieve_issue),
+            OpenAIFunction(self.create_pull_request),
+        ]
+
+    def get_github_access_token(self) -> str:
+        r"""Retrieve the GitHub access token from environment variables.
+
+        Returns:
+            str: A string containing the GitHub access token.
+
+        Raises:
+            ValueError: If the API key or secret is not found in the environment variables.
+        """
+        # Get `GITHUB_ACCESS_TOKEN` here: https://github.com/settings/tokens
+        GITHUB_ACCESS_TOKEN = os.environ.get("GITHUB_ACCESS_TOKEN")
+
+        if not GITHUB_ACCESS_TOKEN:
+            raise ValueError(
+                "`GITHUB_ACCESS_TOKEN` not found in environment variables. Get it "
+                "here: `https://github.com/settings/tokens`."
+            )
+        return GITHUB_ACCESS_TOKEN
+
+    def retrieve_issue_list(self) -> List[GithubIssue]:
+        r"""Retrieve a list of open issues from the repository.
+
+        Returns:
+            A list of GithubIssue objects representing the open issues.
+        """
+        issues = self.repo.get_issues(state='open')
+        return [
+            GithubIssue(
+                title=issue.title,
+                body=issue.body,
+                number=issue.number,
+                file_path=issue.labels[
+                    0
+                ].name,  # for now we require file path to be the first label in the PR
+                file_content=self.retrieve_file_content(issue.labels[0].name),
+            )
+            for issue in issues
+            if not issue.pull_request
+        ]
+
+    def retrieve_issue(self, issue_number: int) -> Optional[str]:
+        r"""Retrieves an issue from a GitHub repository.
+
+        This function retrieves an issue from a specified repository using the
+        issue number.
+
+        Args:
+            issue_number (int): The number of the issue to retrieve.
+
+        Returns:
+            str: A formatted report of the retrieved issue.
+        """
+        issues = self.retrieve_issue_list()
+        for issue in issues:
+            if issue.number == issue_number:
+                return issue.summary()
+        return None
+
+    def create_pull_request(
+        self,
+        file_path: str,
+        new_content: str,
+        pr_title: str,
+        body: str,
+        branch_name: str,
+    ) -> str:
+        r"""Creates a pull request.
+
+        This function creates a pull request in specified repository, which updates a
+        file in the specific path with new content. The pull request description
+        contains information about the issue title and number.
+
+        Args:
+            file_path (str): The path of the file to be updated in the repository.
+            new_content (str): The specified new content of the specified file.
+            pr_title (str): The title of the issue that is solved by this pull request.
+            body (str): The commit message for the pull request.
+            branch_name (str): The name of the branch to create and submit the pull request from.
+
+        Returns:
+            str: A formatted report of whether the pull request was created successfully or not.
+        """
+        sb = self.repo.get_branch(self.repo.default_branch)
+        self.repo.create_git_ref(
+            ref=f"refs/heads/{branch_name}", sha=sb.commit.sha
+        )
+
+        file = self.repo.get_contents(file_path)
+        from github.ContentFile import ContentFile
+
+        if isinstance(file, ContentFile):
+            self.repo.update_file(
+                file.path, body, new_content, file.sha, branch=branch_name
+            )
+            pr = self.repo.create_pull(
+                title=pr_title,
+                body=body,
+                head=branch_name,
+                base=self.repo.default_branch,
+            )
+
+            if pr is not None:
+                return f"Title: {pr.title}\n" f"Body: {pr.body}\n"
+            else:
+                return "Failed to create pull request."
+        else:
+            raise ValueError("PRs with multiple files aren't supported yet.")
+
+    def retrieve_file_content(self, file_path: str) -> str:
+        r"""Retrieves the content of a file from the GitHub repository.
+
+        Args:
+            file_path (str): The path of the file to retrieve.
+
+        Returns:
+            str: The decoded content of the file.
+        """
+        file_content = self.repo.get_contents(file_path)
+
+        from github.ContentFile import ContentFile
+
+        if isinstance(file_content, ContentFile):
+            return file_content.decoded_content.decode()
+        else:
+            raise ValueError("PRs with multiple files aren't supported yet.")

camel/types/__init__.py

@@ -12,16 +12,19 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from .enums import (
+    AudioModelType,
     EmbeddingModelType,
     ModelType,
     OpenAIBackendRole,
     OpenAIImageDetailType,
     OpenAIImageType,
+    OpenAPIName,
     RoleType,
     StorageType,
     TaskType,
     TerminationMode,
     VectorDistance,
+    VoiceType,
 )
 from .openai_types import (
     ChatCompletion,
@@ -57,4 +60,7 @@ __all__ = [
     'CompletionUsage',
     'OpenAIImageType',
     'OpenAIImageDetailType',
+    'OpenAPIName',
+    'AudioModelType',
+    'VoiceType',
 ]