camel-ai 0.1.6.2__py3-none-any.whl → 0.1.6.4__py3-none-any.whl

camel/storages/object_storages/azure_blob.py

@@ -0,0 +1,166 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from pathlib import Path, PurePath
+from typing import Optional, Tuple
+from warnings import warn
+
+from camel.loaders import File
+from camel.storages.object_storages.base import BaseObjectStorage
+
+
+class AzureBlobStorage(BaseObjectStorage):
+    r"""A class to connect to Azure Blob Storage. It will connect to one
+    container in the storage account.
+
+    Args:
+        storage_account_name (str): The name of the storage account.
+        container_name (str): The name of the container.
+        access_key (Optional[str], optional): The access key of the storage
+            account. Defaults to None.
+
+    References:
+        https://azure.microsoft.com/en-us/products/storage/blobs
+    """
+
+    def __init__(
+        self,
+        storage_account_name: str,
+        container_name: str,
+        create_if_not_exists: bool = True,
+        access_key: Optional[str] = None,
+    ) -> None:
+        access_key = access_key or os.getenv("AZURE_ACCESS_KEY")
+        self._create_if_not_exists = create_if_not_exists
+
+        if not access_key:
+            warn("AZURE_ACCESS_KEY not provided.")
+            # Make all the empty values None
+            access_key = None
+
+        from azure.storage.blob import ContainerClient
+
+        self._client = ContainerClient(
+            account_url="https://"
+            f"{storage_account_name}.blob.core.windows.net",
+            credential=access_key,
+            container_name=container_name,
+        )
+
+        self._prepare_and_check()
+
+    def _prepare_and_check(self) -> None:
+        r"""Check privileges and existence of the container."""
+        from azure.core.exceptions import ClientAuthenticationError
+
+        try:
+            exists = self._client.exists()
+            if not exists and self._create_if_not_exists:
+                self._client.create_container()
+                warn(
+                    f"Container {self._client.container_name} not found. "
+                    f"Automatically created."
+                )
+            elif not exists:
+                raise FileNotFoundError(
+                    f"Failed to access container {self._client.container_name}"
+                    f": Not found."
+                )
+        except ClientAuthenticationError:
+            raise PermissionError(
+                f"Failed to access container {self._client.container_name}: "
+                f"No permission."
+            )
+
+    @staticmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        r"""Canonicalize file path for Azure Blob Storage.
+
+        Args:
+            file_path (PurePath): The path to be canonicalized.
+
+        Returns:
+            Tuple[str, str]: The canonicalized file key and file name.
+        """
+        # for Azure, both slash and backslash will be treated as separator
+        filename = file_path.name
+        if "\\" in filename:
+            raise ValueError(
+                "Azure Blob Storage does not support backslash in filename."
+            )
+        return file_path.as_posix(), filename
+
+    def _put_file(self, file_key: str, file: File) -> None:
+        r"""Put a file to the Azure Blob Storage container.
+
+        Args:
+            file_key (str): The path to the object in the container.
+            file (File): The file to be uploaded.
+        """
+        self._client.upload_blob(
+            name=file_key, data=file.raw_bytes, overwrite=True
+        )
+
+    def _get_file(self, file_key: str, filename: str) -> File:
+        r"""Get a file from the Azure Blob Storage container.
+
+        Args:
+            file_key (str): The path to the object in the container.
+            filename (str): The name of the file.
+
+        Returns:
+            File: The object from the container.
+        """
+        raw_bytes = self._client.download_blob(file_key).readall()
+        file = File.create_file_from_raw_bytes(raw_bytes, filename)
+        return file
+
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Upload a local file to the Azure Blob Storage container.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_key (str): The path to the object in the container.
+        """
+        with open(local_file_path, "rb") as f:
+            self._client.upload_blob(
+                name=remote_file_key, data=f, overwrite=True
+            )
+
+    def _download_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Download a file from the Azure Blob Storage container to the local
+        system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_key (str): The key of the object in the container.
+        """
+        with open(local_file_path, "wb") as f:
+            f.write(self._client.download_blob(remote_file_key).readall())
+
+    def _object_exists(self, file_key: str) -> bool:
+        r"""
+        Check if the object exists in the Azure Blob Storage container.
+
+        Args:
+            file_key: The key of the object in the container.
+
+        Returns:
+            bool: Whether the object exists in the container.
+        """
+        return self._client.get_blob_client(file_key).exists()

camel/storages/object_storages/base.py

@@ -0,0 +1,115 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from abc import ABC, abstractmethod
+from pathlib import Path, PurePath
+from typing import Tuple
+
+from camel.loaders import File
+
+
+class BaseObjectStorage(ABC):
+    def object_exists(self, file_path: PurePath) -> bool:
+        r"""Check if the object exists in the storage.
+
+        Args:
+            file_path (PurePath): The path to the object in the storage.
+
+        Returns:
+            bool: True if the object exists, False otherwise.
+        """
+        file_key, _ = self.canonicalize_path(file_path)
+        return self._object_exists(file_key)
+
+    @staticmethod
+    @abstractmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        pass
+
+    def put_file(self, file_path: PurePath, file: File) -> None:
+        r"""Put a file to the object storage.
+
+        Args:
+            file_path (PurePath): The path to the object in the storage.
+            file (File): The file to be put.
+        """
+        file_key, _ = self.canonicalize_path(file_path)
+        self._put_file(file_key, file)
+
+    def get_file(self, file_path: PurePath) -> File:
+        r"""Get a file from the object storage.
+
+        Args:
+            file_path (PurePath): The path to the object in the storage.
+
+        Returns:
+            File: The file object get from the storage.
+        """
+        file_key, filename = self.canonicalize_path(file_path)
+        return self._get_file(file_key, filename)
+
+    def upload_file(
+        self, local_file_path: Path, remote_file_path: PurePath
+    ) -> None:
+        r"""Upload a local file to the object storage.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_path (PurePath): The path to the object in storage.
+        """
+        file_key, _ = self.canonicalize_path(remote_file_path)
+        # check if the local file exists
+        if not local_file_path.exists():
+            raise FileNotFoundError(
+                f"Local file {local_file_path} does not exist."
+            )
+        self._upload_file(local_file_path, file_key)
+
+    def download_file(
+        self, local_file_path: Path, remote_file_path: PurePath
+    ) -> None:
+        r"""Download a file from the object storage to the local system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_path (PurePath): The path to the object in storage.
+        """
+        file_key, _ = self.canonicalize_path(remote_file_path)
+        self._download_file(local_file_path, file_key)
+
+    @abstractmethod
+    def _put_file(self, file_key: str, file: File) -> None:
+        pass
+
+    @abstractmethod
+    def _get_file(self, file_key: str, filename: str) -> File:
+        pass
+
+    @abstractmethod
+    def _object_exists(self, file_key: str) -> bool:
+        pass
+
+    @abstractmethod
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def _download_file(
+        self,
+        local_file_path: Path,
+        remote_file_key: str,
+    ) -> None:
+        pass

camel/storages/object_storages/google_cloud.py

@@ -0,0 +1,152 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from pathlib import Path, PurePath
+from typing import Tuple
+from warnings import warn
+
+from camel.loaders import File
+from camel.storages.object_storages.base import BaseObjectStorage
+
+
+class GoogleCloudStorage(BaseObjectStorage):
+    r"""A class to connect to Google Cloud Storage. It will connect to one
+    bucket in the storage account.
+
+    Note that Google Cloud Storage does not support api key authentication.
+    Therefore, before using this class, you need to log in with gcloud command
+    line tool and save the credentials first.
+
+    Args:
+        bucket_name (str): The name of the bucket.
+        create_if_not_exists (bool, optional): Whether to create the bucket if
+            it does not exist. Defaults to True.
+        anonymous (bool, optional): Whether to use anonymous access. Defaults
+            to False.
+
+    References:
+        https://cloud.google.com/storage
+
+        https://cloud.google.com/docs/authentication/api-keys
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        create_if_not_exists: bool = True,
+        anonymous: bool = False,
+    ) -> None:
+        from google.cloud import storage
+
+        self.create_if_not_exists = create_if_not_exists
+
+        if anonymous:
+            client = storage.Client.create_anonymous_client()
+        else:
+            client = storage.Client()
+        self._client = client.bucket(bucket_name)
+
+        self._prepare_and_check()
+
+    @staticmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        r"""Canonicalize the path for Google Cloud Storage.
+
+        Args:
+            file_path (PurePath): The path to be canonicalized.
+
+        Returns:
+            Tuple[str, str]: The canonicalized file key and file name.
+        """
+        return file_path.as_posix(), file_path.name
+
+    def _prepare_and_check(self) -> None:
+        r"""Check privileges and existence of the bucket."""
+        from google.auth.exceptions import InvalidOperation
+
+        try:
+            exists = self._client.exists()
+            if not exists and self.create_if_not_exists:
+                self._client.create()
+                warn(
+                    f"Bucket {self._client.name} not found. Automatically "
+                    f"created."
+                )
+            elif not exists:
+                raise FileNotFoundError(
+                    f"Failed to access bucket {self._client.name}: Not found."
+                )
+        except InvalidOperation:
+            raise PermissionError(
+                f"Failed to access bucket {self._client.name}: No permission."
+            )
+
+    def _put_file(self, file_key: str, file: File) -> None:
+        r"""Put a file to the GCloud bucket.
+
+        Args:
+            file_key (str): The path to the object in the bucket.
+            file (File): The file to be uploaded.
+        """
+        self._client.blob(file_key).upload_from_string(file.raw_bytes)
+
+    def _get_file(self, file_key: str, filename: str) -> File:
+        r"""Get a file from the GCloud bucket.
+
+        Args:
+            file_key (str): The path to the object in the bucket.
+            filename (str): The name of the file.
+
+        Returns:
+            File: The object from the S3 bucket.
+        """
+        raw_bytes = self._client.get_blob(file_key).download_as_bytes()
+        return File.create_file_from_raw_bytes(raw_bytes, filename)
+
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Upload a local file to the GCloud bucket.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_key (str): The path to the object in the bucket.
+        """
+        self._client.blob(remote_file_key).upload_from_filename(
+            local_file_path
+        )
+
+    def _download_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Download a file from the GCloud bucket to the local system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_key (str): The key of the object in the bucket.
+        """
+        self._client.get_blob(remote_file_key).download_to_filename(
+            local_file_path
+        )
+
+    def _object_exists(self, file_key: str) -> bool:
+        r"""
+        Check if the object exists in the GCloud bucket.
+
+        Args:
+            file_key: The key of the object in the bucket.
+
+        Returns:
+            bool: Whether the object exists in the bucket.
+        """
+        return self._client.blob(file_key).exists()

camel/toolkits/retrieval_toolkit.py

@@ -27,7 +27,7 @@ class RetrievalToolkit(BaseToolkit):
     """
 
     def information_retrieval(
-        self, query: str, content_input_paths: Union[str, List[str]]
+        self, query: str, contents: Union[str, List[str]]
     ) -> str:
         r"""Retrieves information from a local vector storage based on the
         specified query. This function connects to a local vector storage
@@ -37,8 +37,8 @@ class RetrievalToolkit(BaseToolkit):
 
         Args:
             query (str): The question or query for which an answer is required.
-            content_input_paths (Union[str, List[str]]): Paths to local
-                files or remote URLs.
+            contents (Union[str, List[str]]): Local file paths, remote URLs or
+                string contents.
 
         Returns:
             str: The information retrieved in response to the query, aggregated
@@ -47,7 +47,7 @@ class RetrievalToolkit(BaseToolkit):
         Example:
             # Retrieve information about CAMEL AI.
             information_retrieval(query = "what is CAMEL AI?",
-                                content_input_paths="https://www.camel-ai.org/")
+                                contents="https://www.camel-ai.org/")
         """
         auto_retriever = AutoRetriever(
             vector_storage_local_path="camel/temp_storage",
@@ -55,7 +55,7 @@ class RetrievalToolkit(BaseToolkit):
         )
 
         retrieved_info = auto_retriever.run_vector_retriever(
-            query=query, content_input_paths=content_input_paths, top_k=3
+            query=query, contents=contents, top_k=3
         )
         return retrieved_info
 

camel/toolkits/search_toolkit.py

@@ -65,7 +65,7 @@ class SearchToolkit(BaseToolkit):
         return result
 
     def search_duckduckgo(
-        self, query: str, source: str = "text", max_results: int = 10
+        self, query: str, source: str = "text", max_results: int = 5
     ) -> List[Dict[str, Any]]:
         r"""Use DuckDuckGo search engine to search information for
         the given query.
@@ -78,7 +78,7 @@ class SearchToolkit(BaseToolkit):
             query (str): The query to be searched.
             source (str): The type of information to query (e.g., "text",
                 "images", "videos"). Defaults to "text".
-            max_results (int): Max number of results, defaults to `10`.
+            max_results (int): Max number of results, defaults to `5`.
 
         Returns:
             List[Dict[str, Any]]: A list of dictionaries where each dictionary
@@ -152,7 +152,7 @@ class SearchToolkit(BaseToolkit):
         return responses
 
     def search_google(
-        self, query: str, num_result_pages: int = 10
+        self, query: str, num_result_pages: int = 5
     ) -> List[Dict[str, Any]]:
         r"""Use Google search engine to search information for the given query.
 
@@ -196,7 +196,7 @@ class SearchToolkit(BaseToolkit):
         # Different language may get different result
         search_language = "en"
         # How many pages to return
-        num_result_pages = 10
+        num_result_pages = num_result_pages
         # Constructing the URL
         # Doc: https://developers.google.com/custom-search/v1/using_rest
         url = (

camel/types/enums.py

@@ -448,6 +448,7 @@ class ModelPlatformType(Enum):
     GEMINI = "gemini"
     VLLM = "vllm"
     MISTRAL = "mistral"
+    OPENAICOMPATIBILITYMODEL = "openai-compatibility-model"
 
     @property
     def is_openai(self) -> bool:
@@ -499,6 +500,12 @@ class ModelPlatformType(Enum):
         r"""Returns whether this platform is opensource."""
         return self is ModelPlatformType.OPENSOURCE
 
+    @property
+    def is_openai_compatibility_model(self) -> bool:
+        r"""Returns whether this is a platform supporting openai
+        compatibility"""
+        return self is ModelPlatformType.OPENAICOMPATIBILITYMODEL
+
     @property
     def is_gemini(self) -> bool:
         r"""Returns whether this platform is Gemini."""

camel/utils/token_counting.py

@@ -26,7 +26,9 @@ from PIL import Image
 from camel.types import ModelType, OpenAIImageType, OpenAIVisionDetailType
 
 if TYPE_CHECKING:
-    from mistral_common.protocol.instruct.request import ChatCompletionRequest
+    from mistral_common.protocol.instruct.request import (  # type:ignore[import-not-found]
+        ChatCompletionRequest,
+    )
 
     from camel.messages import OpenAIMessage
 
@@ -517,7 +519,9 @@ class MistralTokenCounter(BaseTokenCounter):
             model_type (ModelType): Model type for which tokens will be
                 counted.
         """
-        from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+        from mistral_common.tokens.tokenizers.mistral import (  # type:ignore[import-not-found]
+            MistralTokenizer,
+        )
 
         self.model_type = model_type
 
@@ -565,7 +569,7 @@ class MistralTokenCounter(BaseTokenCounter):
         """
 
         from mistral_common.protocol.instruct.request import (
-            ChatCompletionRequest,
+            ChatCompletionRequest,  # type:ignore[import-not-found]
         )
 
         mistral_request = ChatCompletionRequest(  # type: ignore[type-var]

{camel_ai-0.1.6.2.dist-info → camel_ai-0.1.6.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: camel-ai
-Version: 0.1.6.2
+Version: 0.1.6.4
 Summary: Communicative Agents for AI Society Study
 Home-page: https://www.camel-ai.org/
 License: Apache-2.0
@@ -18,6 +18,7 @@ Provides-Extra: graph-storages
 Provides-Extra: huggingface-agent
 Provides-Extra: kv-stroages
 Provides-Extra: model-platforms
+Provides-Extra: object-storages
 Provides-Extra: retrievers
 Provides-Extra: test
 Provides-Extra: tools
@@ -26,7 +27,9 @@ Requires-Dist: PyMuPDF (>=1.22.5,<2.0.0) ; extra == "tools" or extra == "all"
 Requires-Dist: accelerate (>=0,<1) ; extra == "huggingface-agent" or extra == "all"
 Requires-Dist: agentops (>=0.3.6,<0.4.0) ; extra == "tools" or extra == "all"
 Requires-Dist: anthropic (>=0.29.0,<0.30.0)
+Requires-Dist: azure-storage-blob (>=12.21.0,<13.0.0) ; extra == "object-storages" or extra == "all"
 Requires-Dist: beautifulsoup4 (>=4,<5) ; extra == "tools" or extra == "all"
+Requires-Dist: boto3 (>=1.34.149,<2.0.0) ; extra == "object-storages" or extra == "all"
 Requires-Dist: cohere (>=4.56,<5.0) ; extra == "retrievers" or extra == "all"
 Requires-Dist: colorama (>=0,<1)
 Requires-Dist: curl_cffi (==0.6.2)
@@ -39,6 +42,7 @@ Requires-Dist: docx2txt (>=0.8,<0.9) ; extra == "tools" or extra == "all"
 Requires-Dist: duckduckgo-search (>=6.1.0,<7.0.0) ; extra == "tools" or extra == "all"
 Requires-Dist: eval-type-backport (==0.2.0)
 Requires-Dist: firecrawl-py (>=0.0.20,<0.0.21) ; extra == "tools" or extra == "all"
+Requires-Dist: google-cloud-storage (>=2.18.0,<3.0.0) ; extra == "object-storages" or extra == "all"
 Requires-Dist: google-generativeai (>=0.6.0,<0.7.0) ; extra == "model-platforms" or extra == "all"
 Requires-Dist: googlemaps (>=4.10.0,<5.0.0) ; extra == "tools" or extra == "all"
 Requires-Dist: groq (>=0.5.0,<0.6.0)
@@ -47,11 +51,11 @@ Requires-Dist: ipykernel (>=6.0.0,<7.0.0)
 Requires-Dist: jsonschema (>=4,<5)
 Requires-Dist: jupyter_client (>=8.6.2,<9.0.0) ; extra == "tools" or extra == "all"
 Requires-Dist: litellm (>=1.38.1,<2.0.0) ; extra == "model-platforms" or extra == "all"
-Requires-Dist: mistral-common (>=1.3.3,<2.0.0) ; extra == "model-platforms" or extra == "all"
-Requires-Dist: mistralai (>=0.4.2,<0.5.0) ; extra == "model-platforms" or extra == "all"
+Requires-Dist: mistralai (>=1.0.0,<2.0.0) ; extra == "model-platforms" or extra == "all"
 Requires-Dist: mock (>=5,<6) ; extra == "test"
 Requires-Dist: neo4j (>=5.18.0,<6.0.0) ; extra == "graph-storages" or extra == "all"
 Requires-Dist: newspaper3k (>=0.2.8,<0.3.0) ; extra == "tools" or extra == "all"
+Requires-Dist: nltk (==3.8.1) ; extra == "tools" or extra == "all"
 Requires-Dist: numpy (>=1,<2)
 Requires-Dist: openai (>=1.2.3,<2.0.0)
 Requires-Dist: openapi-spec-validator (>=0.7.1,<0.8.0) ; extra == "tools" or extra == "all"
@@ -79,7 +83,7 @@ Requires-Dist: soundfile (>=0,<1) ; extra == "huggingface-agent" or extra == "al
 Requires-Dist: tiktoken (>=0.7.0,<0.8.0)
 Requires-Dist: torch (>=2,<3) ; extra == "huggingface-agent" or extra == "all"
 Requires-Dist: transformers (>=4,<5) ; extra == "huggingface-agent" or extra == "all"
-Requires-Dist: unstructured[all-docs] (>=0.10.30,<0.11.0) ; extra == "tools" or extra == "all"
+Requires-Dist: unstructured[all-docs] (>=0.10,<0.11) ; extra == "tools" or extra == "all"
 Requires-Dist: wikipedia (>=1,<2) ; extra == "tools" or extra == "all"
 Requires-Dist: wolframalpha (>=5.0.0,<6.0.0) ; extra == "tools" or extra == "all"
 Project-URL: Documentation, https://docs.camel-ai.org
@@ -198,7 +202,7 @@ conda create --name camel python=3.9
 conda activate camel
 
 # Clone github repo
-git clone -b v0.1.6.2 https://github.com/camel-ai/camel.git
+git clone -b v0.1.6.4 https://github.com/camel-ai/camel.git
 
 # Change directory into project directory
 cd camel