camel-ai 0.1.6.2__py3-none-any.whl → 0.1.6.3__py3-none-any.whl

camel/storages/object_storages/amazon_s3.py

@@ -0,0 +1,205 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from pathlib import Path, PurePath
+from typing import Optional, Tuple
+from warnings import warn
+
+from camel.loaders import File
+from camel.storages.object_storages.base import BaseObjectStorage
+
+
+class AmazonS3Storage(BaseObjectStorage):
+    r"""A class to connect with AWS S3 object storage to put and get objects
+    from one S3 bucket. The class will first try to use the credentials passed
+    as arguments, if not provided, it will look for the environment variables
+    `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`. If none of these are
+    provided, it will try to use the local credentials (will be created if
+    logged in with AWS CLI).
+
+    Args:
+        bucket_name (str): The name of the S3 bucket.
+        create_if_not_exists (bool, optional): Whether to create the bucket if
+            it does not exist. Defaults to True.
+        access_key_id (Optional[str], optional): The AWS access key ID.
+            Defaults to None.
+        secret_access_key (Optional[str], optional): The AWS secret access key.
+            Defaults to None.
+        anonymous (bool, optional): Whether to use anonymous access. Defaults
+            to False.
+
+    References:
+        https://aws.amazon.com/pm/serv-s3/
+
+        https://aws.amazon.com/cli/
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        create_if_not_exists: bool = True,
+        access_key_id: Optional[str] = None,
+        secret_access_key: Optional[str] = None,
+        anonymous: bool = False,
+    ) -> None:
+        self._bucket_name = bucket_name
+        self._create_if_not_exists = create_if_not_exists
+
+        aws_key_id = access_key_id or os.getenv("AWS_ACCESS_KEY_ID")
+        aws_secret_key = secret_access_key or os.getenv(
+            "AWS_SECRET_ACCESS_KEY"
+        )
+        if not all([aws_key_id, aws_secret_key]) and not anonymous:
+            warn(
+                "AWS access key not configured. Local credentials will be "
+                "used."
+            )
+            # Make all the empty values None
+            aws_key_id = None
+            aws_secret_key = None
+
+        import boto3
+        from botocore import UNSIGNED
+        from botocore.config import Config
+
+        if not anonymous:
+            self._client = boto3.client(
+                "s3",
+                aws_access_key_id=aws_key_id,
+                aws_secret_access_key=aws_secret_key,
+            )
+        else:
+            self._client = boto3.client(
+                "s3", config=Config(signature_version=UNSIGNED)
+            )
+
+        self._prepare_and_check()
+
+    def _prepare_and_check(self) -> None:
+        r"""Check privileges and existence of the bucket."""
+        from botocore.exceptions import ClientError, NoCredentialsError
+
+        try:
+            self._client.head_bucket(Bucket=self._bucket_name)
+        except ClientError as e:
+            error_code = e.response['Error']['Code']
+            if error_code == '403':
+                raise PermissionError(
+                    f"Failed to access bucket {self._bucket_name}: "
+                    f"No permission."
+                )
+            elif error_code == '404':
+                if self._create_if_not_exists:
+                    self._client.create_bucket(Bucket=self._bucket_name)
+                    warn(
+                        f"Bucket {self._bucket_name} not found. Automatically "
+                        f"created."
+                    )
+                else:
+                    raise FileNotFoundError(
+                        f"Failed to access bucket {self._bucket_name}: Not "
+                        f"found."
+                    )
+            else:
+                raise e
+        except NoCredentialsError as e:
+            raise PermissionError("No AWS credentials found.") from e
+
+    @staticmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        r"""Canonicalize file path for Amazon S3.
+
+        Args:
+            file_path (PurePath): The path to be canonicalized.
+
+        Returns:
+            Tuple[str, str]: The canonicalized file key and file name.
+        """
+        return file_path.as_posix(), file_path.name
+
+    def _put_file(self, file_key: str, file: File) -> None:
+        r"""Put a file to the Amazon S3 bucket.
+
+        Args:
+            file_key (str): The path to the object in the bucket.
+            file (File): The file to be uploaded.
+        """
+        self._client.put_object(
+            Bucket=self._bucket_name, Key=file_key, Body=file.raw_bytes
+        )
+
+    def _get_file(self, file_key: str, filename: str) -> File:
+        r"""Get a file from the Amazon S3 bucket.
+
+        Args:
+            file_key (str): The path to the object in the bucket.
+            filename (str): The name of the file.
+
+        Returns:
+            File: The object from the S3 bucket.
+        """
+        response = self._client.get_object(
+            Bucket=self._bucket_name, Key=file_key
+        )
+        raw_bytes = response["Body"].read()
+        return File.create_file_from_raw_bytes(raw_bytes, filename)
+
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Upload a local file to the Amazon S3 bucket.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_key (str): The path to the object in the bucket.
+        """
+        self._client.upload_file(
+            Bucket=self._bucket_name,
+            Key=remote_file_key,
+            Filename=local_file_path,
+        )
+
+    def _download_file(
+        self,
+        local_file_path: Path,
+        remote_file_key: str,
+    ) -> None:
+        r"""Download a file from the Amazon S3 bucket to the local system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_key (str): The key of the object in the bucket.
+        """
+        self._client.download_file(
+            Bucket=self._bucket_name,
+            Key=remote_file_key,
+            Filename=local_file_path,
+        )
+
+    def _object_exists(self, file_key: str) -> bool:
+        r"""
+        Check if the object exists in the Amazon S3 bucket.
+
+        Args:
+            file_key: The key of the object in the bucket.
+
+        Returns:
+            bool: Whether the object exists in the bucket.
+        """
+        try:
+            self._client.head_object(Bucket=self._bucket_name, Key=file_key)
+            return True
+        except self._client.exceptions.ClientError:
+            return False

camel/storages/object_storages/azure_blob.py

@@ -0,0 +1,166 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from pathlib import Path, PurePath
+from typing import Optional, Tuple
+from warnings import warn
+
+from camel.loaders import File
+from camel.storages.object_storages.base import BaseObjectStorage
+
+
+class AzureBlobStorage(BaseObjectStorage):
+    r"""A class to connect to Azure Blob Storage. It will connect to one
+    container in the storage account.
+
+    Args:
+        storage_account_name (str): The name of the storage account.
+        container_name (str): The name of the container.
+        access_key (Optional[str], optional): The access key of the storage
+            account. Defaults to None.
+
+    References:
+        https://azure.microsoft.com/en-us/products/storage/blobs
+    """
+
+    def __init__(
+        self,
+        storage_account_name: str,
+        container_name: str,
+        create_if_not_exists: bool = True,
+        access_key: Optional[str] = None,
+    ) -> None:
+        access_key = access_key or os.getenv("AZURE_ACCESS_KEY")
+        self._create_if_not_exists = create_if_not_exists
+
+        if not access_key:
+            warn("AZURE_ACCESS_KEY not provided.")
+            # Make all the empty values None
+            access_key = None
+
+        from azure.storage.blob import ContainerClient
+
+        self._client = ContainerClient(
+            account_url="https://"
+            f"{storage_account_name}.blob.core.windows.net",
+            credential=access_key,
+            container_name=container_name,
+        )
+
+        self._prepare_and_check()
+
+    def _prepare_and_check(self) -> None:
+        r"""Check privileges and existence of the container."""
+        from azure.core.exceptions import ClientAuthenticationError
+
+        try:
+            exists = self._client.exists()
+            if not exists and self._create_if_not_exists:
+                self._client.create_container()
+                warn(
+                    f"Container {self._client.container_name} not found. "
+                    f"Automatically created."
+                )
+            elif not exists:
+                raise FileNotFoundError(
+                    f"Failed to access container {self._client.container_name}"
+                    f": Not found."
+                )
+        except ClientAuthenticationError:
+            raise PermissionError(
+                f"Failed to access container {self._client.container_name}: "
+                f"No permission."
+            )
+
+    @staticmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        r"""Canonicalize file path for Azure Blob Storage.
+
+        Args:
+            file_path (PurePath): The path to be canonicalized.
+
+        Returns:
+            Tuple[str, str]: The canonicalized file key and file name.
+        """
+        # for Azure, both slash and backslash will be treated as separator
+        filename = file_path.name
+        if "\\" in filename:
+            raise ValueError(
+                "Azure Blob Storage does not support backslash in filename."
+            )
+        return file_path.as_posix(), filename
+
+    def _put_file(self, file_key: str, file: File) -> None:
+        r"""Put a file to the Azure Blob Storage container.
+
+        Args:
+            file_key (str): The path to the object in the container.
+            file (File): The file to be uploaded.
+        """
+        self._client.upload_blob(
+            name=file_key, data=file.raw_bytes, overwrite=True
+        )
+
+    def _get_file(self, file_key: str, filename: str) -> File:
+        r"""Get a file from the Azure Blob Storage container.
+
+        Args:
+            file_key (str): The path to the object in the container.
+            filename (str): The name of the file.
+
+        Returns:
+            File: The object from the container.
+        """
+        raw_bytes = self._client.download_blob(file_key).readall()
+        file = File.create_file_from_raw_bytes(raw_bytes, filename)
+        return file
+
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Upload a local file to the Azure Blob Storage container.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_key (str): The path to the object in the container.
+        """
+        with open(local_file_path, "rb") as f:
+            self._client.upload_blob(
+                name=remote_file_key, data=f, overwrite=True
+            )
+
+    def _download_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Download a file from the Azure Blob Storage container to the local
+        system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_key (str): The key of the object in the container.
+        """
+        with open(local_file_path, "wb") as f:
+            f.write(self._client.download_blob(remote_file_key).readall())
+
+    def _object_exists(self, file_key: str) -> bool:
+        r"""
+        Check if the object exists in the Azure Blob Storage container.
+
+        Args:
+            file_key: The key of the object in the container.
+
+        Returns:
+            bool: Whether the object exists in the container.
+        """
+        return self._client.get_blob_client(file_key).exists()

camel/storages/object_storages/base.py

@@ -0,0 +1,115 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+from abc import ABC, abstractmethod
+from pathlib import Path, PurePath
+from typing import Tuple
+
+from camel.loaders import File
+
+
+class BaseObjectStorage(ABC):
+    def object_exists(self, file_path: PurePath) -> bool:
+        r"""Check if the object exists in the storage.
+
+        Args:
+            file_path (PurePath): The path to the object in the storage.
+
+        Returns:
+            bool: True if the object exists, False otherwise.
+        """
+        file_key, _ = self.canonicalize_path(file_path)
+        return self._object_exists(file_key)
+
+    @staticmethod
+    @abstractmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        pass
+
+    def put_file(self, file_path: PurePath, file: File) -> None:
+        r"""Put a file to the object storage.
+
+        Args:
+            file_path (PurePath): The path to the object in the storage.
+            file (File): The file to be put.
+        """
+        file_key, _ = self.canonicalize_path(file_path)
+        self._put_file(file_key, file)
+
+    def get_file(self, file_path: PurePath) -> File:
+        r"""Get a file from the object storage.
+
+        Args:
+            file_path (PurePath): The path to the object in the storage.
+
+        Returns:
+            File: The file object get from the storage.
+        """
+        file_key, filename = self.canonicalize_path(file_path)
+        return self._get_file(file_key, filename)
+
+    def upload_file(
+        self, local_file_path: Path, remote_file_path: PurePath
+    ) -> None:
+        r"""Upload a local file to the object storage.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_path (PurePath): The path to the object in storage.
+        """
+        file_key, _ = self.canonicalize_path(remote_file_path)
+        # check if the local file exists
+        if not local_file_path.exists():
+            raise FileNotFoundError(
+                f"Local file {local_file_path} does not exist."
+            )
+        self._upload_file(local_file_path, file_key)
+
+    def download_file(
+        self, local_file_path: Path, remote_file_path: PurePath
+    ) -> None:
+        r"""Download a file from the object storage to the local system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_path (PurePath): The path to the object in storage.
+        """
+        file_key, _ = self.canonicalize_path(remote_file_path)
+        self._download_file(local_file_path, file_key)
+
+    @abstractmethod
+    def _put_file(self, file_key: str, file: File) -> None:
+        pass
+
+    @abstractmethod
+    def _get_file(self, file_key: str, filename: str) -> File:
+        pass
+
+    @abstractmethod
+    def _object_exists(self, file_key: str) -> bool:
+        pass
+
+    @abstractmethod
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        pass
+
+    @abstractmethod
+    def _download_file(
+        self,
+        local_file_path: Path,
+        remote_file_key: str,
+    ) -> None:
+        pass

camel/storages/object_storages/google_cloud.py

@@ -0,0 +1,152 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from pathlib import Path, PurePath
+from typing import Tuple
+from warnings import warn
+
+from camel.loaders import File
+from camel.storages.object_storages.base import BaseObjectStorage
+
+
+class GoogleCloudStorage(BaseObjectStorage):
+    r"""A class to connect to Google Cloud Storage. It will connect to one
+    bucket in the storage account.
+
+    Note that Google Cloud Storage does not support api key authentication.
+    Therefore, before using this class, you need to log in with gcloud command
+    line tool and save the credentials first.
+
+    Args:
+        bucket_name (str): The name of the bucket.
+        create_if_not_exists (bool, optional): Whether to create the bucket if
+            it does not exist. Defaults to True.
+        anonymous (bool, optional): Whether to use anonymous access. Defaults
+            to False.
+
+    References:
+        https://cloud.google.com/storage
+
+        https://cloud.google.com/docs/authentication/api-keys
+    """
+
+    def __init__(
+        self,
+        bucket_name: str,
+        create_if_not_exists: bool = True,
+        anonymous: bool = False,
+    ) -> None:
+        from google.cloud import storage
+
+        self.create_if_not_exists = create_if_not_exists
+
+        if anonymous:
+            client = storage.Client.create_anonymous_client()
+        else:
+            client = storage.Client()
+        self._client = client.bucket(bucket_name)
+
+        self._prepare_and_check()
+
+    @staticmethod
+    def canonicalize_path(file_path: PurePath) -> Tuple[str, str]:
+        r"""Canonicalize the path for Google Cloud Storage.
+
+        Args:
+            file_path (PurePath): The path to be canonicalized.
+
+        Returns:
+            Tuple[str, str]: The canonicalized file key and file name.
+        """
+        return file_path.as_posix(), file_path.name
+
+    def _prepare_and_check(self) -> None:
+        r"""Check privileges and existence of the bucket."""
+        from google.auth.exceptions import InvalidOperation
+
+        try:
+            exists = self._client.exists()
+            if not exists and self.create_if_not_exists:
+                self._client.create()
+                warn(
+                    f"Bucket {self._client.name} not found. Automatically "
+                    f"created."
+                )
+            elif not exists:
+                raise FileNotFoundError(
+                    f"Failed to access bucket {self._client.name}: Not found."
+                )
+        except InvalidOperation:
+            raise PermissionError(
+                f"Failed to access bucket {self._client.name}: No permission."
+            )
+
+    def _put_file(self, file_key: str, file: File) -> None:
+        r"""Put a file to the GCloud bucket.
+
+        Args:
+            file_key (str): The path to the object in the bucket.
+            file (File): The file to be uploaded.
+        """
+        self._client.blob(file_key).upload_from_string(file.raw_bytes)
+
+    def _get_file(self, file_key: str, filename: str) -> File:
+        r"""Get a file from the GCloud bucket.
+
+        Args:
+            file_key (str): The path to the object in the bucket.
+            filename (str): The name of the file.
+
+        Returns:
+            File: The object from the S3 bucket.
+        """
+        raw_bytes = self._client.get_blob(file_key).download_as_bytes()
+        return File.create_file_from_raw_bytes(raw_bytes, filename)
+
+    def _upload_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Upload a local file to the GCloud bucket.
+
+        Args:
+            local_file_path (Path): The path to the local file to be uploaded.
+            remote_file_key (str): The path to the object in the bucket.
+        """
+        self._client.blob(remote_file_key).upload_from_filename(
+            local_file_path
+        )
+
+    def _download_file(
+        self, local_file_path: Path, remote_file_key: str
+    ) -> None:
+        r"""Download a file from the GCloud bucket to the local system.
+
+        Args:
+            local_file_path (Path): The path to the local file to be saved.
+            remote_file_key (str): The key of the object in the bucket.
+        """
+        self._client.get_blob(remote_file_key).download_to_filename(
+            local_file_path
+        )
+
+    def _object_exists(self, file_key: str) -> bool:
+        r"""
+        Check if the object exists in the GCloud bucket.
+
+        Args:
+            file_key: The key of the object in the bucket.
+
+        Returns:
+            bool: Whether the object exists in the bucket.
+        """
+        return self._client.blob(file_key).exists()

camel/toolkits/retrieval_toolkit.py

@@ -27,7 +27,7 @@ class RetrievalToolkit(BaseToolkit):
     """
 
     def information_retrieval(
-        self, query: str, content_input_paths: Union[str, List[str]]
+        self, query: str, contents: Union[str, List[str]]
     ) -> str:
         r"""Retrieves information from a local vector storage based on the
         specified query. This function connects to a local vector storage
@@ -37,8 +37,8 @@ class RetrievalToolkit(BaseToolkit):
 
         Args:
             query (str): The question or query for which an answer is required.
-            content_input_paths (Union[str, List[str]]): Paths to local
-                files or remote URLs.
+            contents (Union[str, List[str]]): Local file paths, remote URLs or
+                string contents.
 
         Returns:
             str: The information retrieved in response to the query, aggregated
@@ -47,7 +47,7 @@ class RetrievalToolkit(BaseToolkit):
         Example:
             # Retrieve information about CAMEL AI.
             information_retrieval(query = "what is CAMEL AI?",
-                                content_input_paths="https://www.camel-ai.org/")
+                                contents="https://www.camel-ai.org/")
         """
         auto_retriever = AutoRetriever(
             vector_storage_local_path="camel/temp_storage",
@@ -55,7 +55,7 @@ class RetrievalToolkit(BaseToolkit):
         )
 
         retrieved_info = auto_retriever.run_vector_retriever(
-            query=query, content_input_paths=content_input_paths, top_k=3
+            query=query, contents=contents, top_k=3
         )
         return retrieved_info