camel-ai 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

camel/datahubs/base.py

@@ -0,0 +1,136 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from typing import Any, List
+
+from camel.datahubs.models import Record
+
+
+class BaseDatasetManager(ABC):
+    r"""Abstract base class for dataset managers."""
+
+    @abstractmethod
+    def create_dataset(self, name: str, **kwargs: Any) -> str:
+        r"""Creates a new dataset.
+
+        Args:
+            name (str): The name of the dataset.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            str: The URL of the created dataset.
+        """
+        pass
+
+    @abstractmethod
+    def list_datasets(
+        self, username: str, limit: int = 100, **kwargs: Any
+    ) -> List[str]:
+        r"""Lists all datasets for the current user.
+
+        Args:
+            username (str): The username of the user whose datasets to list.
+            limit (int): The maximum number of datasets to list.
+                (default::obj:`100`)
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            List[str]: A list of dataset ids.
+        """
+        pass
+
+    @abstractmethod
+    def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:
+        r"""Deletes a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset to delete.
+            kwargs (Any): Additional keyword arguments.
+        """
+        pass
+
+    @abstractmethod
+    def add_records(
+        self,
+        dataset_name: str,
+        records: List[Record],
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Adds records to a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            records (List[Record]): A list of records to add to the dataset.
+            filepath (str): The path to the file containing the records.
+                (default::obj:`"records/records.json"`)
+            kwargs (Any): Additional keyword arguments.
+        """
+        pass
+
+    @abstractmethod
+    def update_records(
+        self,
+        dataset_name: str,
+        records: List[Record],
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Updates records in a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            records (List[Record]): A list of records to update in the dataset.
+            filepath (str): The path to the file containing the records.
+                (default::obj:`"records/records.json"`)
+            kwargs (Any): Additional keyword arguments.
+        """
+        pass
+
+    @abstractmethod
+    def list_records(
+        self,
+        dataset_name: str,
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> List[Record]:
+        r"""Lists records in a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            filepath (str): The path to the file containing the records.
+                (default::obj:`"records/records.json"`)
+            kwargs (Any): Additional keyword arguments.
+        """
+        pass
+
+    # New method for record deletion
+    @abstractmethod
+    def delete_record(
+        self,
+        dataset_name: str,
+        record_id: str,
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Deletes a record from the dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            record_id (str): The ID of the record to delete.
+            filepath (str): The path to the file containing the records.
+                (default::obj:`"records/records.json"`)
+            kwargs (Any): Additional keyword arguments.
+        """
+        pass

camel/datahubs/huggingface.py

@@ -0,0 +1,433 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import os
+import tempfile
+from typing import Any, List, Optional
+
+from camel.datahubs.base import BaseDatasetManager
+from camel.datahubs.models import Record
+from camel.logger import get_logger
+from camel.types import HuggingFaceRepoType
+from camel.utils import api_keys_required, dependencies_required
+
+logger = get_logger(__name__)
+
+
+class HuggingFaceDatasetManager(BaseDatasetManager):
+    r"""A dataset manager for Hugging Face datasets. This class provides
+    methods to create, add, update, delete, and list records in a dataset on
+    the Hugging Face Hub.
+
+    Args:
+        token (str): The Hugging Face API token. If not provided, the token
+            will be read from the environment variable `HUGGING_FACE_TOKEN`.
+    """
+
+    @api_keys_required("HUGGING_FACE_TOKEN")
+    @dependencies_required('huggingface_hub')
+    def __init__(self, token: Optional[str] = None):
+        from huggingface_hub import HfApi
+
+        self._api_key = token or os.getenv("HUGGING_FACE_TOKEN")
+        self.api = HfApi(token=self._api_key)
+
+    def create_dataset_card(
+        self,
+        dataset_name: str,
+        description: str,
+        license: Optional[str] = None,
+        version: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        authors: Optional[List[str]] = None,
+        size_category: Optional[List[str]] = None,
+        language: Optional[List[str]] = None,
+        task_categories: Optional[List[str]] = None,
+        content: Optional[str] = None,
+    ) -> None:
+        r"""Creates and uploads a dataset card to the Hugging Face Hub in YAML
+            format.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            description (str): A description of the dataset.
+            license (str): The license of the dataset. (default: :obj:`None`)
+            version (str): The version of the dataset. (default: :obj:`None`)
+            tags (list): A list of tags for the dataset.(default: :obj:`None`)
+            authors (list): A list of authors of the dataset. (default:
+                :obj:`None`)
+            size_category (list): A size category for the dataset. (default:
+                :obj:`None`)
+            language (list): A list of languages the dataset is in. (default:
+                :obj:`None`)
+            task_categories (list): A list of task categories. (default:
+                :obj:`None`)
+            content (str): Custom markdown content that the user wants to add
+                to the dataset card. (default: :obj:`None`)
+        """
+        import yaml
+
+        metadata = {
+            "license": license,
+            "authors": authors,
+            "task_categories": task_categories,
+            "language": language,
+            "tags": tags,
+            "pretty_name": dataset_name,
+            "size_categories": size_category,
+            "version": version,
+            "description": description,
+        }
+
+        # Remove keys with None values
+        metadata = {k: v for k, v in metadata.items() if v}
+
+        card_content = (
+            "---\n"
+            + yaml.dump(metadata, default_flow_style=False, allow_unicode=True)
+            + "\n---"
+        )
+
+        if content:
+            card_content += f"\n\n# Additional Information\n{content}\n"
+
+        self._upload_file(
+            file_content=card_content,
+            dataset_name=dataset_name,
+            filepath="README.md",
+            file_type="md",
+        )
+
+    def create_dataset(
+        self, name: str, private: bool = False, **kwargs: Any
+    ) -> str:
+        r"""Creates a new dataset on the Hugging Face Hub.
+
+        Args:
+            name (str): The name of the dataset.
+            private (bool): Whether the dataset should be private. defaults to
+                False.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            str: The URL of the created dataset.
+        """
+        from huggingface_hub.errors import RepositoryNotFoundError
+
+        try:
+            self.api.repo_info(
+                repo_id=name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+        except RepositoryNotFoundError:
+            self.api.create_repo(
+                repo_id=name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                private=private,
+            )
+
+        return f"https://huggingface.co/datasets/{name}"
+
+    def list_datasets(
+        self, username: str, limit: int = 100, **kwargs: Any
+    ) -> List[str]:
+        r"""Lists all datasets for the current user.
+
+        Args:
+            username (str): The username of the user whose datasets to list.
+            limit (int): The maximum number of datasets to list.
+                (default: :obj:`100`)
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            List[str]: A list of dataset ids.
+        """
+        try:
+            return [
+                dataset.id
+                for dataset in self.api.list_datasets(
+                    author=username, limit=limit, **kwargs
+                )
+            ]
+        except Exception as e:
+            logger.error(f"Error listing datasets: {e}")
+            return []
+
+    def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:
+        r"""Deletes a dataset from the Hugging Face Hub.
+
+        Args:
+            dataset_name (str): The name of the dataset to delete.
+            kwargs (Any): Additional keyword arguments.
+        """
+        try:
+            self.api.delete_repo(
+                repo_id=dataset_name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+            logger.info(f"Dataset '{dataset_name}' deleted successfully.")
+        except Exception as e:
+            logger.error(f"Error deleting dataset '{dataset_name}': {e}")
+            raise
+
+    def add_records(
+        self,
+        dataset_name: str,
+        records: List[Record],
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Adds records to a dataset on the Hugging Face Hub.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            records (List[Record]): A list of records to add to the dataset.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Raises:
+            ValueError: If the dataset already has a records file.
+        """
+        existing_records = self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+        if existing_records:
+            raise ValueError(
+                f"Dataset '{filepath}' already exists. "
+                f"Use `update_records` to modify."
+            )
+
+        self._upload_records(
+            records=records,
+            dataset_name=dataset_name,
+            filepath=filepath,
+            **kwargs,
+        )
+
+    def update_records(
+        self,
+        dataset_name: str,
+        records: List[Record],
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Updates records in a dataset on the Hugging Face Hub.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            records (List[Record]): A list of records to update in the dataset.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Raises:
+            ValueError: If the dataset does not have an existing file to update
+                records in.
+        """
+        existing_records = self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+        if not existing_records:
+            logger.warning(
+                f"Dataset '{dataset_name}' does not have existing "
+                "records. Adding new records."
+            )
+            self._upload_records(
+                records=records,
+                dataset_name=dataset_name,
+                filepath=filepath,
+                **kwargs,
+            )
+            return
+
+        old_dict = {record.id: record for record in existing_records}
+        new_dict = {record.id: record for record in records}
+        merged_dict = old_dict.copy()
+        merged_dict.update(new_dict)
+
+        self._upload_records(
+            records=list(merged_dict.values()),
+            dataset_name=dataset_name,
+            filepath=filepath,
+            **kwargs,
+        )
+
+    def delete_record(
+        self,
+        dataset_name: str,
+        record_id: str,
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Deletes a record from the dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            record_id (str): The ID of the record to delete.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Raises:
+            ValueError: If the dataset does not have an existing file to delete
+                records from.
+        """
+        existing_records = self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+        if not existing_records:
+            raise ValueError(
+                f"Dataset '{dataset_name}' does not have an existing file to "
+                f"delete records from."
+            )
+
+        filtered_records = [
+            record for record in existing_records if record.id != record_id
+        ]
+
+        self._upload_records(
+            records=filtered_records,
+            dataset_name=dataset_name,
+            filepath=filepath,
+            **kwargs,
+        )
+
+    def list_records(
+        self,
+        dataset_name: str,
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> List[Record]:
+        r"""Lists all records in a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            List[Record]: A list of records in the dataset.
+        """
+        return self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+    def _download_records(
+        self, dataset_name: str, filepath: str, **kwargs: Any
+    ) -> List[Record]:
+        from huggingface_hub import hf_hub_download
+        from huggingface_hub.errors import EntryNotFoundError
+
+        try:
+            downloaded_file_path = hf_hub_download(
+                repo_id=dataset_name,
+                filename=filepath,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                token=self._api_key,
+                **kwargs,
+            )
+
+            with open(downloaded_file_path, "r") as f:
+                records_data = json.load(f)
+
+            return [Record(**record) for record in records_data]
+        except EntryNotFoundError:
+            logger.info(f"No records found for dataset '{dataset_name}'.")
+            return []
+        except Exception as e:
+            logger.error(f"Error downloading or processing records: {e}")
+            raise e
+
+    def _upload_records(
+        self,
+        records: List[Record],
+        dataset_name: str,
+        filepath: str,
+        **kwargs: Any,
+    ):
+        with tempfile.NamedTemporaryFile(
+            delete=False, mode="w", newline="", encoding="utf-8"
+        ) as f:
+            json.dump([record.model_dump() for record in records], f)
+            temp_file_path = f.name
+
+        try:
+            self.api.upload_file(
+                path_or_fileobj=temp_file_path,
+                path_in_repo=filepath,
+                repo_id=dataset_name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+        except Exception as e:
+            logger.error(f"Error uploading records file: {e}")
+            raise
+        finally:
+            if os.path.exists(temp_file_path):
+                os.remove(temp_file_path)
+
+    def _upload_file(
+        self,
+        file_content: str,
+        dataset_name: str,
+        filepath: str,
+        file_type: str = "json",
+        **kwargs: Any,
+    ):
+        with tempfile.NamedTemporaryFile(
+            mode="w", delete=False, suffix=f".{file_type}"
+        ) as f:
+            if file_type == "json":
+                if isinstance(file_content, str):
+                    try:
+                        json_content = json.loads(file_content)
+                    except json.JSONDecodeError:
+                        raise ValueError(
+                            "Invalid JSON string provided for file_content."
+                        )
+                else:
+                    try:
+                        json.dumps(file_content)
+                        json_content = file_content
+                    except (TypeError, ValueError):
+                        raise ValueError(
+                            "file_content is not JSON serializable."
+                        )
+
+                json.dump(json_content, f)
+            elif file_type == "md" or file_type == "txt":
+                f.write(file_content)
+            else:
+                raise ValueError(f"Unsupported file type: {file_type}")
+
+            temp_file_path = f.name
+
+        try:
+            self.api.upload_file(
+                path_or_fileobj=temp_file_path,
+                path_in_repo=filepath,
+                repo_id=dataset_name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+            logger.info(f"File uploaded successfully: {filepath}")
+        except Exception as e:
+            logger.error(f"Error uploading file: {e}")
+            raise
+
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)

camel/datahubs/models.py

@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel
+
+
+class Record(BaseModel):
+    id: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+    content: Dict[str, Any]

camel/embeddings/vlm_embedding.py

@@ -16,6 +16,9 @@ from typing import Any, List, Optional, Union
 from PIL import Image
 
 from camel.embeddings import BaseEmbedding
+from camel.logger import get_logger
+
+logger = get_logger(__name__)
 
 
 class VisionLanguageEmbedding(BaseEmbedding[Union[str, Image.Image]]):
@@ -60,7 +63,7 @@ class VisionLanguageEmbedding(BaseEmbedding[Union[str, Image.Image]]):
                 "interpolate_pos_encoding",
             ]
         except Exception:
-            print("Warning: not typically processor and model structure")
+            logger.warning("not typically processor and model structure")
             pass
         self.dim: Optional[int] = None
 

camel/interpreters/__init__.py

@@ -14,6 +14,7 @@
 
 from .base import BaseInterpreter
 from .docker_interpreter import DockerInterpreter
+from .e2b_interpreter import E2BInterpreter
 from .internal_python_interpreter import InternalPythonInterpreter
 from .interpreter_error import InterpreterError
 from .ipython_interpreter import JupyterKernelInterpreter
@@ -26,4 +27,5 @@ __all__ = [
     'SubprocessInterpreter',
     'DockerInterpreter',
     'JupyterKernelInterpreter',
+    'E2BInterpreter',
 ]

camel/interpreters/docker_interpreter.py

@@ -23,11 +23,14 @@ from colorama import Fore
 
 from camel.interpreters.base import BaseInterpreter
 from camel.interpreters.interpreter_error import InterpreterError
+from camel.logger import get_logger
 from camel.utils import is_docker_running
 
 if TYPE_CHECKING:
     from docker.models.containers import Container
 
+logger = get_logger(__name__)
+
 
 class DockerInterpreter(BaseInterpreter):
     r"""A class for executing code files or code strings in a docker container.
@@ -187,8 +190,10 @@ class DockerInterpreter(BaseInterpreter):
 
         # Print code for security checking
         if self.require_confirm:
-            print(f"The following {code_type} code will run in container:")
-            print(Fore.CYAN + code + Fore.RESET)
+            logger.info(
+                f"The following {code_type} code will run on your "
+                "computer: {code}"
+            )
             while True:
                 choice = input("Running code? [Y/n]:").lower()
                 if choice in ["y", "yes", "ye", ""]: