camel-ai 0.2.11__py3-none-any.whl → 0.2.13__py3-none-any.whl

camel/datahubs/huggingface.py

@@ -0,0 +1,433 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import json
+import os
+import tempfile
+from typing import Any, List, Optional
+
+from camel.datahubs.base import BaseDatasetManager
+from camel.datahubs.models import Record
+from camel.logger import get_logger
+from camel.types import HuggingFaceRepoType
+from camel.utils import api_keys_required, dependencies_required
+
+logger = get_logger(__name__)
+
+
+class HuggingFaceDatasetManager(BaseDatasetManager):
+    r"""A dataset manager for Hugging Face datasets. This class provides
+    methods to create, add, update, delete, and list records in a dataset on
+    the Hugging Face Hub.
+
+    Args:
+        token (str): The Hugging Face API token. If not provided, the token
+            will be read from the environment variable `HUGGING_FACE_TOKEN`.
+    """
+
+    @api_keys_required("HUGGING_FACE_TOKEN")
+    @dependencies_required('huggingface_hub')
+    def __init__(self, token: Optional[str] = None):
+        from huggingface_hub import HfApi
+
+        self._api_key = token or os.getenv("HUGGING_FACE_TOKEN")
+        self.api = HfApi(token=self._api_key)
+
+    def create_dataset_card(
+        self,
+        dataset_name: str,
+        description: str,
+        license: Optional[str] = None,
+        version: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        authors: Optional[List[str]] = None,
+        size_category: Optional[List[str]] = None,
+        language: Optional[List[str]] = None,
+        task_categories: Optional[List[str]] = None,
+        content: Optional[str] = None,
+    ) -> None:
+        r"""Creates and uploads a dataset card to the Hugging Face Hub in YAML
+            format.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            description (str): A description of the dataset.
+            license (str): The license of the dataset. (default: :obj:`None`)
+            version (str): The version of the dataset. (default: :obj:`None`)
+            tags (list): A list of tags for the dataset.(default: :obj:`None`)
+            authors (list): A list of authors of the dataset. (default:
+                :obj:`None`)
+            size_category (list): A size category for the dataset. (default:
+                :obj:`None`)
+            language (list): A list of languages the dataset is in. (default:
+                :obj:`None`)
+            task_categories (list): A list of task categories. (default:
+                :obj:`None`)
+            content (str): Custom markdown content that the user wants to add
+                to the dataset card. (default: :obj:`None`)
+        """
+        import yaml
+
+        metadata = {
+            "license": license,
+            "authors": authors,
+            "task_categories": task_categories,
+            "language": language,
+            "tags": tags,
+            "pretty_name": dataset_name,
+            "size_categories": size_category,
+            "version": version,
+            "description": description,
+        }
+
+        # Remove keys with None values
+        metadata = {k: v for k, v in metadata.items() if v}
+
+        card_content = (
+            "---\n"
+            + yaml.dump(metadata, default_flow_style=False, allow_unicode=True)
+            + "\n---"
+        )
+
+        if content:
+            card_content += f"\n\n# Additional Information\n{content}\n"
+
+        self._upload_file(
+            file_content=card_content,
+            dataset_name=dataset_name,
+            filepath="README.md",
+            file_type="md",
+        )
+
+    def create_dataset(
+        self, name: str, private: bool = False, **kwargs: Any
+    ) -> str:
+        r"""Creates a new dataset on the Hugging Face Hub.
+
+        Args:
+            name (str): The name of the dataset.
+            private (bool): Whether the dataset should be private. defaults to
+                False.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            str: The URL of the created dataset.
+        """
+        from huggingface_hub.errors import RepositoryNotFoundError
+
+        try:
+            self.api.repo_info(
+                repo_id=name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+        except RepositoryNotFoundError:
+            self.api.create_repo(
+                repo_id=name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                private=private,
+            )
+
+        return f"https://huggingface.co/datasets/{name}"
+
+    def list_datasets(
+        self, username: str, limit: int = 100, **kwargs: Any
+    ) -> List[str]:
+        r"""Lists all datasets for the current user.
+
+        Args:
+            username (str): The username of the user whose datasets to list.
+            limit (int): The maximum number of datasets to list.
+                (default: :obj:`100`)
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            List[str]: A list of dataset ids.
+        """
+        try:
+            return [
+                dataset.id
+                for dataset in self.api.list_datasets(
+                    author=username, limit=limit, **kwargs
+                )
+            ]
+        except Exception as e:
+            logger.error(f"Error listing datasets: {e}")
+            return []
+
+    def delete_dataset(self, dataset_name: str, **kwargs: Any) -> None:
+        r"""Deletes a dataset from the Hugging Face Hub.
+
+        Args:
+            dataset_name (str): The name of the dataset to delete.
+            kwargs (Any): Additional keyword arguments.
+        """
+        try:
+            self.api.delete_repo(
+                repo_id=dataset_name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+            logger.info(f"Dataset '{dataset_name}' deleted successfully.")
+        except Exception as e:
+            logger.error(f"Error deleting dataset '{dataset_name}': {e}")
+            raise
+
+    def add_records(
+        self,
+        dataset_name: str,
+        records: List[Record],
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Adds records to a dataset on the Hugging Face Hub.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            records (List[Record]): A list of records to add to the dataset.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Raises:
+            ValueError: If the dataset already has a records file.
+        """
+        existing_records = self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+        if existing_records:
+            raise ValueError(
+                f"Dataset '{filepath}' already exists. "
+                f"Use `update_records` to modify."
+            )
+
+        self._upload_records(
+            records=records,
+            dataset_name=dataset_name,
+            filepath=filepath,
+            **kwargs,
+        )
+
+    def update_records(
+        self,
+        dataset_name: str,
+        records: List[Record],
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Updates records in a dataset on the Hugging Face Hub.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            records (List[Record]): A list of records to update in the dataset.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Raises:
+            ValueError: If the dataset does not have an existing file to update
+                records in.
+        """
+        existing_records = self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+        if not existing_records:
+            logger.warning(
+                f"Dataset '{dataset_name}' does not have existing "
+                "records. Adding new records."
+            )
+            self._upload_records(
+                records=records,
+                dataset_name=dataset_name,
+                filepath=filepath,
+                **kwargs,
+            )
+            return
+
+        old_dict = {record.id: record for record in existing_records}
+        new_dict = {record.id: record for record in records}
+        merged_dict = old_dict.copy()
+        merged_dict.update(new_dict)
+
+        self._upload_records(
+            records=list(merged_dict.values()),
+            dataset_name=dataset_name,
+            filepath=filepath,
+            **kwargs,
+        )
+
+    def delete_record(
+        self,
+        dataset_name: str,
+        record_id: str,
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> None:
+        r"""Deletes a record from the dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            record_id (str): The ID of the record to delete.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Raises:
+            ValueError: If the dataset does not have an existing file to delete
+                records from.
+        """
+        existing_records = self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+        if not existing_records:
+            raise ValueError(
+                f"Dataset '{dataset_name}' does not have an existing file to "
+                f"delete records from."
+            )
+
+        filtered_records = [
+            record for record in existing_records if record.id != record_id
+        ]
+
+        self._upload_records(
+            records=filtered_records,
+            dataset_name=dataset_name,
+            filepath=filepath,
+            **kwargs,
+        )
+
+    def list_records(
+        self,
+        dataset_name: str,
+        filepath: str = "records/records.json",
+        **kwargs: Any,
+    ) -> List[Record]:
+        r"""Lists all records in a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            filepath (str): The path to the file containing the records.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            List[Record]: A list of records in the dataset.
+        """
+        return self._download_records(
+            dataset_name=dataset_name, filepath=filepath, **kwargs
+        )
+
+    def _download_records(
+        self, dataset_name: str, filepath: str, **kwargs: Any
+    ) -> List[Record]:
+        from huggingface_hub import hf_hub_download
+        from huggingface_hub.errors import EntryNotFoundError
+
+        try:
+            downloaded_file_path = hf_hub_download(
+                repo_id=dataset_name,
+                filename=filepath,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                token=self._api_key,
+                **kwargs,
+            )
+
+            with open(downloaded_file_path, "r") as f:
+                records_data = json.load(f)
+
+            return [Record(**record) for record in records_data]
+        except EntryNotFoundError:
+            logger.info(f"No records found for dataset '{dataset_name}'.")
+            return []
+        except Exception as e:
+            logger.error(f"Error downloading or processing records: {e}")
+            raise e
+
+    def _upload_records(
+        self,
+        records: List[Record],
+        dataset_name: str,
+        filepath: str,
+        **kwargs: Any,
+    ):
+        with tempfile.NamedTemporaryFile(
+            delete=False, mode="w", newline="", encoding="utf-8"
+        ) as f:
+            json.dump([record.model_dump() for record in records], f)
+            temp_file_path = f.name
+
+        try:
+            self.api.upload_file(
+                path_or_fileobj=temp_file_path,
+                path_in_repo=filepath,
+                repo_id=dataset_name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+        except Exception as e:
+            logger.error(f"Error uploading records file: {e}")
+            raise
+        finally:
+            if os.path.exists(temp_file_path):
+                os.remove(temp_file_path)
+
+    def _upload_file(
+        self,
+        file_content: str,
+        dataset_name: str,
+        filepath: str,
+        file_type: str = "json",
+        **kwargs: Any,
+    ):
+        with tempfile.NamedTemporaryFile(
+            mode="w", delete=False, suffix=f".{file_type}"
+        ) as f:
+            if file_type == "json":
+                if isinstance(file_content, str):
+                    try:
+                        json_content = json.loads(file_content)
+                    except json.JSONDecodeError:
+                        raise ValueError(
+                            "Invalid JSON string provided for file_content."
+                        )
+                else:
+                    try:
+                        json.dumps(file_content)
+                        json_content = file_content
+                    except (TypeError, ValueError):
+                        raise ValueError(
+                            "file_content is not JSON serializable."
+                        )
+
+                json.dump(json_content, f)
+            elif file_type == "md" or file_type == "txt":
+                f.write(file_content)
+            else:
+                raise ValueError(f"Unsupported file type: {file_type}")
+
+            temp_file_path = f.name
+
+        try:
+            self.api.upload_file(
+                path_or_fileobj=temp_file_path,
+                path_in_repo=filepath,
+                repo_id=dataset_name,
+                repo_type=HuggingFaceRepoType.DATASET.value,
+                **kwargs,
+            )
+            logger.info(f"File uploaded successfully: {filepath}")
+        except Exception as e:
+            logger.error(f"Error uploading file: {e}")
+            raise
+
+        if os.path.exists(temp_file_path):
+            os.remove(temp_file_path)

camel/datahubs/models.py

@@ -0,0 +1,22 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel
+
+
+class Record(BaseModel):
+    id: Optional[str] = None
+    metadata: Optional[Dict[str, Any]] = None
+    content: Dict[str, Any]

camel/embeddings/openai_compatible_embedding.py

@@ -46,7 +46,7 @@ class OpenAICompatibleEmbedding(BaseEmbedding[str]):
         )
         self._url = url or os.environ.get("OPENAI_COMPATIBILIY_API_BASE_URL")
         self._client = OpenAI(
-            timeout=60,
+            timeout=180,
             max_retries=3,
             api_key=self._api_key,
             base_url=self._url,

camel/embeddings/openai_embedding.py

@@ -56,7 +56,7 @@ class OpenAIEmbedding(BaseEmbedding[str]):
             assert isinstance(dimensions, int)
             self.output_dim = dimensions
         self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
-        self.client = OpenAI(timeout=60, max_retries=3, api_key=self._api_key)
+        self.client = OpenAI(timeout=180, max_retries=3, api_key=self._api_key)
 
     @api_keys_required("OPENAI_API_KEY")
     def embed_list(

camel/interpreters/__init__.py

@@ -14,6 +14,7 @@
 
 from .base import BaseInterpreter
 from .docker_interpreter import DockerInterpreter
+from .e2b_interpreter import E2BInterpreter
 from .internal_python_interpreter import InternalPythonInterpreter
 from .interpreter_error import InterpreterError
 from .ipython_interpreter import JupyterKernelInterpreter
@@ -26,4 +27,5 @@ __all__ = [
     'SubprocessInterpreter',
     'DockerInterpreter',
     'JupyterKernelInterpreter',
+    'E2BInterpreter',
 ]

camel/interpreters/e2b_interpreter.py

@@ -0,0 +1,136 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from typing import Any, ClassVar, Dict, List, Optional
+
+from camel.interpreters.base import BaseInterpreter
+from camel.interpreters.interpreter_error import InterpreterError
+from camel.logger import get_logger
+from camel.utils import api_keys_required
+
+logger = get_logger(__name__)
+
+
+class E2BInterpreter(BaseInterpreter):
+    r"""E2B Code Interpreter implementation.
+
+    Args:
+        require_confirm (bool, optional): If True, prompt user before running
+            code strings for security. (default: :obj:`True`)
+    """
+
+    _CODE_TYPE_MAPPING: ClassVar[Dict[str, Optional[str]]] = {
+        "python": None,
+        "py3": None,
+        "python3": None,
+        "py": None,
+        "shell": "bash",
+        "bash": "bash",
+        "sh": "bash",
+        "java": "java",
+        "javascript": "js",
+        "r": "r",
+    }
+
+    @api_keys_required("E2B_API_KEY")
+    def __init__(
+        self,
+        require_confirm: bool = True,
+    ) -> None:
+        from e2b_code_interpreter import Sandbox
+
+        self.require_confirm = require_confirm
+        self._sandbox = Sandbox(api_key=os.environ.get("E2B_API_KEY"))
+
+    def __del__(self) -> None:
+        r"""Destructor for the E2BInterpreter class.
+
+        This method ensures that the e2b sandbox is killed when the
+        interpreter is deleted.
+        """
+        if (
+            hasattr(self, '_sandbox')
+            and self._sandbox is not None
+            and self._sandbox.is_running()
+        ):
+            self._sandbox.kill()
+
+    def run(
+        self,
+        code: str,
+        code_type: str,
+    ) -> str:
+        r"""Executes the given code in the e2b sandbox.
+
+        Args:
+            code (str): The code string to execute.
+            code_type (str): The type of code to execute (e.g., 'python',
+                'bash').
+
+        Returns:
+            str: The string representation of the output of the executed code.
+
+        Raises:
+            InterpreterError: If the `code_type` is not supported or if any
+                runtime error occurs during the execution of the code.
+        """
+        if code_type not in self._CODE_TYPE_MAPPING:
+            raise InterpreterError(
+                f"Unsupported code type {code_type}. "
+                f"`{self.__class__.__name__}` only supports "
+                f"{', '.join(list(self._CODE_TYPE_MAPPING.keys()))}."
+            )
+        # Print code for security checking
+        if self.require_confirm:
+            logger.info(
+                f"The following {code_type} code will run on your "
+                "e2b sandbox: {code}"
+            )
+            while True:
+                choice = input("Running code? [Y/n]:").lower()
+                if choice in ["y", "yes", "ye"]:
+                    break
+                elif choice not in ["no", "n"]:
+                    continue
+                raise InterpreterError(
+                    "Execution halted: User opted not to run the code. "
+                    "This choice stops the current operation and any "
+                    "further code execution."
+                )
+
+        if self._CODE_TYPE_MAPPING[code_type] is None:
+            execution = self._sandbox.run_code(code)
+        else:
+            execution = self._sandbox.run_code(
+                code=code, language=self._CODE_TYPE_MAPPING[code_type]
+            )
+
+        if execution.text and execution.text.lower() != "none":
+            return execution.text
+
+        if execution.logs:
+            if execution.logs.stdout:
+                return ",".join(execution.logs.stdout)
+            elif execution.logs.stderr:
+                return ",".join(execution.logs.stderr)
+
+        return str(execution.error)
+
+    def supported_code_types(self) -> List[str]:
+        r"""Provides supported code types by the interpreter."""
+        return list(self._CODE_TYPE_MAPPING.keys())
+
+    def update_action_space(self, action_space: Dict[str, Any]) -> None:
+        r"""Updates action space for *python* interpreter"""
+        raise RuntimeError("E2B doesn't support " "`action_space`.")

camel/loaders/__init__.py

@@ -13,7 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 from .apify_reader import Apify
-from .base_io import File
+from .base_io import File, create_file, create_file_from_raw_bytes
 from .chunkr_reader import ChunkrReader
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
@@ -21,6 +21,8 @@ from .unstructured_io import UnstructuredIO
 
 __all__ = [
     'File',
+    'create_file',
+    'create_file_from_raw_bytes',
     'UnstructuredIO',
     'JinaURLReader',
     'Firecrawl',