camel-ai 0.1.5.3__py3-none-any.whl → 0.1.5.5__py3-none-any.whl

camel/interpreters/docker_interpreter.py

@@ -0,0 +1,235 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import io
+import shlex
+import tarfile
+import uuid
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional
+
+from colorama import Fore
+
+from camel.interpreters.base import BaseInterpreter
+from camel.interpreters.interpreter_error import InterpreterError
+from camel.utils import is_docker_running
+
+if TYPE_CHECKING:
+    from docker.models.containers import Container
+
+
+class DockerInterpreter(BaseInterpreter):
+    r"""A class for executing code files or code strings in a docker container.
+
+    This class handles the execution of code in different scripting languages
+    (currently Python and Bash) within a docker container, capturing their
+    stdout and stderr streams, and allowing user checking before executing code
+    strings.
+
+    Args:
+        require_confirm (bool, optional): If `True`, prompt user before
+            running code strings for security. Defaults to `True`.
+        print_stdout (bool, optional): If `True`, print the standard
+            output of the executed code. Defaults to `False`.
+        print_stderr (bool, optional): If `True`, print the standard error
+            of the executed code. Defaults to `True`.
+    """
+
+    _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
+        "python": "python {file_name}",
+        "bash": "bash {file_name}",
+    }
+
+    _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
+        "python": "py",
+        "bash": "sh",
+    }
+
+    _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
+        "python": "python",
+        "py3": "python",
+        "python3": "python",
+        "py": "python",
+        "shell": "bash",
+        "bash": "bash",
+        "sh": "bash",
+    }
+
+    def __init__(
+        self,
+        require_confirm: bool = True,
+        print_stdout: bool = False,
+        print_stderr: bool = True,
+    ) -> None:
+        self.require_confirm = require_confirm
+        self.print_stdout = print_stdout
+        self.print_stderr = print_stderr
+
+        # lazy initialization of container
+        self._container: Optional[Container] = None
+
+    def __del__(self) -> None:
+        if self._container is not None:
+            self._container.remove(force=True)
+
+    def _initialize_if_needed(self) -> None:
+        if self._container is not None:
+            return
+
+        if not is_docker_running():
+            raise InterpreterError(
+                "Docker daemon is not running. Please install/start docker "
+                "and try again."
+            )
+
+        import docker
+
+        client = docker.from_env()
+        self._container = client.containers.run(
+            "python:3.10",
+            detach=True,
+            name=f"camel-interpreter-{uuid.uuid4()}",
+            command="tail -f /dev/null",
+        )
+
+    def _create_file_in_container(self, content: str) -> Path:
+        # get a random name for the file
+        filename = str(uuid.uuid4())
+        # create a tar in memory
+        tar_stream = io.BytesIO()
+        with tarfile.open(fileobj=tar_stream, mode='w') as tar:
+            tarinfo = tarfile.TarInfo(name=filename)
+            tarinfo.size = len(content)
+            tar.addfile(tarinfo, io.BytesIO(content.encode('utf-8')))
+        tar_stream.seek(0)
+
+        # copy the tar into the container
+        if self._container is None:
+            raise InterpreterError(
+                "Container is not initialized. Try running the code again."
+            )
+        self._container.put_archive("/tmp", tar_stream)
+        return Path(f"/tmp/{filename}")
+
+    def _run_file_in_container(
+        self,
+        file: Path,
+        code_type: str,
+    ) -> str:
+        code_type = self._check_code_type(code_type)
+        commands = shlex.split(
+            self._CODE_EXECUTE_CMD_MAPPING[code_type].format(
+                file_name=str(file)
+            )
+        )
+        if self._container is None:
+            raise InterpreterError(
+                "Container is not initialized. Try running the code again."
+            )
+        stdout, stderr = self._container.exec_run(
+            commands,
+            demux=True,
+        ).output
+
+        if self.print_stdout and stdout:
+            print("======stdout======")
+            print(Fore.GREEN + stdout.decode() + Fore.RESET)
+            print("==================")
+        if self.print_stderr and stderr:
+            print("======stderr======")
+            print(Fore.RED + stderr.decode() + Fore.RESET)
+            print("==================")
+        exec_result = f"{stdout.decode()}" if stdout else ""
+        exec_result += f"(stderr: {stderr.decode()})" if stderr else ""
+        return exec_result
+
+    def run(
+        self,
+        code: str,
+        code_type: str,
+    ) -> str:
+        r"""Executes the given code in the conatiner attached to the
+        interpreter, and captures the stdout and stderr streams.
+
+        Args:
+            code (str): The code string to execute.
+            code_type (str): The type of code to execute (e.g., 'python',
+                'bash').
+
+        Returns:
+            str: A string containing the captured stdout and stderr of the
+                executed code.
+
+        Raises:
+            InterpreterError: If the user declines to run the code, or the
+                code type is unsupported, or there is an error in the docker
+                API/container
+        """
+        import docker.errors
+
+        code_type = self._check_code_type(code_type)
+
+        # Print code for security checking
+        if self.require_confirm:
+            print(f"The following {code_type} code will run in container:")
+            print(Fore.CYAN + code + Fore.RESET)
+            while True:
+                choice = input("Running code? [Y/n]:").lower()
+                if choice in ["y", "yes", "ye", ""]:
+                    break
+                elif choice not in ["no", "n"]:
+                    continue
+                raise InterpreterError(
+                    "Execution halted: User opted not to run the code. "
+                    "This choice stops the current operation and any "
+                    "further code execution."
+                )
+
+        self._initialize_if_needed()
+
+        try:
+            temp_file_path = self._create_file_in_container(code)
+            result = self._run_file_in_container(temp_file_path, code_type)
+        except docker.errors.APIError as e:
+            raise InterpreterError(
+                f"Execution halted due to docker API error: {e.explanation}. "
+                "This choice stops the current operation and any "
+                "further code execution."
+            ) from e
+        except docker.errors.DockerException as e:
+            raise InterpreterError(
+                f"Execution halted due to docker exceptoin: {e}. "
+                "This choice stops the current operation and any "
+                "further code execution."
+            ) from e
+        return result
+
+    def _check_code_type(self, code_type: str) -> str:
+        if code_type not in self._CODE_TYPE_MAPPING:
+            raise InterpreterError(
+                f"Unsupported code type {code_type}. Currently "
+                f"`{self.__class__.__name__}` only supports "
+                f"{', '.join(self._CODE_EXTENSION_MAPPING.keys())}."
+            )
+        return self._CODE_TYPE_MAPPING[code_type]
+
+    def supported_code_types(self) -> List[str]:
+        r"""Provides supported code types by the interpreter."""
+        return list(self._CODE_EXTENSION_MAPPING.keys())
+
+    def update_action_space(self, action_space: Dict[str, Any]) -> None:
+        r"""Updates action space for *python* interpreter"""
+        raise RuntimeError(
+            "SubprocessInterpreter doesn't support " "`action_space`."
+        )

camel/loaders/__init__.py

@@ -13,10 +13,12 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 
 from .base_io import File, read_file
+from .jina_url_reader import JinaURLReader
 from .unstructured_io import UnstructuredIO
 
 __all__ = [
     'File',
     'read_file',
     'UnstructuredIO',
+    'JinaURLReader',
 ]

camel/loaders/base_io.py

@@ -19,6 +19,8 @@ from hashlib import md5
 from io import BytesIO
 from typing import Any, Dict, List, Optional
 
+from camel.utils import dependencies_required
+
 
 class File(ABC):
     r"""Represents an uploaded file comprised of Documents"""
@@ -94,6 +96,7 @@ def strip_consecutive_newlines(text: str) -> str:
 
 class DocxFile(File):
     @classmethod
+    @dependencies_required('docx2txt')
     def from_bytes(cls, file: BytesIO) -> "DocxFile":
         r"""Creates a DocxFile object from a BytesIO object.
 
@@ -104,15 +107,8 @@ class DocxFile(File):
         Returns:
             DocxFile: A DocxFile object.
         """
-        # Use docx2txt to extract text from docx files
-        try:
-            import docx2txt
-        except ImportError:
-            raise ImportError(
-                "Please install `docx2txt` first. "
-                "You can install it by running "
-                "`pip install docx2txt`."
-            )
+        import docx2txt
+
         text = docx2txt.process(file)
         text = strip_consecutive_newlines(text)
         # Create a dictionary with the extracted text

camel/loaders/jina_url_reader.py

@@ -0,0 +1,99 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from typing import Any, Optional
+
+from camel.types.enums import JinaReturnFormat
+
+JINA_ENDPOINT = "https://r.jina.ai/"
+
+
+class JinaURLReader:
+    r"""URL Reader provided by Jina AI. The output is cleaner and more
+    LLM-friendly than the URL Reader of UnstructuredIO. Can be configured to
+    replace the UnstructuredIO URL Reader in the pipeline.
+
+    Args:
+        api_key (Optional[str], optional): The API key for Jina AI. If not
+            provided, the reader will have a lower rate limit. Defaults to
+            None.
+        return_format (ReturnFormat, optional): The level of detail
+            of the returned content, which is optimized for LLMs. For
+            now screenshots are not supported. Defaults to
+            ReturnFormat.DEFAULT.
+        json_response (bool, optional): Whether to return the response
+            in JSON format. Defaults to False.
+        timeout (int, optional): The maximum time in seconds to wait for
+            the page to be rendered. Defaults to 30.
+        **kwargs (Any): Additional keyword arguments, including proxies,
+            cookies, etc. It should align with the HTTP Header field and
+            value pairs listed in the reference.
+
+    References:
+        https://jina.ai/reader
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        return_format: JinaReturnFormat = JinaReturnFormat.DEFAULT,
+        json_response: bool = False,
+        timeout: int = 30,
+        **kwargs: Any,
+    ) -> None:
+        api_key = api_key or os.getenv('JINA_API_KEY')
+        if api_key is None:
+            print(
+                "[JinaURLReader] JINA_API_KEY not set. This will result in a "
+                "low rate limit of Jina URL Reader. Get API key here: "
+                "https://jina.ai/reader."
+            )
+
+        # if the following field not provided, it will be None
+        api_field = f"Bearer {api_key}" if api_key else None
+        json_field = "application/json" if json_response else None
+
+        raw_headers = {
+            "Authorization": api_field,
+            "X-Return-Format": return_format.value,
+            "Accept": json_field,
+            "X-Timeout": str(timeout),
+            **kwargs,
+        }
+
+        # eliminate None values
+        self._headers = {k: v for k, v in raw_headers.items() if v}
+
+    def read_content(self, url: str) -> str:
+        r"""Reads the content of a URL and returns it as a string with
+        given form.
+
+        Args:
+            url (str): The URL to read.
+
+        Returns:
+            str: The content of the URL.
+        """
+
+        import requests
+
+        full_url = f"{JINA_ENDPOINT}{url}"
+        try:
+            resp = requests.get(full_url, headers=self._headers)
+            resp.raise_for_status()
+        except Exception as e:
+            raise Exception(f"Failed to read content from {url}: {e}") from e
+
+        return resp.text

camel/loaders/unstructured_io.py

@@ -16,6 +16,8 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 from unstructured.documents.elements import Element
 
+from camel.utils import dependencies_required
+
 
 class UnstructuredIO:
     r"""A class to handle various functionalities provided by the
@@ -37,6 +39,7 @@ class UnstructuredIO:
         """
         self._ensure_unstructured_version(self.UNSTRUCTURED_MIN_VERSION)
 
+    @dependencies_required('unstructured')
     def _ensure_unstructured_version(self, min_version: str) -> None:
         r"""Validates that the installed 'Unstructured' library version
         satisfies the specified minimum version requirement. This function is
@@ -58,12 +61,7 @@ class UnstructuredIO:
                 strings.
         """
         from packaging import version
-
-        try:
-            from unstructured.__version__ import __version__
-
-        except ImportError as e:
-            raise ImportError("Package `unstructured` not installed.") from e
+        from unstructured.__version__ import __version__
 
         # Use packaging.version to compare versions
         min_ver = version.parse(min_version)

camel/models/anthropic_model.py

@@ -23,7 +23,7 @@ from camel.types import ChatCompletion, ModelType
 from camel.utils import (
     AnthropicTokenCounter,
     BaseTokenCounter,
-    model_api_key_required,
+    api_keys_required,
 )
 
 
@@ -46,11 +46,13 @@ class AnthropicModel(BaseModelBackend):
                 be fed into Anthropic.messages.create().
             api_key (Optional[str]): The API key for authenticating with the
                 Anthropic service. (default: :obj:`None`)
-            url (Optional[str]): The url to the model service.
+            url (Optional[str]): The url to the Anthropic service. (default:
+                :obj:`None`)
         """
         super().__init__(model_type, model_config_dict, api_key, url)
         self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
-        self.client = Anthropic(api_key=self._api_key, base_url=url)
+        self._url = url or os.environ.get("ANTHROPIC_API_BASE_URL")
+        self.client = Anthropic(api_key=self._api_key, base_url=self._url)
         self._token_counter: Optional[BaseTokenCounter] = None
 
     def _convert_response_from_anthropic_to_openai(self, response):
@@ -96,7 +98,7 @@ class AnthropicModel(BaseModelBackend):
         """
         return self.client.count_tokens(prompt)
 
-    @model_api_key_required
+    @api_keys_required("ANTHROPIC_API_KEY")
     def run(
         self,
         messages: List[OpenAIMessage],

camel/models/litellm_model.py

@@ -11,24 +11,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 from camel.configs import LITELLM_API_PARAMS
 from camel.messages import OpenAIMessage
+from camel.types import ChatCompletion
 from camel.utils import LiteLLMTokenCounter
 
-if TYPE_CHECKING:
-    from litellm.utils import CustomStreamWrapper, ModelResponse
-
 
 class LiteLLMModel:
     r"""Constructor for LiteLLM backend with OpenAI compatibility."""
 
-    # NOTE: Currently "stream": True is not supported with LiteLLM due to the
-    # limitation of the current camel design.
+    # NOTE: Currently stream mode is not supported.
 
     def __init__(
-        self, model_type: str, model_config_dict: Dict[str, Any]
+        self,
+        model_type: str,
+        model_config_dict: Dict[str, Any],
+        api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Constructor for LiteLLM backend.
 
@@ -37,12 +38,48 @@ class LiteLLMModel:
                 such as GPT-3.5-turbo, Claude-2, etc.
             model_config_dict (Dict[str, Any]): A dictionary of parameters for
                 the model configuration.
+            api_key (Optional[str]): The API key for authenticating with the
+                model service. (default: :obj:`None`)
+            url (Optional[str]): The url to the model service. (default:
+                :obj:`None`)
         """
         self.model_type = model_type
         self.model_config_dict = model_config_dict
         self._client = None
         self._token_counter: Optional[LiteLLMTokenCounter] = None
         self.check_model_config()
+        self._url = url
+        self._api_key = api_key
+
+    def _convert_response_from_litellm_to_openai(
+        self, response
+    ) -> ChatCompletion:
+        r"""Converts a response from the LiteLLM format to the OpenAI format.
+
+        Parameters:
+            response (LiteLLMResponse): The response object from LiteLLM.
+
+        Returns:
+            ChatCompletion: The response object in OpenAI's format.
+        """
+        return ChatCompletion.construct(
+            id=response.id,
+            choices=[
+                {
+                    "index": response.choices[0].index,
+                    "message": {
+                        "role": response.choices[0].message.role,
+                        "content": response.choices[0].message.content,
+                    },
+                    "finish_reason": response.choices[0].finish_reason,
+                }
+            ],
+            created=response.created,
+            model=response.model,
+            object=response.object,
+            system_fingerprint=response.system_fingerprint,
+            usage=response.usage,
+        )
 
     @property
     def client(self):
@@ -67,7 +104,7 @@ class LiteLLMModel:
     def run(
         self,
         messages: List[OpenAIMessage],
-    ) -> Union['ModelResponse', 'CustomStreamWrapper']:
+    ) -> ChatCompletion:
         r"""Runs inference of LiteLLM chat completion.
 
         Args:
@@ -75,15 +112,16 @@ class LiteLLMModel:
                 in OpenAI format.
 
         Returns:
-            Union[ModelResponse, CustomStreamWrapper]:
-                `ModelResponse` in the non-stream mode, or
-                `CustomStreamWrapper` in the stream mode.
+            ChatCompletion
         """
         response = self.client(
+            api_key=self._api_key,
+            base_url=self._url,
             model=self.model_type,
             messages=messages,
             **self.model_config_dict,
         )
+        response = self._convert_response_from_litellm_to_openai(response)
         return response
 
     def check_model_config(self):
@@ -100,13 +138,3 @@ class LiteLLMModel:
                     f"Unexpected argument `{param}` is "
                     "input into LiteLLM model backend."
                 )
-
-    @property
-    def stream(self) -> bool:
-        r"""Returns whether the model is in stream mode, which sends partial
-        results each time.
-
-        Returns:
-            bool: Whether the model is in stream mode.
-        """
-        return self.model_config_dict.get('stream', False)

camel/models/model_factory.py

@@ -80,6 +80,7 @@ class ModelFactory:
         elif isinstance(model_type, str):
             if model_platform.is_ollama:
                 model_class = OllamaModel
+                return model_class(model_type, model_config_dict, url)
             elif model_platform.is_litellm:
                 model_class = LiteLLMModel
             else:

camel/models/nemotron_model.py

@@ -20,7 +20,7 @@ from camel.messages import OpenAIMessage
 from camel.types import ChatCompletion, ModelType
 from camel.utils import (
     BaseTokenCounter,
-    model_api_key_required,
+    api_keys_required,
 )
 
 
@@ -33,6 +33,7 @@ class NemotronModel:
         self,
         model_type: ModelType,
         api_key: Optional[str] = None,
+        url: Optional[str] = None,
     ) -> None:
         r"""Constructor for Nvidia backend.
 
@@ -40,18 +41,25 @@ class NemotronModel:
             model_type (ModelType): Model for which a backend is created.
             api_key (Optional[str]): The API key for authenticating with the
                 Nvidia service. (default: :obj:`None`)
+            url (Optional[str]): The url to the Nvidia service. (default:
+                :obj:`None`)
         """
         self.model_type = model_type
-        url = os.environ.get('NVIDIA_API_BASE_URL', None)
+        self._url = url or os.environ.get("NVIDIA_API_BASE_URL")
         self._api_key = api_key or os.environ.get("NVIDIA_API_KEY")
-        if not url or not self._api_key:
-            raise ValueError("The NVIDIA API base url and key should be set.")
+        if not self._url or not self._api_key:
+            raise ValueError(
+                "NVIDIA_API_BASE_URL and NVIDIA_API_KEY should be set."
+            )
         self._client = OpenAI(
-            timeout=60, max_retries=3, base_url=url, api_key=self._api_key
+            timeout=60,
+            max_retries=3,
+            base_url=self._url,
+            api_key=self._api_key,
         )
         self._token_counter: Optional[BaseTokenCounter] = None
 
-    @model_api_key_required
+    @api_keys_required("NVIDIA_API_KEY")
     def run(
         self,
         messages: List[OpenAIMessage],