camel-ai 0.2.20a1__py3-none-any.whl → 0.2.22__py3-none-any.whl

camel/datagen/self_instruct/self_instruct.py

@@ -361,7 +361,7 @@ class SelfInstructPipeline:
         in JSON format.
         """
         with open(self.data_output_path, 'w') as f:
-            json.dump(self.machine_tasks, f, indent=4)
+            json.dump(self.machine_tasks, f, indent=4, ensure_ascii=False)
 
     def generate(self):
         r"""Execute the entire pipeline to generate machine instructions

camel/embeddings/openai_embedding.py

@@ -30,6 +30,8 @@ class OpenAIEmbedding(BaseEmbedding[str]):
         model_type (EmbeddingModelType, optional): The model type to be
             used for text embeddings.
             (default: :obj:`TEXT_EMBEDDING_3_SMALL`)
+        url (Optional[str], optional): The url to the OpenAI service.
+            (default: :obj:`None`)
         api_key (str, optional): The API key for authenticating with the
             OpenAI service. (default: :obj:`None`)
         dimensions (int, optional): The text embedding output dimensions.
@@ -49,6 +51,7 @@ class OpenAIEmbedding(BaseEmbedding[str]):
         model_type: EmbeddingModelType = (
             EmbeddingModelType.TEXT_EMBEDDING_3_SMALL
         ),
+        url: str | None = None,
         api_key: str | None = None,
         dimensions: int | NotGiven = NOT_GIVEN,
     ) -> None:
@@ -61,7 +64,13 @@ class OpenAIEmbedding(BaseEmbedding[str]):
             assert isinstance(dimensions, int)
             self.output_dim = dimensions
         self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
-        self.client = OpenAI(timeout=180, max_retries=3, api_key=self._api_key)
+        self._url = url or os.environ.get("OPENAI_API_BASE_URL")
+        self.client = OpenAI(
+            timeout=180,
+            max_retries=3,
+            base_url=self._url,
+            api_key=self._api_key,
+        )
 
     def embed_list(
         self,

camel/interpreters/docker/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.9-slim
+
+# Install R and required dependencies
+RUN apt-get update && apt-get install -y \
+    r-base \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /workspace
+
+# Keep container running
+CMD ["tail", "-f", "/dev/null"]

camel/interpreters/docker_interpreter.py

@@ -52,11 +52,13 @@ class DockerInterpreter(BaseInterpreter):
     _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "python {file_name}",
         "bash": "bash {file_name}",
+        "r": "Rscript {file_name}",
     }
 
     _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "py",
         "bash": "sh",
+        "r": "R",
     }
 
     _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
@@ -67,6 +69,8 @@ class DockerInterpreter(BaseInterpreter):
         "shell": "bash",
         "bash": "bash",
         "sh": "bash",
+        "r": "r",
+        "R": "r",
     }
 
     def __init__(
@@ -104,8 +108,22 @@ class DockerInterpreter(BaseInterpreter):
         import docker
 
         client = docker.from_env()
+
+        # Build custom image with Python and R
+        dockerfile_path = Path(__file__).parent / "docker"
+        image_tag = "camel-interpreter:latest"
+        try:
+            client.images.get(image_tag)
+        except docker.errors.ImageNotFound:
+            logger.info("Building custom interpreter image...")
+            client.images.build(
+                path=str(dockerfile_path),
+                tag=image_tag,
+                rm=True,
+            )
+
         self._container = client.containers.run(
-            "python:3.10",
+            image_tag,
             detach=True,
             name=f"camel-interpreter-{uuid.uuid4()}",
             command="tail -f /dev/null",

camel/interpreters/subprocess_interpreter.py

@@ -48,11 +48,13 @@ class SubprocessInterpreter(BaseInterpreter):
     _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "python {file_name}",
         "bash": "bash {file_name}",
+        "r": "Rscript {file_name}",
     }
 
     _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "py",
         "bash": "sh",
+        "r": "R",
     }
 
     _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
@@ -63,6 +65,8 @@ class SubprocessInterpreter(BaseInterpreter):
         "shell": "bash",
         "bash": "bash",
         "sh": "bash",
+        "r": "r",
+        "R": "r",
     }
 
     def __init__(
@@ -98,15 +102,91 @@ class SubprocessInterpreter(BaseInterpreter):
         if not file.is_file():
             raise RuntimeError(f"{file} is not a file.")
         code_type = self._check_code_type(code_type)
-        cmd = shlex.split(
-            self._CODE_EXECUTE_CMD_MAPPING[code_type].format(
-                file_name=str(file)
+        if self._CODE_TYPE_MAPPING[code_type] == "python":
+            # For Python code, use ast to analyze and modify the code
+            import ast
+
+            import astor
+
+            with open(file, 'r') as f:
+                source = f.read()
+
+            # Parse the source code
+            try:
+                tree = ast.parse(source)
+                # Get the last node
+                if tree.body:
+                    last_node = tree.body[-1]
+                    # Handle expressions that would normally not produce output
+                    # For example: In a REPL, typing '1 + 2' should show '3'
+
+                    if isinstance(last_node, ast.Expr):
+                        # Only wrap in print(repr()) if it's not already a
+                        # print call
+                        if not (
+                            isinstance(last_node.value, ast.Call)
+                            and isinstance(last_node.value.func, ast.Name)
+                            and last_node.value.func.id == 'print'
+                        ):
+                            # Transform the AST to wrap the expression in print
+                            # (repr())
+                            # Example transformation:
+                            #   Before: x + y
+                            #   After:  print(repr(x + y))
+                            tree.body[-1] = ast.Expr(
+                                value=ast.Call(
+                                    # Create print() function call
+                                    func=ast.Name(id='print', ctx=ast.Load()),
+                                    args=[
+                                        ast.Call(
+                                            # Create repr() function call
+                                            func=ast.Name(
+                                                id='repr', ctx=ast.Load()
+                                            ),
+                                            # Pass the original expression as
+                                            # argument to repr()
+                                            args=[last_node.value],
+                                            keywords=[],
+                                        )
+                                    ],
+                                    keywords=[],
+                                )
+                            )
+                    # Fix missing source locations
+                    ast.fix_missing_locations(tree)
+                    # Convert back to source
+                    modified_source = astor.to_source(tree)
+                    # Create a temporary file with the modified source
+                    temp_file = self._create_temp_file(modified_source, "py")
+                    cmd = shlex.split(f"python {temp_file!s}")
+            except SyntaxError:
+                # If parsing fails, run the original file
+                cmd = shlex.split(
+                    self._CODE_EXECUTE_CMD_MAPPING[code_type].format(
+                        file_name=str(file)
+                    )
+                )
+        else:
+            # For non-Python code, use standard execution
+            cmd = shlex.split(
+                self._CODE_EXECUTE_CMD_MAPPING[code_type].format(
+                    file_name=str(file)
+                )
             )
-        )
+
         proc = subprocess.Popen(
             cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True
         )
         stdout, stderr = proc.communicate()
+        return_code = proc.returncode
+
+        # Clean up temporary file if it was created
+        if (
+            self._CODE_TYPE_MAPPING[code_type] == "python"
+            and 'temp_file' in locals()
+        ):
+            temp_file.unlink()
+
         if self.print_stdout and stdout:
             print("======stdout======")
             print(Fore.GREEN + stdout + Fore.RESET)
@@ -115,8 +195,19 @@ class SubprocessInterpreter(BaseInterpreter):
             print("======stderr======")
             print(Fore.RED + stderr + Fore.RESET)
             print("==================")
-        exec_result = f"{stdout}"
-        exec_result += f"(stderr: {stderr})" if stderr else ""
+
+        # Build the execution result
+        exec_result = ""
+        if stdout:
+            exec_result += stdout
+        if stderr:
+            exec_result += f"(stderr: {stderr})"
+        if return_code != 0:
+            error_msg = f"(Execution failed with return code {return_code})"
+            if not stderr:
+                exec_result += error_msg
+            elif error_msg not in stderr:
+                exec_result += error_msg
         return exec_result
 
     def run(

camel/loaders/__init__.py

@@ -17,6 +17,7 @@ from .base_io import File, create_file, create_file_from_raw_bytes
 from .chunkr_reader import ChunkrReader
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
+from .mineru_extractor import MinerU
 from .panda_reader import PandaReader
 from .unstructured_io import UnstructuredIO
 
@@ -30,4 +31,5 @@ __all__ = [
     'Apify',
     'ChunkrReader',
     'PandaReader',
+    'MinerU',
 ]

camel/loaders/mineru_extractor.py

@@ -0,0 +1,250 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import time
+from typing import Dict, List, Optional, Union
+
+import requests
+
+from camel.utils import api_keys_required
+
+
+class MinerU:
+    r"""Document extraction service supporting OCR, formula recognition
+        and tables.
+
+    Args:
+        api_key (str, optional): Authentication key for MinerU API service.
+            If not provided, will use MINERU_API_KEY environment variable.
+            (default: :obj:`None`)
+        api_url (str, optional): Base URL endpoint for the MinerU API service.
+            (default: :obj:`"https://mineru.net/api/v4"`)
+
+    Note:
+        - Single file size limit: 200MB
+        - Page limit per file: 600 pages
+        - Daily high-priority parsing quota: 2000 pages
+        - Some URLs (GitHub, AWS) may timeout due to network restrictions
+    """
+
+    @api_keys_required(
+        [
+            ("api_key", "MINERU_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = "https://mineru.net/api/v4",
+        is_ocr: bool = False,
+        enable_formula: bool = False,
+        enable_table: bool = True,
+        layout_model: str = "doclayout_yolo",
+        language: str = "en",
+    ) -> None:
+        r"""Initialize MinerU extractor.
+
+        Args:
+            api_key (str, optional): Authentication key for MinerU API service.
+                If not provided, will use MINERU_API_KEY environment variable.
+            api_url (str, optional): Base URL endpoint for MinerU API service.
+                (default: "https://mineru.net/api/v4")
+            is_ocr (bool, optional): Enable optical character recognition.
+                (default: :obj:`False`)
+            enable_formula (bool, optional): Enable formula recognition.
+                (default: :obj:`False`)
+            enable_table (bool, optional): Enable table detection, extraction.
+                (default: :obj:`True`)
+            layout_model (str, optional): Model for document layout detection.
+                Options are 'doclayout_yolo' or 'layoutlmv3'.
+                (default: :obj:`"doclayout_yolo"`)
+            language (str, optional): Primary language of the document.
+                (default: :obj:`"en"`)
+        """
+        self._api_key = api_key or os.environ.get("MINERU_API_KEY")
+        self._api_url = api_url
+        self._headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "Accept": "*/*",
+        }
+        self.is_ocr = is_ocr
+        self.enable_formula = enable_formula
+        self.enable_table = enable_table
+        self.layout_model = layout_model
+        self.language = language
+
+    def extract_url(self, url: str) -> Dict:
+        r"""Extract content from a URL document.
+
+        Args:
+            url (str): Document URL to extract content from.
+
+        Returns:
+            Dict: Task identifier for tracking extraction progress.
+        """
+        endpoint = f"{self._api_url}/extract/task"
+        payload = {"url": url}
+
+        try:
+            response = requests.post(
+                endpoint,
+                headers=self._headers,
+                json=payload,
+            )
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to extract URL: {e}")
+
+    def batch_extract_urls(
+        self,
+        files: List[Dict[str, Union[str, bool]]],
+    ) -> str:
+        r"""Extract content from multiple document URLs in batch.
+
+        Args:
+            files (List[Dict[str, Union[str, bool]]]): List of document
+                configurations. Each document requires 'url' and optionally
+                'is_ocr' and 'data_id' parameters.
+
+        Returns:
+            str: Batch identifier for tracking extraction progress.
+        """
+        endpoint = f"{self._api_url}/extract/task/batch"
+        payload = {"files": files}
+
+        try:
+            response = requests.post(
+                endpoint,
+                headers=self._headers,
+                json=payload,
+            )
+            response.raise_for_status()
+            return response.json()["data"]["batch_id"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to batch extract URLs: {e}")
+
+    def get_task_status(self, task_id: str) -> Dict:
+        r"""Retrieve status of a single extraction task.
+
+        Args:
+            task_id (str): Unique identifier of the extraction task.
+
+        Returns:
+            Dict: Current task status and results if completed.
+        """
+        endpoint = f"{self._api_url}/extract/task/{task_id}"
+
+        try:
+            response = requests.get(endpoint, headers=self._headers)
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to get task status: {e}")
+
+    def get_batch_status(self, batch_id: str) -> Dict:
+        r"""Retrieve status of a batch extraction task.
+
+        Args:
+            batch_id (str): Unique identifier of the batch extraction task.
+
+        Returns:
+            Dict: Current status and results for all documents in the batch.
+        """
+        endpoint = f"{self._api_url}/extract-results/batch/{batch_id}"
+
+        try:
+            response = requests.get(endpoint, headers=self._headers)
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to get batch status: {e}")
+
+    def wait_for_completion(
+        self,
+        task_id: str,
+        is_batch: bool = False,
+        timeout: float = 100,
+        check_interval: float = 5,
+    ) -> Dict:
+        r"""Monitor task until completion or timeout.
+
+        Args:
+            task_id (str): Unique identifier of the task or batch.
+            is_batch (bool, optional): Indicates if task is a batch operation.
+                (default: :obj:`False`)
+            timeout (float, optional): Maximum wait time in seconds.
+                (default: :obj:`100`)
+            check_interval (float, optional): Time between status checks in
+                seconds. (default: :obj:`5`)
+
+        Returns:
+            Dict: Final task status and extraction results.
+
+        Raises:
+            TimeoutError: If task exceeds specified timeout duration.
+            RuntimeError: If task fails or encounters processing error.
+        """
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} timed out after {timeout}s"
+                )
+
+            try:
+                status = (
+                    self.get_batch_status(task_id)
+                    if is_batch
+                    else self.get_task_status(task_id)
+                )
+
+                if is_batch:
+                    # Check batch status
+                    all_done = True
+                    failed_tasks = []
+                    for result in status.get('extract_result', []):
+                        if result.get('state') == 'failed':
+                            failed_tasks.append(
+                                f"{result.get('data_id')}:"
+                                f" {result.get('err_msg')}"
+                            )
+                        elif result.get('state') != 'done':
+                            all_done = False
+                            break
+
+                    if failed_tasks:
+                        raise RuntimeError(
+                            f"Batch tasks failed: {'; '.join(failed_tasks)}"
+                        )
+                    if all_done:
+                        return status
+                else:
+                    # Check single task status
+                    state = status.get('state')
+                    if state == 'failed':
+                        raise RuntimeError(
+                            f"Task failed: {status.get('err_msg')}"
+                        )
+                    elif state == 'done':
+                        return status
+
+            except Exception as e:
+                if not isinstance(e, RuntimeError):
+                    raise RuntimeError(f"Error checking status: {e}")
+                raise
+
+            time.sleep(check_interval)

camel/models/__init__.py

@@ -11,6 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+from .aiml_model import AIMLModel
 from .anthropic_model import AnthropicModel
 from .azure_openai_model import AzureOpenAIModel
 from .base_model import BaseModelBackend
@@ -72,4 +73,5 @@ __all__ = [
     'FishAudioModel',
     'InternLMModel',
     'MoonshotModel',
+    'AIMLModel',
 ]

camel/models/aiml_model.py

@@ -0,0 +1,147 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from typing import Any, Dict, List, Optional, Union
+
+from openai import OpenAI, Stream
+
+from camel.configs import AIML_API_PARAMS, AIMLConfig
+from camel.messages import OpenAIMessage
+from camel.models.base_model import BaseModelBackend
+from camel.types import (
+    ChatCompletion,
+    ChatCompletionChunk,
+    ModelType,
+)
+from camel.utils import (
+    BaseTokenCounter,
+    OpenAITokenCounter,
+    api_keys_required,
+)
+
+
+class AIMLModel(BaseModelBackend):
+    r"""AIML API in a unified BaseModelBackend interface.
+
+    Args:
+        model_type (Union[ModelType, str]): Model for which a backend is
+            created.
+        model_config_dict (Optional[Dict[str, Any]], optional): A dictionary
+            that will be fed into OpenAI client. If :obj:`None`,
+            :obj:`AIMLConfig().as_dict()` will be used.
+            (default: :obj:`None`)
+        api_key (Optional[str], optional): The API key for authenticating with
+            the AIML service. (default: :obj:`None`)
+        url (Optional[str], optional): The URL to the AIML service. If
+            not provided, :obj:`https://api.aimlapi.com/v1` will be used.
+            (default: :obj:`None`)
+        token_counter (Optional[BaseTokenCounter], optional): Token counter to
+            use for the model. If not provided, :obj:`OpenAITokenCounter(
+            ModelType.GPT_4O_MINI)` will be used.
+            (default: :obj:`None`)
+    """
+
+    @api_keys_required(
+        [
+            ("api_key", 'AIML_API_KEY'),
+        ]
+    )
+    def __init__(
+        self,
+        model_type: Union[ModelType, str],
+        model_config_dict: Optional[Dict[str, Any]] = None,
+        api_key: Optional[str] = None,
+        url: Optional[str] = None,
+        token_counter: Optional[BaseTokenCounter] = None,
+    ) -> None:
+        if model_config_dict is None:
+            model_config_dict = AIMLConfig().as_dict()
+        api_key = api_key or os.environ.get("AIML_API_KEY")
+        url = url or os.environ.get(
+            "AIML_API_BASE_URL",
+            "https://api.aimlapi.com/v1",
+        )
+        super().__init__(
+            model_type, model_config_dict, api_key, url, token_counter
+        )
+        self._client = OpenAI(
+            timeout=180,
+            max_retries=3,
+            api_key=self._api_key,
+            base_url=self._url,
+        )
+
+    def run(
+        self,
+        messages: List[OpenAIMessage],
+    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+        r"""Runs inference of OpenAI chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+                `ChatCompletion` in the non-stream mode, or
+                `Stream[ChatCompletionChunk]` in the stream mode.
+        """
+        # Process model configuration parameters
+        model_config = self.model_config_dict.copy()
+
+        # Handle special case for tools parameter
+        if model_config.get('tools') is None:
+            model_config['tools'] = []
+
+        response = self._client.chat.completions.create(
+            messages=messages, model=self.model_type, **model_config
+        )
+        return response
+
+    @property
+    def token_counter(self) -> BaseTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            BaseTokenCounter: The token counter following the model's
+                tokenization style.
+        """
+        if not self._token_counter:
+            self._token_counter = OpenAITokenCounter(ModelType.GPT_4O_MINI)
+        return self._token_counter
+
+    def check_model_config(self):
+        r"""Check whether the model configuration contains any
+        unexpected arguments to AIML API.
+
+        Raises:
+            ValueError: If the model configuration dictionary contains any
+                unexpected arguments to AIML API.
+        """
+        for param in self.model_config_dict:
+            if param not in AIML_API_PARAMS:
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into AIML model backend."
+                )
+
+    @property
+    def stream(self) -> bool:
+        """Returns whether the model is in stream mode, which sends partial
+        results each time.
+
+        Returns:
+            bool: Whether the model is in stream mode.
+        """
+        return self.model_config_dict.get('stream', False)