camel-ai 0.2.21__tar.gz → 0.2.22__tar.gz

{camel_ai-0.2.21 → camel_ai-0.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: camel-ai
-Version: 0.2.21
+Version: 0.2.22
 Summary: Communicative Agents for AI Society Study
 License: Apache-2.0
 Keywords: communicative-ai,ai-societies,artificial-intelligence,deep-learning,multi-agent-systems,cooperative-ai,natural-language-processing,large-language-models
@@ -91,7 +91,6 @@ Requires-Dist: pymilvus (>=2.4.0,<3.0.0) ; extra == "rag" or extra == "storage"
 Requires-Dist: pyowm (>=3.3.0,<4.0.0) ; extra == "web-tools" or extra == "all"
 Requires-Dist: pytest (>=7,<8) ; extra == "test"
 Requires-Dist: pytest-asyncio (>=0.23.0,<0.24.0) ; extra == "test"
-Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: qdrant-client (>=1.9.0,<2.0.0) ; extra == "rag" or extra == "storage" or extra == "all"
 Requires-Dist: rank-bm25 (>=0.2.2,<0.3.0) ; extra == "rag" or extra == "all"
 Requires-Dist: redis (>=5.0.6,<6.0.0) ; extra == "storage" or extra == "all"

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/__init__.py

@@ -14,7 +14,7 @@
 
 from camel.logger import disable_logging, enable_logging, set_log_level
 
-__version__ = '0.2.21'
+__version__ = '0.2.22'
 
 __all__ = [
     '__version__',

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/datagen/self_instruct/self_instruct.py

@@ -361,7 +361,7 @@ class SelfInstructPipeline:
         in JSON format.
         """
         with open(self.data_output_path, 'w') as f:
-            json.dump(self.machine_tasks, f, indent=4)
+            json.dump(self.machine_tasks, f, indent=4, ensure_ascii=False)
 
     def generate(self):
         r"""Execute the entire pipeline to generate machine instructions

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/embeddings/openai_embedding.py

@@ -30,6 +30,8 @@ class OpenAIEmbedding(BaseEmbedding[str]):
         model_type (EmbeddingModelType, optional): The model type to be
             used for text embeddings.
             (default: :obj:`TEXT_EMBEDDING_3_SMALL`)
+        url (Optional[str], optional): The url to the OpenAI service.
+            (default: :obj:`None`)
         api_key (str, optional): The API key for authenticating with the
             OpenAI service. (default: :obj:`None`)
         dimensions (int, optional): The text embedding output dimensions.
@@ -49,6 +51,7 @@ class OpenAIEmbedding(BaseEmbedding[str]):
         model_type: EmbeddingModelType = (
             EmbeddingModelType.TEXT_EMBEDDING_3_SMALL
         ),
+        url: str | None = None,
         api_key: str | None = None,
         dimensions: int | NotGiven = NOT_GIVEN,
     ) -> None:
@@ -61,7 +64,13 @@ class OpenAIEmbedding(BaseEmbedding[str]):
             assert isinstance(dimensions, int)
             self.output_dim = dimensions
         self._api_key = api_key or os.environ.get("OPENAI_API_KEY")
-        self.client = OpenAI(timeout=180, max_retries=3, api_key=self._api_key)
+        self._url = url or os.environ.get("OPENAI_API_BASE_URL")
+        self.client = OpenAI(
+            timeout=180,
+            max_retries=3,
+            base_url=self._url,
+            api_key=self._api_key,
+        )
 
     def embed_list(
         self,

camel_ai-0.2.22/camel/interpreters/docker/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.9-slim
+
+# Install R and required dependencies
+RUN apt-get update && apt-get install -y \
+    r-base \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /workspace
+
+# Keep container running
+CMD ["tail", "-f", "/dev/null"]

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/interpreters/docker_interpreter.py

@@ -52,11 +52,13 @@ class DockerInterpreter(BaseInterpreter):
     _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "python {file_name}",
         "bash": "bash {file_name}",
+        "r": "Rscript {file_name}",
     }
 
     _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "py",
         "bash": "sh",
+        "r": "R",
     }
 
     _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
@@ -67,6 +69,8 @@ class DockerInterpreter(BaseInterpreter):
         "shell": "bash",
         "bash": "bash",
         "sh": "bash",
+        "r": "r",
+        "R": "r",
     }
 
     def __init__(
@@ -104,8 +108,22 @@ class DockerInterpreter(BaseInterpreter):
         import docker
 
         client = docker.from_env()
+
+        # Build custom image with Python and R
+        dockerfile_path = Path(__file__).parent / "docker"
+        image_tag = "camel-interpreter:latest"
+        try:
+            client.images.get(image_tag)
+        except docker.errors.ImageNotFound:
+            logger.info("Building custom interpreter image...")
+            client.images.build(
+                path=str(dockerfile_path),
+                tag=image_tag,
+                rm=True,
+            )
+
         self._container = client.containers.run(
-            "python:3.10",
+            image_tag,
             detach=True,
             name=f"camel-interpreter-{uuid.uuid4()}",
             command="tail -f /dev/null",

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/interpreters/subprocess_interpreter.py

@@ -48,11 +48,13 @@ class SubprocessInterpreter(BaseInterpreter):
     _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "python {file_name}",
         "bash": "bash {file_name}",
+        "r": "Rscript {file_name}",
     }
 
     _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "py",
         "bash": "sh",
+        "r": "R",
     }
 
     _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
@@ -63,6 +65,8 @@ class SubprocessInterpreter(BaseInterpreter):
         "shell": "bash",
         "bash": "bash",
         "sh": "bash",
+        "r": "r",
+        "R": "r",
     }
 
     def __init__(
@@ -98,7 +102,7 @@ class SubprocessInterpreter(BaseInterpreter):
         if not file.is_file():
             raise RuntimeError(f"{file} is not a file.")
         code_type = self._check_code_type(code_type)
-        if code_type == "python":
+        if self._CODE_TYPE_MAPPING[code_type] == "python":
             # For Python code, use ast to analyze and modify the code
             import ast
 
@@ -113,23 +117,41 @@ class SubprocessInterpreter(BaseInterpreter):
                 # Get the last node
                 if tree.body:
                     last_node = tree.body[-1]
-                    # If it's an expression, wrap it in a print
+                    # Handle expressions that would normally not produce output
+                    # For example: In a REPL, typing '1 + 2' should show '3'
+
                     if isinstance(last_node, ast.Expr):
-                        tree.body[-1] = ast.Expr(
-                            value=ast.Call(
-                                func=ast.Name(id='print', ctx=ast.Load()),
-                                args=[
-                                    ast.Call(
-                                        func=ast.Name(
-                                            id='repr', ctx=ast.Load()
-                                        ),
-                                        args=[last_node.value],
-                                        keywords=[],
-                                    )
-                                ],
-                                keywords=[],
+                        # Only wrap in print(repr()) if it's not already a
+                        # print call
+                        if not (
+                            isinstance(last_node.value, ast.Call)
+                            and isinstance(last_node.value.func, ast.Name)
+                            and last_node.value.func.id == 'print'
+                        ):
+                            # Transform the AST to wrap the expression in print
+                            # (repr())
+                            # Example transformation:
+                            #   Before: x + y
+                            #   After:  print(repr(x + y))
+                            tree.body[-1] = ast.Expr(
+                                value=ast.Call(
+                                    # Create print() function call
+                                    func=ast.Name(id='print', ctx=ast.Load()),
+                                    args=[
+                                        ast.Call(
+                                            # Create repr() function call
+                                            func=ast.Name(
+                                                id='repr', ctx=ast.Load()
+                                            ),
+                                            # Pass the original expression as
+                                            # argument to repr()
+                                            args=[last_node.value],
+                                            keywords=[],
+                                        )
+                                    ],
+                                    keywords=[],
+                                )
                             )
-                        )
                     # Fix missing source locations
                     ast.fix_missing_locations(tree)
                     # Convert back to source
@@ -159,7 +181,10 @@ class SubprocessInterpreter(BaseInterpreter):
         return_code = proc.returncode
 
         # Clean up temporary file if it was created
-        if code_type == "python" and 'temp_file' in locals():
+        if (
+            self._CODE_TYPE_MAPPING[code_type] == "python"
+            and 'temp_file' in locals()
+        ):
             temp_file.unlink()
 
         if self.print_stdout and stdout:

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/loaders/__init__.py

@@ -17,6 +17,7 @@ from .base_io import File, create_file, create_file_from_raw_bytes
 from .chunkr_reader import ChunkrReader
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
+from .mineru_extractor import MinerU
 from .panda_reader import PandaReader
 from .unstructured_io import UnstructuredIO
 
@@ -30,4 +31,5 @@ __all__ = [
     'Apify',
     'ChunkrReader',
     'PandaReader',
+    'MinerU',
 ]

camel_ai-0.2.22/camel/loaders/mineru_extractor.py

@@ -0,0 +1,250 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import time
+from typing import Dict, List, Optional, Union
+
+import requests
+
+from camel.utils import api_keys_required
+
+
+class MinerU:
+    r"""Document extraction service supporting OCR, formula recognition
+        and tables.
+
+    Args:
+        api_key (str, optional): Authentication key for MinerU API service.
+            If not provided, will use MINERU_API_KEY environment variable.
+            (default: :obj:`None`)
+        api_url (str, optional): Base URL endpoint for the MinerU API service.
+            (default: :obj:`"https://mineru.net/api/v4"`)
+
+    Note:
+        - Single file size limit: 200MB
+        - Page limit per file: 600 pages
+        - Daily high-priority parsing quota: 2000 pages
+        - Some URLs (GitHub, AWS) may timeout due to network restrictions
+    """
+
+    @api_keys_required(
+        [
+            ("api_key", "MINERU_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = "https://mineru.net/api/v4",
+        is_ocr: bool = False,
+        enable_formula: bool = False,
+        enable_table: bool = True,
+        layout_model: str = "doclayout_yolo",
+        language: str = "en",
+    ) -> None:
+        r"""Initialize MinerU extractor.
+
+        Args:
+            api_key (str, optional): Authentication key for MinerU API service.
+                If not provided, will use MINERU_API_KEY environment variable.
+            api_url (str, optional): Base URL endpoint for MinerU API service.
+                (default: "https://mineru.net/api/v4")
+            is_ocr (bool, optional): Enable optical character recognition.
+                (default: :obj:`False`)
+            enable_formula (bool, optional): Enable formula recognition.
+                (default: :obj:`False`)
+            enable_table (bool, optional): Enable table detection, extraction.
+                (default: :obj:`True`)
+            layout_model (str, optional): Model for document layout detection.
+                Options are 'doclayout_yolo' or 'layoutlmv3'.
+                (default: :obj:`"doclayout_yolo"`)
+            language (str, optional): Primary language of the document.
+                (default: :obj:`"en"`)
+        """
+        self._api_key = api_key or os.environ.get("MINERU_API_KEY")
+        self._api_url = api_url
+        self._headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "Accept": "*/*",
+        }
+        self.is_ocr = is_ocr
+        self.enable_formula = enable_formula
+        self.enable_table = enable_table
+        self.layout_model = layout_model
+        self.language = language
+
+    def extract_url(self, url: str) -> Dict:
+        r"""Extract content from a URL document.
+
+        Args:
+            url (str): Document URL to extract content from.
+
+        Returns:
+            Dict: Task identifier for tracking extraction progress.
+        """
+        endpoint = f"{self._api_url}/extract/task"
+        payload = {"url": url}
+
+        try:
+            response = requests.post(
+                endpoint,
+                headers=self._headers,
+                json=payload,
+            )
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to extract URL: {e}")
+
+    def batch_extract_urls(
+        self,
+        files: List[Dict[str, Union[str, bool]]],
+    ) -> str:
+        r"""Extract content from multiple document URLs in batch.
+
+        Args:
+            files (List[Dict[str, Union[str, bool]]]): List of document
+                configurations. Each document requires 'url' and optionally
+                'is_ocr' and 'data_id' parameters.
+
+        Returns:
+            str: Batch identifier for tracking extraction progress.
+        """
+        endpoint = f"{self._api_url}/extract/task/batch"
+        payload = {"files": files}
+
+        try:
+            response = requests.post(
+                endpoint,
+                headers=self._headers,
+                json=payload,
+            )
+            response.raise_for_status()
+            return response.json()["data"]["batch_id"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to batch extract URLs: {e}")
+
+    def get_task_status(self, task_id: str) -> Dict:
+        r"""Retrieve status of a single extraction task.
+
+        Args:
+            task_id (str): Unique identifier of the extraction task.
+
+        Returns:
+            Dict: Current task status and results if completed.
+        """
+        endpoint = f"{self._api_url}/extract/task/{task_id}"
+
+        try:
+            response = requests.get(endpoint, headers=self._headers)
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to get task status: {e}")
+
+    def get_batch_status(self, batch_id: str) -> Dict:
+        r"""Retrieve status of a batch extraction task.
+
+        Args:
+            batch_id (str): Unique identifier of the batch extraction task.
+
+        Returns:
+            Dict: Current status and results for all documents in the batch.
+        """
+        endpoint = f"{self._api_url}/extract-results/batch/{batch_id}"
+
+        try:
+            response = requests.get(endpoint, headers=self._headers)
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to get batch status: {e}")
+
+    def wait_for_completion(
+        self,
+        task_id: str,
+        is_batch: bool = False,
+        timeout: float = 100,
+        check_interval: float = 5,
+    ) -> Dict:
+        r"""Monitor task until completion or timeout.
+
+        Args:
+            task_id (str): Unique identifier of the task or batch.
+            is_batch (bool, optional): Indicates if task is a batch operation.
+                (default: :obj:`False`)
+            timeout (float, optional): Maximum wait time in seconds.
+                (default: :obj:`100`)
+            check_interval (float, optional): Time between status checks in
+                seconds. (default: :obj:`5`)
+
+        Returns:
+            Dict: Final task status and extraction results.
+
+        Raises:
+            TimeoutError: If task exceeds specified timeout duration.
+            RuntimeError: If task fails or encounters processing error.
+        """
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} timed out after {timeout}s"
+                )
+
+            try:
+                status = (
+                    self.get_batch_status(task_id)
+                    if is_batch
+                    else self.get_task_status(task_id)
+                )
+
+                if is_batch:
+                    # Check batch status
+                    all_done = True
+                    failed_tasks = []
+                    for result in status.get('extract_result', []):
+                        if result.get('state') == 'failed':
+                            failed_tasks.append(
+                                f"{result.get('data_id')}:"
+                                f" {result.get('err_msg')}"
+                            )
+                        elif result.get('state') != 'done':
+                            all_done = False
+                            break
+
+                    if failed_tasks:
+                        raise RuntimeError(
+                            f"Batch tasks failed: {'; '.join(failed_tasks)}"
+                        )
+                    if all_done:
+                        return status
+                else:
+                    # Check single task status
+                    state = status.get('state')
+                    if state == 'failed':
+                        raise RuntimeError(
+                            f"Task failed: {status.get('err_msg')}"
+                        )
+                    elif state == 'done':
+                        return status
+
+            except Exception as e:
+                if not isinstance(e, RuntimeError):
+                    raise RuntimeError(f"Error checking status: {e}")
+                raise
+
+            time.sleep(check_interval)

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/models/base_model.py

@@ -11,6 +11,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import abc
+import re
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Union
 
@@ -27,7 +29,30 @@ from camel.types import (
 from camel.utils import BaseTokenCounter
 
 
-class BaseModelBackend(ABC):
+class ModelBackendMeta(abc.ABCMeta):
+    r"""Metaclass that automatically preprocesses messages in run method.
+
+    Automatically wraps the run method of any class inheriting from
+    BaseModelBackend to preprocess messages (remove <think> tags) before they
+    are sent to the model.
+    """
+
+    def __new__(mcs, name, bases, namespace):
+        r"""Wraps run method with preprocessing if it exists in the class."""
+        if 'run' in namespace:
+            original_run = namespace['run']
+
+            def wrapped_run(
+                self, messages: List[OpenAIMessage], *args, **kwargs
+            ):
+                messages = self.preprocess_messages(messages)
+                return original_run(self, messages, *args, **kwargs)
+
+            namespace['run'] = wrapped_run
+        return super().__new__(mcs, name, bases, namespace)
+
+
+class BaseModelBackend(ABC, metaclass=ModelBackendMeta):
     r"""Base class for different model backends.
     It may be OpenAI API, a local LLM, a stub for unit tests, etc.
 
@@ -73,6 +98,34 @@ class BaseModelBackend(ABC):
         """
         pass
 
+    def preprocess_messages(
+        self, messages: List[OpenAIMessage]
+    ) -> List[OpenAIMessage]:
+        r"""Preprocess messages before sending to model API.
+        Removes thinking content and other model-specific preprocessing.
+
+        Args:
+            messages (List[OpenAIMessage]): Original messages
+
+        Returns:
+            List[OpenAIMessage]: Preprocessed messages
+        """
+        # Remove thinking content from messages before sending to API
+        # This ensures only the final response is sent, excluding
+        # intermediate thought processes
+        return [
+            {  # type: ignore[misc]
+                **msg,
+                'content': re.sub(
+                    r'<think>.*?</think>',
+                    '',
+                    msg['content'],  # type: ignore[arg-type]
+                    flags=re.DOTALL,
+                ).strip(),
+            }
+            for msg in messages
+        ]
+
     @abstractmethod
     def run(
         self,

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/models/deepseek_model.py

@@ -118,8 +118,6 @@ class DeepSeekModel(BaseModelBackend):
         if self.model_type in [
             ModelType.DEEPSEEK_REASONER,
         ]:
-            import re
-
             logger.warning(
                 "You are using a DeepSeek Reasoner model, "
                 "which has certain limitations, reference: "
@@ -141,22 +139,6 @@ class DeepSeekModel(BaseModelBackend):
                 if key in self.model_config_dict:
                     del self.model_config_dict[key]
 
-            # Remove thinking content from messages before sending to API
-            # This ensures only the final response is sent, excluding
-            # intermediate thought processes
-            messages = [
-                {  # type: ignore[misc]
-                    **msg,
-                    'content': re.sub(
-                        r'<think>.*?</think>',
-                        '',
-                        msg['content'],  # type: ignore[arg-type]
-                        flags=re.DOTALL,
-                    ).strip(),
-                }
-                for msg in messages
-            ]
-
         response = self._client.chat.completions.create(
             messages=messages,
             model=self.model_type,

{camel_ai-0.2.21 → camel_ai-0.2.22}/camel/toolkits/__init__.py

@@ -45,8 +45,10 @@ from .human_toolkit import HumanToolkit
 from .stripe_toolkit import StripeToolkit
 from .video_toolkit import VideoDownloaderToolkit
 from .dappier_toolkit import DappierToolkit
-from .sympy_toolkit import SymPyToolkit
 from .semantic_scholar_toolkit import SemanticScholarToolkit
+from .sympy_toolkit import SymPyToolkit
+from .mineru_toolkit import MinerUToolkit
+
 
 __all__ = [
     'BaseToolkit',
@@ -79,6 +81,7 @@ __all__ = [
     'MeshyToolkit',
     'OpenBBToolkit',
     'DappierToolkit',
-    'SymPyToolkit',
     'SemanticScholarToolkit',
+    'SymPyToolkit',
+    'MinerUToolkit',
 ]