camel-ai 0.2.21__py3-none-any.whl → 0.2.23a0__py3-none-any.whl

camel/extractors/base.py

@@ -0,0 +1,263 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from abc import ABC, abstractmethod
+from types import TracebackType
+from typing import Any, Dict, Optional, Type
+
+from typing_extensions import Self
+
+from camel.logger import get_logger
+from camel.utils import BatchProcessor
+
+logger = get_logger(__name__)
+
+
+class BaseExtractor(ABC):
+    r"""Base class for all response extractors.
+
+    An extractor takes the response and extracts the relevant parts,
+    converting them into a format that the verifier can handle.
+    Implements async context manager protocol for proper resource management.
+    """
+
+    def __init__(
+        self,
+        cache_templates: bool = True,
+        max_cache_size: int = 1000,
+        extraction_timeout: float = 30.0,
+        batch_size: int = 10,
+        monitoring_interval: float = 5.0,
+        cpu_threshold: float = 80.0,
+        memory_threshold: float = 85.0,
+        **kwargs,
+    ):
+        r"""Initialize the extractor.
+
+        Args:
+            cache_templates (bool): Whether to cache extraction templates.
+                (default: :obj:`True`)
+            max_cache_size (int): Maximum number of templates to cache.
+                (default: :obj:`1000`)
+            extraction_timeout (float): Maximum time for extraction in seconds.
+                (default: :obj:`30.0`)
+            batch_size (int): Size of batches for parallel extraction.
+                (default: :obj:`10`)
+            monitoring_interval (float): Interval in seconds between resource
+                checks. (default: :obj:`5.0`)
+            cpu_threshold (float): CPU usage percentage threshold for scaling
+                down. (default: :obj:`80.0`)
+            memory_threshold (float): Memory usage percentage threshold for
+                scaling down. (default: :obj:`85.0`)
+            **kwargs: Additional extractor parameters.
+
+        Raises:
+            ValueError: If invalid parameter values are provided
+        """
+        # Store all parameters in metadata dict for compatibility
+        self._metadata = {
+            'cache_templates': cache_templates,
+            'max_cache_size': max_cache_size,
+            'extraction_timeout': extraction_timeout,
+            'batch_size': batch_size,
+            'monitoring_interval': monitoring_interval,
+            'cpu_threshold': cpu_threshold,
+            'memory_threshold': memory_threshold,
+            **kwargs,
+        }
+
+        self._is_setup = False
+        self._cache: Dict[str, Any] = {}
+        self._batch_processor: Optional[BatchProcessor] = None
+
+        # Store configuration parameters
+        self._cache_templates = cache_templates
+        self._max_cache_size = max_cache_size
+        self._extraction_timeout = extraction_timeout
+        self._batch_size = batch_size
+        self._monitoring_interval = monitoring_interval
+        self._cpu_threshold = cpu_threshold
+        self._memory_threshold = memory_threshold
+
+    async def setup(self) -> None:
+        r"""Set up the extractor with necessary resources.
+
+        This method:
+        1. Initializes template cache if enabled
+        2. Sets up any parallel processing resources
+        3. Validates extraction patterns
+
+        Raises:
+            RuntimeError: If initialization fails
+        """
+        if self._is_setup:
+            logger.debug(f"{self.__class__.__name__} already initialized")
+            return
+
+        try:
+            # Initialize template cache if enabled
+            if self._cache_templates:
+                self._template_cache: Dict[str, Any] = {}
+
+            # Set up batch processing if needed
+            if self._batch_size > 1:
+                self._batch_processor = BatchProcessor(
+                    initial_batch_size=self._batch_size,
+                    monitoring_interval=self._monitoring_interval,
+                    cpu_threshold=self._cpu_threshold,
+                    memory_threshold=self._memory_threshold,
+                )
+
+            self._is_setup = True
+            logger.info(f"{self.__class__.__name__} initialized successfully")
+
+        except Exception as e:
+            error_msg = f"Error during {self.__class__.__name__} setup: {e}"
+            logger.error(error_msg)
+            await self.cleanup()
+            raise RuntimeError(error_msg) from e
+
+    async def cleanup(self) -> None:
+        r"""Clean up extractor resources.
+
+        This method handles cleanup of resources and resets the extractor
+        state.
+        It ensures:
+        1. All resources are properly released
+        2. Template cache is cleared
+        3. Parallel processing resources are shutdown
+        4. State is reset to initial
+        5. Cleanup happens even if errors occur
+
+        Raises:
+            RuntimeError: If cleanup fails (after resetting initialization
+                state).
+        """
+        if not self._is_setup:
+            logger.debug(
+                f"{self.__class__.__name__} not initialized, skipping cleanup"
+            )
+            return
+
+        errors = []
+        try:
+            # Clear template cache
+            if hasattr(self, '_template_cache'):
+                try:
+                    self._template_cache.clear()
+                except Exception as e:
+                    errors.append(f"Failed to clear template cache: {e}")
+
+            # Shutdown parallel processing
+            if self._batch_processor is not None:
+                try:
+                    # Get final performance metrics before cleanup
+                    metrics = self._batch_processor.get_performance_metrics()
+                    logger.info(f"Batch processor final metrics: {metrics}")
+                except Exception as e:
+                    errors.append(
+                        f"Failed to get batch processor metrics: {e}"
+                    )
+
+            # Preserve init config in metadata
+            self._metadata = {
+                'cache_templates': self._cache_templates,
+                'max_cache_size': self._max_cache_size,
+                'extraction_timeout': self._extraction_timeout,
+                'batch_size': self._batch_size,
+            }
+
+            if not errors:
+                logger.info(
+                    f"{self.__class__.__name__} cleaned up successfully"
+                )
+
+        except Exception as e:
+            errors.append(f"Unexpected error during cleanup: {e}")
+
+        finally:
+            # Always mark as uninitialized, even if cleanup fails
+            self._is_setup = False
+            self._batch_processor = None
+
+        if errors:
+            error_msg = (
+                f"Errors during {self.__class__.__name__} cleanup: "
+                f"{'; '.join(errors)}"
+            )
+            logger.error(error_msg)
+            raise RuntimeError(error_msg)
+
+    async def __aenter__(self) -> Self:
+        r"""Async context manager entry.
+
+        Returns:
+            Self reference for context manager usage.
+        """
+        await self.setup()
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        r"""Async context manager exit.
+
+        Args:
+            exc_type (Optional[Type[BaseException]]): Exception type if an
+                error occurred.
+            exc_val (Optional[BaseException]): Exception value if an error
+                occurred.
+            exc_tb (Optional[TracebackType]): Exception traceback if an error
+                occurred.
+        """
+        await self.cleanup()
+
+    @abstractmethod
+    async def extract(
+        self, response: str, context: Optional[Dict[str, Any]] = None
+    ) -> str:
+        r"""Extract relevant parts from a response.
+
+        Extracts:
+        1. Final answer or output
+        2. Chain of thought reasoning steps
+        3. Difficulty assessment
+
+        Args:
+            response (str): Raw response from agent generation.
+            context (Optional[Dict[str, Any]]): Optional context for
+            extraction like:
+                - final_answer
+                - rationale
+                - complexity
+
+        Returns:
+            str: Extracted content string.
+
+        Raises:
+            ValueError: If response is empty or invalid.
+            NotImplementedError: If no implementation is provided.
+            RuntimeError: If extractor is not initialized.
+        """
+        if not self._is_setup:
+            raise RuntimeError(
+                f"{self.__class__.__name__} must be initialized "
+                "before extraction"
+            )
+        if not response or not response.strip():
+            raise ValueError("Empty or whitespace-only response")
+        raise NotImplementedError("Subclasses must implement extract()")

camel/interpreters/docker/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.9-slim
+
+# Install R and required dependencies
+RUN apt-get update && apt-get install -y \
+    r-base \
+    && rm -rf /var/lib/apt/lists/*
+
+# Set working directory
+WORKDIR /workspace
+
+# Keep container running
+CMD ["tail", "-f", "/dev/null"]

camel/interpreters/docker_interpreter.py

@@ -52,11 +52,13 @@ class DockerInterpreter(BaseInterpreter):
     _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "python {file_name}",
         "bash": "bash {file_name}",
+        "r": "Rscript {file_name}",
     }
 
     _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "py",
         "bash": "sh",
+        "r": "R",
     }
 
     _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
@@ -67,6 +69,8 @@ class DockerInterpreter(BaseInterpreter):
         "shell": "bash",
         "bash": "bash",
         "sh": "bash",
+        "r": "r",
+        "R": "r",
     }
 
     def __init__(
@@ -104,8 +108,22 @@ class DockerInterpreter(BaseInterpreter):
         import docker
 
         client = docker.from_env()
+
+        # Build custom image with Python and R
+        dockerfile_path = Path(__file__).parent / "docker"
+        image_tag = "camel-interpreter:latest"
+        try:
+            client.images.get(image_tag)
+        except docker.errors.ImageNotFound:
+            logger.info("Building custom interpreter image...")
+            client.images.build(
+                path=str(dockerfile_path),
+                tag=image_tag,
+                rm=True,
+            )
+
         self._container = client.containers.run(
-            "python:3.10",
+            image_tag,
             detach=True,
             name=f"camel-interpreter-{uuid.uuid4()}",
             command="tail -f /dev/null",

camel/interpreters/subprocess_interpreter.py

@@ -48,11 +48,13 @@ class SubprocessInterpreter(BaseInterpreter):
     _CODE_EXECUTE_CMD_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "python {file_name}",
         "bash": "bash {file_name}",
+        "r": "Rscript {file_name}",
     }
 
     _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = {
         "python": "py",
         "bash": "sh",
+        "r": "R",
     }
 
     _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = {
@@ -63,6 +65,8 @@ class SubprocessInterpreter(BaseInterpreter):
         "shell": "bash",
         "bash": "bash",
         "sh": "bash",
+        "r": "r",
+        "R": "r",
     }
 
     def __init__(
@@ -98,7 +102,7 @@ class SubprocessInterpreter(BaseInterpreter):
         if not file.is_file():
             raise RuntimeError(f"{file} is not a file.")
         code_type = self._check_code_type(code_type)
-        if code_type == "python":
+        if self._CODE_TYPE_MAPPING[code_type] == "python":
             # For Python code, use ast to analyze and modify the code
             import ast
 
@@ -113,23 +117,41 @@ class SubprocessInterpreter(BaseInterpreter):
                 # Get the last node
                 if tree.body:
                     last_node = tree.body[-1]
-                    # If it's an expression, wrap it in a print
+                    # Handle expressions that would normally not produce output
+                    # For example: In a REPL, typing '1 + 2' should show '3'
+
                     if isinstance(last_node, ast.Expr):
-                        tree.body[-1] = ast.Expr(
-                            value=ast.Call(
-                                func=ast.Name(id='print', ctx=ast.Load()),
-                                args=[
-                                    ast.Call(
-                                        func=ast.Name(
-                                            id='repr', ctx=ast.Load()
-                                        ),
-                                        args=[last_node.value],
-                                        keywords=[],
-                                    )
-                                ],
-                                keywords=[],
+                        # Only wrap in print(repr()) if it's not already a
+                        # print call
+                        if not (
+                            isinstance(last_node.value, ast.Call)
+                            and isinstance(last_node.value.func, ast.Name)
+                            and last_node.value.func.id == 'print'
+                        ):
+                            # Transform the AST to wrap the expression in print
+                            # (repr())
+                            # Example transformation:
+                            #   Before: x + y
+                            #   After:  print(repr(x + y))
+                            tree.body[-1] = ast.Expr(
+                                value=ast.Call(
+                                    # Create print() function call
+                                    func=ast.Name(id='print', ctx=ast.Load()),
+                                    args=[
+                                        ast.Call(
+                                            # Create repr() function call
+                                            func=ast.Name(
+                                                id='repr', ctx=ast.Load()
+                                            ),
+                                            # Pass the original expression as
+                                            # argument to repr()
+                                            args=[last_node.value],
+                                            keywords=[],
+                                        )
+                                    ],
+                                    keywords=[],
+                                )
                             )
-                        )
                     # Fix missing source locations
                     ast.fix_missing_locations(tree)
                     # Convert back to source
@@ -159,7 +181,10 @@ class SubprocessInterpreter(BaseInterpreter):
         return_code = proc.returncode
 
         # Clean up temporary file if it was created
-        if code_type == "python" and 'temp_file' in locals():
+        if (
+            self._CODE_TYPE_MAPPING[code_type] == "python"
+            and 'temp_file' in locals()
+        ):
             temp_file.unlink()
 
         if self.print_stdout and stdout:

camel/loaders/__init__.py

@@ -17,6 +17,7 @@ from .base_io import File, create_file, create_file_from_raw_bytes
 from .chunkr_reader import ChunkrReader
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
+from .mineru_extractor import MinerU
 from .panda_reader import PandaReader
 from .unstructured_io import UnstructuredIO
 
@@ -30,4 +31,5 @@ __all__ = [
     'Apify',
     'ChunkrReader',
     'PandaReader',
+    'MinerU',
 ]

camel/loaders/mineru_extractor.py

@@ -0,0 +1,250 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import os
+import time
+from typing import Dict, List, Optional, Union
+
+import requests
+
+from camel.utils import api_keys_required
+
+
+class MinerU:
+    r"""Document extraction service supporting OCR, formula recognition
+        and tables.
+
+    Args:
+        api_key (str, optional): Authentication key for MinerU API service.
+            If not provided, will use MINERU_API_KEY environment variable.
+            (default: :obj:`None`)
+        api_url (str, optional): Base URL endpoint for the MinerU API service.
+            (default: :obj:`"https://mineru.net/api/v4"`)
+
+    Note:
+        - Single file size limit: 200MB
+        - Page limit per file: 600 pages
+        - Daily high-priority parsing quota: 2000 pages
+        - Some URLs (GitHub, AWS) may timeout due to network restrictions
+    """
+
+    @api_keys_required(
+        [
+            ("api_key", "MINERU_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = "https://mineru.net/api/v4",
+        is_ocr: bool = False,
+        enable_formula: bool = False,
+        enable_table: bool = True,
+        layout_model: str = "doclayout_yolo",
+        language: str = "en",
+    ) -> None:
+        r"""Initialize MinerU extractor.
+
+        Args:
+            api_key (str, optional): Authentication key for MinerU API service.
+                If not provided, will use MINERU_API_KEY environment variable.
+            api_url (str, optional): Base URL endpoint for MinerU API service.
+                (default: "https://mineru.net/api/v4")
+            is_ocr (bool, optional): Enable optical character recognition.
+                (default: :obj:`False`)
+            enable_formula (bool, optional): Enable formula recognition.
+                (default: :obj:`False`)
+            enable_table (bool, optional): Enable table detection, extraction.
+                (default: :obj:`True`)
+            layout_model (str, optional): Model for document layout detection.
+                Options are 'doclayout_yolo' or 'layoutlmv3'.
+                (default: :obj:`"doclayout_yolo"`)
+            language (str, optional): Primary language of the document.
+                (default: :obj:`"en"`)
+        """
+        self._api_key = api_key or os.environ.get("MINERU_API_KEY")
+        self._api_url = api_url
+        self._headers = {
+            "Authorization": f"Bearer {self._api_key}",
+            "Content-Type": "application/json",
+            "Accept": "*/*",
+        }
+        self.is_ocr = is_ocr
+        self.enable_formula = enable_formula
+        self.enable_table = enable_table
+        self.layout_model = layout_model
+        self.language = language
+
+    def extract_url(self, url: str) -> Dict:
+        r"""Extract content from a URL document.
+
+        Args:
+            url (str): Document URL to extract content from.
+
+        Returns:
+            Dict: Task identifier for tracking extraction progress.
+        """
+        endpoint = f"{self._api_url}/extract/task"
+        payload = {"url": url}
+
+        try:
+            response = requests.post(
+                endpoint,
+                headers=self._headers,
+                json=payload,
+            )
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to extract URL: {e}")
+
+    def batch_extract_urls(
+        self,
+        files: List[Dict[str, Union[str, bool]]],
+    ) -> str:
+        r"""Extract content from multiple document URLs in batch.
+
+        Args:
+            files (List[Dict[str, Union[str, bool]]]): List of document
+                configurations. Each document requires 'url' and optionally
+                'is_ocr' and 'data_id' parameters.
+
+        Returns:
+            str: Batch identifier for tracking extraction progress.
+        """
+        endpoint = f"{self._api_url}/extract/task/batch"
+        payload = {"files": files}
+
+        try:
+            response = requests.post(
+                endpoint,
+                headers=self._headers,
+                json=payload,
+            )
+            response.raise_for_status()
+            return response.json()["data"]["batch_id"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to batch extract URLs: {e}")
+
+    def get_task_status(self, task_id: str) -> Dict:
+        r"""Retrieve status of a single extraction task.
+
+        Args:
+            task_id (str): Unique identifier of the extraction task.
+
+        Returns:
+            Dict: Current task status and results if completed.
+        """
+        endpoint = f"{self._api_url}/extract/task/{task_id}"
+
+        try:
+            response = requests.get(endpoint, headers=self._headers)
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to get task status: {e}")
+
+    def get_batch_status(self, batch_id: str) -> Dict:
+        r"""Retrieve status of a batch extraction task.
+
+        Args:
+            batch_id (str): Unique identifier of the batch extraction task.
+
+        Returns:
+            Dict: Current status and results for all documents in the batch.
+        """
+        endpoint = f"{self._api_url}/extract-results/batch/{batch_id}"
+
+        try:
+            response = requests.get(endpoint, headers=self._headers)
+            response.raise_for_status()
+            return response.json()["data"]
+        except Exception as e:
+            raise RuntimeError(f"Failed to get batch status: {e}")
+
+    def wait_for_completion(
+        self,
+        task_id: str,
+        is_batch: bool = False,
+        timeout: float = 100,
+        check_interval: float = 5,
+    ) -> Dict:
+        r"""Monitor task until completion or timeout.
+
+        Args:
+            task_id (str): Unique identifier of the task or batch.
+            is_batch (bool, optional): Indicates if task is a batch operation.
+                (default: :obj:`False`)
+            timeout (float, optional): Maximum wait time in seconds.
+                (default: :obj:`100`)
+            check_interval (float, optional): Time between status checks in
+                seconds. (default: :obj:`5`)
+
+        Returns:
+            Dict: Final task status and extraction results.
+
+        Raises:
+            TimeoutError: If task exceeds specified timeout duration.
+            RuntimeError: If task fails or encounters processing error.
+        """
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                raise TimeoutError(
+                    f"Task {task_id} timed out after {timeout}s"
+                )
+
+            try:
+                status = (
+                    self.get_batch_status(task_id)
+                    if is_batch
+                    else self.get_task_status(task_id)
+                )
+
+                if is_batch:
+                    # Check batch status
+                    all_done = True
+                    failed_tasks = []
+                    for result in status.get('extract_result', []):
+                        if result.get('state') == 'failed':
+                            failed_tasks.append(
+                                f"{result.get('data_id')}:"
+                                f" {result.get('err_msg')}"
+                            )
+                        elif result.get('state') != 'done':
+                            all_done = False
+                            break
+
+                    if failed_tasks:
+                        raise RuntimeError(
+                            f"Batch tasks failed: {'; '.join(failed_tasks)}"
+                        )
+                    if all_done:
+                        return status
+                else:
+                    # Check single task status
+                    state = status.get('state')
+                    if state == 'failed':
+                        raise RuntimeError(
+                            f"Task failed: {status.get('err_msg')}"
+                        )
+                    elif state == 'done':
+                        return status
+
+            except Exception as e:
+                if not isinstance(e, RuntimeError):
+                    raise RuntimeError(f"Error checking status: {e}")
+                raise
+
+            time.sleep(check_interval)