camel-ai 0.2.59__py3-none-any.whl → 0.2.82__py3-none-any.whl

camel/loaders/__init__.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,17 +10,17 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 from .apify_reader import Apify
 from .base_io import File, create_file, create_file_from_raw_bytes
-from .chunkr_reader import ChunkrReader
+from .chunkr_reader import ChunkrReader, ChunkrReaderConfig
 from .crawl4ai_reader import Crawl4AI
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
 from .markitdown import MarkItDownLoader
 from .mineru_extractor import MinerU
-from .pandas_reader import PandasReader
+from .mistral_reader import MistralReader
 from .scrapegraph_reader import ScrapeGraphAI
 from .unstructured_io import UnstructuredIO
 
@@ -33,9 +33,21 @@ __all__ = [
     'Firecrawl',
     'Apify',
     'ChunkrReader',
-    'PandasReader',
+    'ChunkrReaderConfig',
     'MinerU',
     'Crawl4AI',
     'MarkItDownLoader',
     'ScrapeGraphAI',
+    'MistralReader',
 ]
+
+
+def __getattr__(name: str):
+    if name == 'PandasReader':
+        raise ImportError(
+            "PandasReader has been removed from camel.loaders. "
+            "The pandasai dependency limited pandas to version 1.5.3. "
+            "Please use ExcelToolkit from camel.toolkits instead for "
+            "handling structured data."
+        )
+    raise AttributeError(f"module '{__name__}' has no attribute '{name}'")

camel/loaders/apify_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 import os
 from typing import TYPE_CHECKING, List, Optional
 

camel/loaders/base_io.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 import json
 import re
 from abc import ABC, abstractmethod

camel/loaders/base_loader.py

@@ -0,0 +1,85 @@
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+
+class BaseLoader(ABC):
+    r"""Abstract base class for all data loaders in CAMEL."""
+
+    @abstractmethod
+    def _load_single(self, source: Union[str, Path]) -> Dict[str, Any]:
+        r"""Load data from a single source.
+
+        Args:
+            source (Union[str, Path]): The data source to load from.
+
+        Returns:
+            Dict[str, Any]: A dictionary containing the loaded data. It is
+                recommended that the dictionary includes a "content" key with
+                the primary data and optional metadata keys.
+        """
+        pass
+
+    def load(
+        self,
+        source: Union[str, Path, List[Union[str, Path]]],
+    ) -> Dict[str, List[Dict[str, Any]]]:
+        r"""Load data from one or multiple sources.
+
+        Args:
+            source (Union[str, Path, List[Union[str, Path]]]): The data source
+                (s) to load from. Can be:
+                - A single path/URL (str or Path)
+                - A list of paths/URLs
+
+        Returns:
+            Dict[str, List[Dict[str, Any]]]: A dictionary with a single key
+                "contents" containing a list of loaded data. If a single source
+                is provided, the list will contain a single item.
+
+        Raises:
+            ValueError: If no sources are provided
+            Exception: If loading fails for any source
+        """
+        if not source:
+            raise ValueError("At least one source must be provided")
+
+        # Convert single source to list for uniform processing
+        sources = [source] if isinstance(source, (str, Path)) else list(source)
+
+        # Process all sources
+        results = []
+        for i, src in enumerate(sources, 1):
+            try:
+                content = self._load_single(src)
+                results.append(content)
+            except Exception as e:
+                raise RuntimeError(
+                    f"Error loading source {i}/{len(sources)}: {src}"
+                ) from e
+
+        return {"contents": results}
+
+    @property
+    @abstractmethod
+    def supported_formats(self) -> set[str]:
+        r"""Get the set of supported file formats or data sources.
+
+        Returns:
+            set[str]: A set of strings representing the supported formats/
+            sources.
+        """
+        pass

camel/loaders/chunkr_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,19 +10,49 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
-import logging
 import os
-import time
-from typing import IO, Any, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
-import requests
+if TYPE_CHECKING:
+    from chunkr_ai.models import Configuration
 
+from camel.logger import get_logger
 from camel.utils import api_keys_required
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
+
+
+class ChunkrReaderConfig:
+    r"""Defines the parameters for configuring the task.
+
+    Args:
+        chunk_processing (int, optional): The target chunk length.
+            (default: :obj:`512`)
+        high_resolution (bool, optional): Whether to use high resolution OCR.
+            (default: :obj:`True`)
+        ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+        **kwargs: Additional keyword arguments to pass to the Chunkr
+            Configuration. This accepts all other Configuration parameters
+            such as expires_in, pipeline, segment_processing,
+            segmentation_strategy, etc.
+            See: https://github.com/lumina-ai-inc/chunkr/blob/main/core/src/
+            models/task.rs#L749
+    """
+
+    def __init__(
+        self,
+        chunk_processing: int = 512,
+        high_resolution: bool = True,
+        ocr_strategy: str = "Auto",
+        **kwargs,
+    ):
+        self.chunk_processing = chunk_processing
+        self.high_resolution = high_resolution
+        self.ocr_strategy = ocr_strategy
+        self.kwargs = kwargs
 
 
 class ChunkrReader:
@@ -35,8 +65,6 @@ class ChunkrReader:
             `CHUNKR_API_KEY`. (default: :obj:`None`)
         url (Optional[str], optional): The url to the Chunkr service.
             (default: :obj:`https://api.chunkr.ai/api/v1/task`)
-        timeout (int, optional): The maximum time in seconds to wait for the
-            API responses. (default: :obj:`30`)
         **kwargs (Any): Additional keyword arguments for request headers.
     """
 
@@ -49,111 +77,80 @@ class ChunkrReader:
         self,
         api_key: Optional[str] = None,
         url: Optional[str] = "https://api.chunkr.ai/api/v1/task",
-        timeout: int = 30,
-        **kwargs: Any,
     ) -> None:
+        from chunkr_ai import Chunkr
+
         self._api_key = api_key or os.getenv('CHUNKR_API_KEY')
-        self._url = os.getenv('CHUNKR_API_URL') or url
-        self._headers = {
-            "Authorization": f"{self._api_key}",
-            **kwargs,
-        }
-        self.timeout = timeout
-
-    def submit_task(
+        self._chunkr = Chunkr(api_key=self._api_key)
+
+    async def submit_task(
         self,
         file_path: str,
-        model: str = "Fast",
-        ocr_strategy: str = "Auto",
-        target_chunk_length: str = "512",
+        chunkr_config: Optional[ChunkrReaderConfig] = None,
     ) -> str:
         r"""Submits a file to the Chunkr API and returns the task ID.
 
         Args:
             file_path (str): The path to the file to be uploaded.
-            model (str, optional): The model to be used for the task.
-                (default: :obj:`Fast`)
-            ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
-            target_chunk_length (str, optional): The target chunk length.
-                (default: :obj:`512`)
+            chunkr_config (ChunkrReaderConfig, optional): The configuration
+                for the Chunkr API. Defaults to None.
 
         Returns:
             str: The task ID.
         """
-        with open(file_path, 'rb') as file:
-            files: dict[
-                str, Union[tuple[None, IO[bytes]], tuple[None, str]]
-            ] = {
-                'file': (
-                    None,
-                    file,
-                ),  # Properly pass the file as a binary stream
-                'model': (None, model),
-                'ocr_strategy': (None, ocr_strategy),
-                'target_chunk_length': (None, target_chunk_length),
-            }
-            try:
-                response = requests.post(
-                    self._url,  # type: ignore[arg-type]
-                    headers=self._headers,
-                    files=files,
-                    timeout=self.timeout,
-                )
-                response.raise_for_status()
-                task_id = response.json().get('task_id')
-                if not task_id:
-                    raise ValueError("Task ID not returned in the response.")
-                logger.info(f"Task submitted successfully. Task ID: {task_id}")
-                return task_id
-            except Exception as e:
-                logger.error(f"Failed to submit task: {e}")
-                raise ValueError(f"Failed to submit task: {e}") from e
-
-    def get_task_output(self, task_id: str, max_retries: int = 5) -> str:
+        chunkr_config = self._to_chunkr_configuration(
+            chunkr_config or ChunkrReaderConfig()
+        )
+
+        try:
+            task = await self._chunkr.create_task(
+                file=file_path, config=chunkr_config
+            )
+            logger.info(
+                f"Task submitted successfully. Task ID: {task.task_id}"
+            )
+            return task.task_id
+        except Exception as e:
+            logger.error(f"Failed to submit task: {e}")
+            raise ValueError(f"Failed to submit task: {e}") from e
+
+    async def get_task_output(self, task_id: str) -> str | None:
         r"""Polls the Chunkr API to check the task status and returns the task
         result.
 
         Args:
             task_id (str): The task ID to check the status for.
-            max_retries (int, optional): Maximum number of retry attempts.
-                (default: :obj:`5`)
 
         Returns:
-            str: The formatted task result in JSON format.
-
-        Raises:
-            ValueError: If the task status cannot be retrieved.
-            RuntimeError: If the maximum number of retries is reached without
-                a successful task completion.
+            Optional[str]: The formatted task result in JSON format, or `None`
+                if the task fails or is canceld.
         """
-        url_get = f"{self._url}/{task_id}"
-        attempts = 0
-
-        while attempts < max_retries:
-            try:
-                response = requests.get(
-                    url_get, headers=self._headers, timeout=self.timeout
+        from chunkr_ai.models import Status
+
+        try:
+            task = await self._chunkr.get_task(task_id)
+        except Exception as e:
+            logger.error(f"Failed to get task by task id: {task_id}: {e}")
+            raise ValueError(
+                f"Failed to get task by task id: {task_id}: {e}"
+            ) from e
+
+        try:
+            await task.poll()
+            if task.status == Status.SUCCEEDED:
+                logger.info(f"Task {task_id} completed successfully.")
+                return self._pretty_print_response(task.json())
+            elif task.status == Status.FAILED:
+                logger.warning(
+                    f"Task {task_id} encountered an error: {task.message}"
                 )
-                response.raise_for_status()
-                task_status = response.json().get('status')
-
-                if task_status == "Succeeded":
-                    logger.info(f"Task {task_id} completed successfully.")
-                    return self._pretty_print_response(response.json())
-                else:
-                    logger.info(
-                        f"Task {task_id} is still {task_status}. Retrying "
-                        "in 5 seconds..."
-                    )
-            except Exception as e:
-                logger.error(f"Failed to retrieve task status: {e}")
-                raise ValueError(f"Failed to retrieve task status: {e}") from e
-
-            attempts += 1
-            time.sleep(5)
-
-        logger.error(f"Max retries reached for task {task_id}.")
-        raise RuntimeError(f"Max retries reached for task {task_id}.")
+                return None
+            else:
+                logger.warning(f"Task {task_id} was manually cancelled.")
+                return None
+        except Exception as e:
+            logger.error(f"Failed to retrieve task status: {e}")
+            raise ValueError(f"Failed to retrieve task status: {e}") from e
 
     def _pretty_print_response(self, response_json: dict) -> str:
         r"""Pretty prints the JSON response.
@@ -164,4 +161,42 @@ class ChunkrReader:
         Returns:
             str: Formatted JSON as a string.
         """
-        return json.dumps(response_json, indent=4, ensure_ascii=False)
+        from datetime import datetime
+
+        return json.dumps(
+            response_json,
+            default=lambda o: o.isoformat()
+            if isinstance(o, datetime)
+            else None,
+            indent=4,
+        )
+
+    def _to_chunkr_configuration(
+        self, chunkr_config: ChunkrReaderConfig
+    ) -> "Configuration":
+        r"""Converts the ChunkrReaderConfig to Chunkr Configuration.
+
+        Args:
+            chunkr_config (ChunkrReaderConfig):
+                The ChunkrReaderConfig to convert.
+
+        Returns:
+            Configuration: Chunkr SDK configuration.
+        """
+        from chunkr_ai.models import (
+            ChunkProcessing,
+            Configuration,
+            OcrStrategy,
+        )
+
+        return Configuration(
+            chunk_processing=ChunkProcessing(
+                target_length=chunkr_config.chunk_processing
+            ),
+            high_resolution=chunkr_config.high_resolution,
+            ocr_strategy={
+                "Auto": OcrStrategy.AUTO,
+                "All": OcrStrategy.ALL,
+            }.get(chunkr_config.ocr_strategy, OcrStrategy.ALL),
+            **chunkr_config.kwargs,
+        )

camel/loaders/crawl4ai_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import asyncio
 import logging

camel/loaders/firecrawl_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
 from typing import Any, Dict, Optional
@@ -98,8 +98,8 @@ class Firecrawl:
     def scrape(
         self,
         url: str,
-        params: Optional[Dict[str, Any]] = None,
-    ) -> Dict:
+        params: Optional[Dict[str, str]] = None,
+    ) -> Dict[str, str]:
         r"""To scrape a single URL. This function supports advanced scraping
         by setting different parameters and returns the full scraped data as a
         dictionary.
@@ -108,11 +108,11 @@ class Firecrawl:
 
         Args:
             url (str): The URL to read.
-            params (Optional[Dict[str, Any]]): Additional parameters for the
+            params (Optional[Dict[str, str]]): Additional parameters for the
                 scrape request.
 
         Returns:
-            Dict: The scraped data.
+            Dict[str, str]: The scraped data.
 
         Raises:
             RuntimeError: If the scrape process fails.

camel/loaders/jina_url_reader.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
 from typing import Any, Optional

camel/loaders/markitdown.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 import os
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import ClassVar, Dict, List, Optional

camel/loaders/mineru_extractor.py

@@ -1,4 +1,4 @@
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# ========= Copyright 2023-2025 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import os
 import time