camel-ai 0.2.59__py3-none-any.whl → 0.2.61__py3-none-any.whl

camel/datasets/static_dataset.py

@@ -398,3 +398,147 @@ class StaticDataset(Dataset):
                     f"got {type(item).__name__}"
                 )
         return data
+
+    def save_to_json(self, file_path: Union[str, Path]) -> None:
+        r"""Save the dataset to a local JSON file.
+
+        Args:
+            file_path (Union[str, Path]): Path to the output JSON file.
+                If a string is provided, it will be converted to a Path object.
+
+        Raises:
+            TypeError: If file_path is not a string or Path object.
+            OSError: If there's an error writing to the file.
+        """
+        if isinstance(file_path, str):
+            file_path = Path(file_path)
+        elif not isinstance(file_path, Path):
+            raise TypeError(
+                f"Expected file_path to be a string or Path object, "
+                f"got {type(file_path).__name__}"
+            )
+
+        # Convert DataPoint objects to dictionaries
+        data_dicts = [datapoint.to_dict() for datapoint in self.data]
+
+        # Ensure the parent directory exists
+        file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        try:
+            logger.debug(f"Saving dataset to {file_path}")
+            with file_path.open('w', encoding='utf-8') as f:
+                json.dump(data_dicts, f, ensure_ascii=False, indent=2)
+            logger.info(
+                f"Successfully saved {len(data_dicts)} items to {file_path}"
+            )
+        except OSError as e:
+            logger.error(f"Error saving dataset to {file_path}: {e}")
+            raise
+
+    def save_to_huggingface(
+        self,
+        dataset_name: str,
+        token: Optional[str] = None,
+        filepath: str = "records/records.json",
+        private: bool = False,
+        description: Optional[str] = None,
+        license: Optional[str] = None,
+        version: Optional[str] = None,
+        tags: Optional[List[str]] = None,
+        language: Optional[List[str]] = None,
+        task_categories: Optional[List[str]] = None,
+        authors: Optional[List[str]] = None,
+        **kwargs: Any,
+    ) -> str:
+        r"""Save the dataset to the Hugging Face Hub using the project's
+        HuggingFaceDatasetManager.
+
+        Args:
+            dataset_name (str): The name of the dataset on Hugging Face Hub.
+                Should be in the format 'username/dataset_name' .
+            token (Optional[str]): The Hugging Face API token. If not provided,
+                the token will be read from the environment variable `HF_TOKEN`
+                (default: :obj:`None`)
+            filepath (str): The path in the repository where the dataset
+                will be saved.  (default: :obj:`"records/records.json"`)
+            private (bool): Whether the dataset should be private.
+                (default: :obj:`False`)
+            description (Optional[str]): A description of the dataset.
+                (default: :obj:`None`)
+            license (Optional[str]): The license of the dataset.
+                (default: :obj:`None`)
+            version (Optional[str]): The version of the dataset.
+                (default: :obj:`None`)
+            tags (Optional[List[str]]): A list of tags for the dataset.
+                (default: :obj:`None`)
+            language (Optional[List[str]]): A list of languages the dataset is
+                in. (default: :obj:`None`)
+            task_categories (Optional[List[str]]): A list of task categories.
+                (default: :obj:`None`)
+            authors (Optional[List[str]]): A list of authors of the dataset.
+                (default: :obj:`None`)
+            **kwargs (Any): Additional keyword arguments to pass to the
+                Hugging Face API.
+
+        Returns:
+            str: The URL of the dataset on the Hugging Face Hub.
+
+        Raises:
+            OSError: If there's an error uploading the dataset.
+        """
+        # lazy import to avoid heavy dependencies
+        from camel.datahubs.huggingface import HuggingFaceDatasetManager
+        from camel.datahubs.models import Record
+
+        # Initialize the HuggingFaceDatasetManager
+        manager = HuggingFaceDatasetManager(token=token)
+
+        # Convert DataPoint objects to Record objects
+        records = []
+        for datapoint in self.data:
+            datapoint_dict = datapoint.to_dict()
+
+            record_dict = {
+                "question": datapoint_dict.get("question", ""),
+                "final_answer": datapoint_dict.get("final_answer", ""),
+                "rationale": datapoint_dict.get("rationale", ""),
+                "metadata": datapoint_dict.get("metadata", {}),
+            }
+            record = Record(**record_dict)
+            records.append(record)
+
+        logger.debug(f"Creating dataset {dataset_name}")
+        try:
+            # Create the dataset
+            dataset_url = manager.create_dataset(
+                name=dataset_name, private=private, **kwargs
+            )
+
+            # Add records to the dataset
+            manager.add_records(
+                dataset_name=dataset_name,
+                records=records,
+                filepath=filepath,
+            )
+
+            # Create dataset card if description is provided
+            if description:
+                manager.create_dataset_card(
+                    dataset_name=dataset_name,
+                    description=description,
+                    license=license,
+                    version=version,
+                    tags=tags,
+                    authors=authors,
+                    language=language,
+                    task_categories=task_categories,
+                )
+
+            logger.info(
+                f"Successfully uploaded dataset to {dataset_name}, "
+                f"the url is {dataset_url}"
+            )
+            return dataset_url
+        except Exception as e:
+            logger.error(f"Error uploading dataset to Hugging Face: {e}")
+            raise

camel/embeddings/jina_embedding.py

@@ -12,6 +12,9 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
+# Enables postponed evaluation of annotations (for string-based type hints)
+from __future__ import annotations
+
 import base64
 import io
 import os
@@ -104,6 +107,7 @@ class JinaEmbedding(BaseEmbedding[Union[str, Image.Image]]):
             ValueError: If the input type is not supported.
             RuntimeError: If the API request fails.
         """
+
         input_data = []
         for obj in objs:
             if isinstance(obj, str):
@@ -111,7 +115,10 @@ class JinaEmbedding(BaseEmbedding[Union[str, Image.Image]]):
                     input_data.append({"text": obj})
                 else:
                     input_data.append(obj)  # type: ignore[arg-type]
-            elif isinstance(obj, Image.Image):
+            elif (
+                obj.__class__.__module__ == "PIL.Image"
+                and obj.__class__.__name__ == "Image"
+            ):
                 if self.model_type != EmbeddingModelType.JINA_CLIP_V2:
                     raise ValueError(
                         f"Model {self.model_type} does not support "

camel/embeddings/sentence_transformers_embeddings.py

@@ -15,8 +15,6 @@ from __future__ import annotations
 
 from typing import Any
 
-from numpy import ndarray
-
 from camel.embeddings.base import BaseEmbedding
 
 
@@ -61,6 +59,8 @@ class SentenceTransformerEncoder(BaseEmbedding[str]):
             list[list[float]]: A list that represents the generated embedding
                 as a list of floating-point numbers.
         """
+        from numpy import ndarray
+
         if not objs:
             raise ValueError("Input text list is empty")
         embeddings = self.model.encode(

camel/embeddings/vlm_embedding.py

@@ -11,6 +11,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+# Enables postponed evaluation of annotations (for string-based type hints)
+from __future__ import annotations
+
 from typing import Any, List, Optional, Union
 
 from PIL import Image
@@ -70,7 +74,7 @@ class VisionLanguageEmbedding(BaseEmbedding[Union[str, Image.Image]]):
     def embed_list(
         self, objs: List[Union[Image.Image, str]], **kwargs: Any
     ) -> List[List[float]]:
-        """Generates embeddings for the given images or texts.
+        r"""Generates embeddings for the given images or texts.
 
         Args:
             objs (List[Image.Image|str]): The list of images or texts for
@@ -98,7 +102,10 @@ class VisionLanguageEmbedding(BaseEmbedding[Union[str, Image.Image]]):
 
         result_list = []
         for obj in objs:
-            if isinstance(obj, Image.Image):
+            if (
+                obj.__class__.__module__ == "PIL.Image"
+                and obj.__class__.__name__ == "Image"
+            ):
                 image_input = self.processor(
                     images=obj,
                     return_tensors="pt",

camel/loaders/__init__.py

@@ -14,12 +14,13 @@
 
 from .apify_reader import Apify
 from .base_io import File, create_file, create_file_from_raw_bytes
-from .chunkr_reader import ChunkrReader
+from .chunkr_reader import ChunkrReader, ChunkrReaderConfig
 from .crawl4ai_reader import Crawl4AI
 from .firecrawl_reader import Firecrawl
 from .jina_url_reader import JinaURLReader
 from .markitdown import MarkItDownLoader
 from .mineru_extractor import MinerU
+from .mistral_reader import MistralReader
 from .pandas_reader import PandasReader
 from .scrapegraph_reader import ScrapeGraphAI
 from .unstructured_io import UnstructuredIO
@@ -32,10 +33,12 @@ __all__ = [
     'JinaURLReader',
     'Firecrawl',
     'Apify',
-    'ChunkrReader',
     'PandasReader',
+    'ChunkrReader',
+    'ChunkrReaderConfig',
     'MinerU',
     'Crawl4AI',
     'MarkItDownLoader',
     'ScrapeGraphAI',
+    'MistralReader',
 ]

camel/loaders/chunkr_reader.py

@@ -13,16 +13,38 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
-import logging
 import os
-import time
-from typing import IO, Any, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
-import requests
+if TYPE_CHECKING:
+    from chunkr_ai.models import Configuration
 
+from camel.logger import get_logger
 from camel.utils import api_keys_required
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
+
+
+class ChunkrReaderConfig:
+    r"""Defines the parameters for configuring the task.
+
+    Args:
+        chunk_processing (int, optional): The target chunk length.
+            (default: :obj:`512`)
+        high_resolution (bool, optional): Whether to use high resolution OCR.
+            (default: :obj:`True`)
+        ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+    """
+
+    def __init__(
+        self,
+        chunk_processing: int = 512,
+        high_resolution: bool = True,
+        ocr_strategy: str = "Auto",
+    ):
+        self.chunk_processing = chunk_processing
+        self.high_resolution = high_resolution
+        self.ocr_strategy = ocr_strategy
 
 
 class ChunkrReader:
@@ -35,8 +57,6 @@ class ChunkrReader:
             `CHUNKR_API_KEY`. (default: :obj:`None`)
         url (Optional[str], optional): The url to the Chunkr service.
             (default: :obj:`https://api.chunkr.ai/api/v1/task`)
-        timeout (int, optional): The maximum time in seconds to wait for the
-            API responses. (default: :obj:`30`)
         **kwargs (Any): Additional keyword arguments for request headers.
     """
 
@@ -49,111 +69,80 @@ class ChunkrReader:
         self,
         api_key: Optional[str] = None,
         url: Optional[str] = "https://api.chunkr.ai/api/v1/task",
-        timeout: int = 30,
-        **kwargs: Any,
     ) -> None:
+        from chunkr_ai import Chunkr
+
         self._api_key = api_key or os.getenv('CHUNKR_API_KEY')
-        self._url = os.getenv('CHUNKR_API_URL') or url
-        self._headers = {
-            "Authorization": f"{self._api_key}",
-            **kwargs,
-        }
-        self.timeout = timeout
-
-    def submit_task(
+        self._chunkr = Chunkr(api_key=self._api_key)
+
+    async def submit_task(
         self,
         file_path: str,
-        model: str = "Fast",
-        ocr_strategy: str = "Auto",
-        target_chunk_length: str = "512",
+        chunkr_config: Optional[ChunkrReaderConfig] = None,
     ) -> str:
         r"""Submits a file to the Chunkr API and returns the task ID.
 
         Args:
             file_path (str): The path to the file to be uploaded.
-            model (str, optional): The model to be used for the task.
-                (default: :obj:`Fast`)
-            ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
-            target_chunk_length (str, optional): The target chunk length.
-                (default: :obj:`512`)
+            chunkr_config (ChunkrReaderConfig, optional): The configuration
+                for the Chunkr API. Defaults to None.
 
         Returns:
             str: The task ID.
         """
-        with open(file_path, 'rb') as file:
-            files: dict[
-                str, Union[tuple[None, IO[bytes]], tuple[None, str]]
-            ] = {
-                'file': (
-                    None,
-                    file,
-                ),  # Properly pass the file as a binary stream
-                'model': (None, model),
-                'ocr_strategy': (None, ocr_strategy),
-                'target_chunk_length': (None, target_chunk_length),
-            }
-            try:
-                response = requests.post(
-                    self._url,  # type: ignore[arg-type]
-                    headers=self._headers,
-                    files=files,
-                    timeout=self.timeout,
-                )
-                response.raise_for_status()
-                task_id = response.json().get('task_id')
-                if not task_id:
-                    raise ValueError("Task ID not returned in the response.")
-                logger.info(f"Task submitted successfully. Task ID: {task_id}")
-                return task_id
-            except Exception as e:
-                logger.error(f"Failed to submit task: {e}")
-                raise ValueError(f"Failed to submit task: {e}") from e
-
-    def get_task_output(self, task_id: str, max_retries: int = 5) -> str:
+        chunkr_config = self._to_chunkr_configuration(
+            chunkr_config or ChunkrReaderConfig()
+        )
+
+        try:
+            task = await self._chunkr.create_task(
+                file=file_path, config=chunkr_config
+            )
+            logger.info(
+                f"Task submitted successfully. Task ID: {task.task_id}"
+            )
+            return task.task_id
+        except Exception as e:
+            logger.error(f"Failed to submit task: {e}")
+            raise ValueError(f"Failed to submit task: {e}") from e
+
+    async def get_task_output(self, task_id: str) -> str | None:
         r"""Polls the Chunkr API to check the task status and returns the task
         result.
 
         Args:
             task_id (str): The task ID to check the status for.
-            max_retries (int, optional): Maximum number of retry attempts.
-                (default: :obj:`5`)
 
         Returns:
-            str: The formatted task result in JSON format.
-
-        Raises:
-            ValueError: If the task status cannot be retrieved.
-            RuntimeError: If the maximum number of retries is reached without
-                a successful task completion.
+            Optional[str]: The formatted task result in JSON format, or `None`
+                if the task fails or is canceld.
         """
-        url_get = f"{self._url}/{task_id}"
-        attempts = 0
-
-        while attempts < max_retries:
-            try:
-                response = requests.get(
-                    url_get, headers=self._headers, timeout=self.timeout
+        from chunkr_ai.models import Status
+
+        try:
+            task = await self._chunkr.get_task(task_id)
+        except Exception as e:
+            logger.error(f"Failed to get task by task id: {task_id}: {e}")
+            raise ValueError(
+                f"Failed to get task by task id: {task_id}: {e}"
+            ) from e
+
+        try:
+            await task.poll()
+            if task.status == Status.SUCCEEDED:
+                logger.info(f"Task {task_id} completed successfully.")
+                return self._pretty_print_response(task.json())
+            elif task.status == Status.FAILED:
+                logger.warning(
+                    f"Task {task_id} encountered an error: {task.message}"
                 )
-                response.raise_for_status()
-                task_status = response.json().get('status')
-
-                if task_status == "Succeeded":
-                    logger.info(f"Task {task_id} completed successfully.")
-                    return self._pretty_print_response(response.json())
-                else:
-                    logger.info(
-                        f"Task {task_id} is still {task_status}. Retrying "
-                        "in 5 seconds..."
-                    )
-            except Exception as e:
-                logger.error(f"Failed to retrieve task status: {e}")
-                raise ValueError(f"Failed to retrieve task status: {e}") from e
-
-            attempts += 1
-            time.sleep(5)
-
-        logger.error(f"Max retries reached for task {task_id}.")
-        raise RuntimeError(f"Max retries reached for task {task_id}.")
+                return None
+            else:
+                logger.warning(f"Task {task_id} was manually cancelled.")
+                return None
+        except Exception as e:
+            logger.error(f"Failed to retrieve task status: {e}")
+            raise ValueError(f"Failed to retrieve task status: {e}") from e
 
     def _pretty_print_response(self, response_json: dict) -> str:
         r"""Pretty prints the JSON response.
@@ -164,4 +153,41 @@ class ChunkrReader:
         Returns:
             str: Formatted JSON as a string.
         """
-        return json.dumps(response_json, indent=4, ensure_ascii=False)
+        from datetime import datetime
+
+        return json.dumps(
+            response_json,
+            default=lambda o: o.isoformat()
+            if isinstance(o, datetime)
+            else None,
+            indent=4,
+        )
+
+    def _to_chunkr_configuration(
+        self, chunkr_config: ChunkrReaderConfig
+    ) -> "Configuration":
+        r"""Converts the ChunkrReaderConfig to Chunkr Configuration.
+
+        Args:
+            chunkr_config (ChunkrReaderConfig):
+                The ChunkrReaderConfig to convert.
+
+        Returns:
+            Configuration: Chunkr SDK configuration.
+        """
+        from chunkr_ai.models import (
+            ChunkProcessing,
+            Configuration,
+            OcrStrategy,
+        )
+
+        return Configuration(
+            chunk_processing=ChunkProcessing(
+                target_length=chunkr_config.chunk_processing
+            ),
+            high_resolution=chunkr_config.high_resolution,
+            ocr_strategy={
+                "Auto": OcrStrategy.AUTO,
+                "All": OcrStrategy.ALL,
+            }.get(chunkr_config.ocr_strategy, OcrStrategy.ALL),
+        )

camel/loaders/mistral_reader.py

@@ -0,0 +1,148 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    from mistralai.models import OCRResponse
+
+from camel.logger import get_logger
+from camel.utils import api_keys_required
+
+logger = get_logger(__name__)
+
+
+class MistralReader:
+    r"""Mistral Document Loader."""
+
+    @api_keys_required(
+        [
+            ("api_key", "MISTRAL_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: Optional[str] = "mistral-ocr-latest",
+    ) -> None:
+        r"""Initialize the MistralReader.
+
+        Args:
+            api_key (Optional[str]): The API key for the Mistral API.
+                (default: :obj:`None`)
+            model (Optional[str]): The model to use for OCR.
+                (default: :obj:`"mistral-ocr-latest"`)
+        """
+        from mistralai import Mistral
+
+        self._api_key = api_key or os.environ.get("MISTRAL_API_KEY")
+        self.client = Mistral(api_key=self._api_key)
+        self.model = model
+
+    def _encode_file(self, file_path: str) -> str:
+        r"""Encode the pdf to base64.
+
+        Args:
+            file_path (str): Path to the input file.
+
+        Returns:
+            str: base64 version of the file.
+        """
+
+        import base64
+
+        try:
+            with open(file_path, "rb") as pdf_file:
+                return base64.b64encode(pdf_file.read()).decode('utf-8')
+        except FileNotFoundError:
+            logger.error(f"Error: The file {file_path} was not found.")
+            return ""
+        except Exception as e:
+            logger.error(f"Error: {e}")
+            return ""
+
+    def extract_text(
+        self,
+        file_path: str,
+        is_image: bool = False,
+        pages: Optional[List[int]] = None,
+        include_image_base64: Optional[bool] = None,
+    ) -> "OCRResponse":
+        r"""Converts the given file to Markdown format.
+
+        Args:
+            file_path (str): Path to the input file or a remote URL.
+            is_image (bool): Whether the file or URL is an image. If True,
+                uses image_url type instead of document_url.
+                (default: :obj:`False`)
+            pages (Optional[List[int]]): Specific pages user wants to process
+                in various formats: single number, range, or list of both.
+                Starts from 0. (default: :obj:`None`)
+            include_image_base64 (Optional[bool]): Whether to include image
+                URLs in response. (default: :obj:`None`)
+
+        Returns:
+            OCRResponse: page wise extractions.
+
+        Raises:
+            FileNotFoundError: If the specified local file does not exist.
+            ValueError: If the file format is not supported.
+            Exception: For other errors during conversion.
+        """
+        # Check if the input is a URL (starts with http:// or https://)
+        is_url = file_path.startswith(('http://', 'https://'))
+
+        if not is_url and not os.path.isfile(file_path):
+            logger.error(f"File not found: {file_path}")
+            raise FileNotFoundError(f"File not found: {file_path}")
+        try:
+            if is_url:
+                logger.info(f"Processing URL: {file_path}")
+                if is_image:
+                    document_config = {
+                        "type": "image_url",
+                        "image_url": file_path,
+                    }
+                else:
+                    document_config = {
+                        "type": "document_url",
+                        "document_url": file_path,
+                    }
+            else:
+                logger.info(f"Converting local file: {file_path}")
+                base64_file = self._encode_file(file_path)
+                if is_image:
+                    document_config = {
+                        "type": "image_url",
+                        "image_url": f"data:image/jpeg;base64,{base64_file}",
+                    }
+                else:
+                    document_config = {
+                        "type": "document_url",
+                        "document_url": f"data:application/"
+                        f"pdf;base64,{base64_file}",
+                    }
+
+            ocr_response = self.client.ocr.process(
+                model=self.model,
+                document=document_config,  # type: ignore[arg-type]
+                pages=None if is_image else pages,
+                include_image_base64=include_image_base64,
+            )
+
+            logger.info(f"Processing completed successfully for: {file_path}")
+            return ocr_response
+        except Exception as e:
+            logger.error(f"Error processing '{file_path}': {e}")
+            raise ValueError(f"Error processing '{file_path}': {e}")

camel/memories/blocks/chat_history_block.py

@@ -73,7 +73,6 @@ class ChatHistoryBlock(MemoryBlock):
             warnings.warn("The `ChatHistoryMemory` is empty.")
             return list()
 
-        chat_records: List[MemoryRecord] = []
         if window_size is not None and window_size >= 0:
             # Initial preserved index: Keep first message
             # if it's SYSTEM/DEVELOPER (index 0)
@@ -117,7 +116,7 @@ class ChatHistoryBlock(MemoryBlock):
             # Return full records when no window restriction
             final_records = record_dicts
 
-        chat_records = [
+        chat_records: List[MemoryRecord] = [
             MemoryRecord.from_dict(record) for record in final_records
         ]