camel-ai 0.2.60__py3-none-any.whl → 0.2.61__py3-none-any.whl

camel/loaders/chunkr_reader.py

@@ -13,16 +13,38 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
-import logging
 import os
-import time
-from typing import IO, Any, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
-import requests
+if TYPE_CHECKING:
+    from chunkr_ai.models import Configuration
 
+from camel.logger import get_logger
 from camel.utils import api_keys_required
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
+
+
+class ChunkrReaderConfig:
+    r"""Defines the parameters for configuring the task.
+
+    Args:
+        chunk_processing (int, optional): The target chunk length.
+            (default: :obj:`512`)
+        high_resolution (bool, optional): Whether to use high resolution OCR.
+            (default: :obj:`True`)
+        ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+    """
+
+    def __init__(
+        self,
+        chunk_processing: int = 512,
+        high_resolution: bool = True,
+        ocr_strategy: str = "Auto",
+    ):
+        self.chunk_processing = chunk_processing
+        self.high_resolution = high_resolution
+        self.ocr_strategy = ocr_strategy
 
 
 class ChunkrReader:
@@ -35,8 +57,6 @@ class ChunkrReader:
             `CHUNKR_API_KEY`. (default: :obj:`None`)
         url (Optional[str], optional): The url to the Chunkr service.
             (default: :obj:`https://api.chunkr.ai/api/v1/task`)
-        timeout (int, optional): The maximum time in seconds to wait for the
-            API responses. (default: :obj:`30`)
         **kwargs (Any): Additional keyword arguments for request headers.
     """
 
@@ -49,111 +69,80 @@ class ChunkrReader:
         self,
         api_key: Optional[str] = None,
         url: Optional[str] = "https://api.chunkr.ai/api/v1/task",
-        timeout: int = 30,
-        **kwargs: Any,
     ) -> None:
+        from chunkr_ai import Chunkr
+
         self._api_key = api_key or os.getenv('CHUNKR_API_KEY')
-        self._url = os.getenv('CHUNKR_API_URL') or url
-        self._headers = {
-            "Authorization": f"{self._api_key}",
-            **kwargs,
-        }
-        self.timeout = timeout
-
-    def submit_task(
+        self._chunkr = Chunkr(api_key=self._api_key)
+
+    async def submit_task(
         self,
         file_path: str,
-        model: str = "Fast",
-        ocr_strategy: str = "Auto",
-        target_chunk_length: str = "512",
+        chunkr_config: Optional[ChunkrReaderConfig] = None,
     ) -> str:
         r"""Submits a file to the Chunkr API and returns the task ID.
 
         Args:
             file_path (str): The path to the file to be uploaded.
-            model (str, optional): The model to be used for the task.
-                (default: :obj:`Fast`)
-            ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
-            target_chunk_length (str, optional): The target chunk length.
-                (default: :obj:`512`)
+            chunkr_config (ChunkrReaderConfig, optional): The configuration
+                for the Chunkr API. Defaults to None.
 
         Returns:
             str: The task ID.
         """
-        with open(file_path, 'rb') as file:
-            files: dict[
-                str, Union[tuple[None, IO[bytes]], tuple[None, str]]
-            ] = {
-                'file': (
-                    None,
-                    file,
-                ),  # Properly pass the file as a binary stream
-                'model': (None, model),
-                'ocr_strategy': (None, ocr_strategy),
-                'target_chunk_length': (None, target_chunk_length),
-            }
-            try:
-                response = requests.post(
-                    self._url,  # type: ignore[arg-type]
-                    headers=self._headers,
-                    files=files,
-                    timeout=self.timeout,
-                )
-                response.raise_for_status()
-                task_id = response.json().get('task_id')
-                if not task_id:
-                    raise ValueError("Task ID not returned in the response.")
-                logger.info(f"Task submitted successfully. Task ID: {task_id}")
-                return task_id
-            except Exception as e:
-                logger.error(f"Failed to submit task: {e}")
-                raise ValueError(f"Failed to submit task: {e}") from e
-
-    def get_task_output(self, task_id: str, max_retries: int = 5) -> str:
+        chunkr_config = self._to_chunkr_configuration(
+            chunkr_config or ChunkrReaderConfig()
+        )
+
+        try:
+            task = await self._chunkr.create_task(
+                file=file_path, config=chunkr_config
+            )
+            logger.info(
+                f"Task submitted successfully. Task ID: {task.task_id}"
+            )
+            return task.task_id
+        except Exception as e:
+            logger.error(f"Failed to submit task: {e}")
+            raise ValueError(f"Failed to submit task: {e}") from e
+
+    async def get_task_output(self, task_id: str) -> str | None:
         r"""Polls the Chunkr API to check the task status and returns the task
         result.
 
         Args:
             task_id (str): The task ID to check the status for.
-            max_retries (int, optional): Maximum number of retry attempts.
-                (default: :obj:`5`)
 
         Returns:
-            str: The formatted task result in JSON format.
-
-        Raises:
-            ValueError: If the task status cannot be retrieved.
-            RuntimeError: If the maximum number of retries is reached without
-                a successful task completion.
+            Optional[str]: The formatted task result in JSON format, or `None`
+                if the task fails or is canceld.
         """
-        url_get = f"{self._url}/{task_id}"
-        attempts = 0
-
-        while attempts < max_retries:
-            try:
-                response = requests.get(
-                    url_get, headers=self._headers, timeout=self.timeout
+        from chunkr_ai.models import Status
+
+        try:
+            task = await self._chunkr.get_task(task_id)
+        except Exception as e:
+            logger.error(f"Failed to get task by task id: {task_id}: {e}")
+            raise ValueError(
+                f"Failed to get task by task id: {task_id}: {e}"
+            ) from e
+
+        try:
+            await task.poll()
+            if task.status == Status.SUCCEEDED:
+                logger.info(f"Task {task_id} completed successfully.")
+                return self._pretty_print_response(task.json())
+            elif task.status == Status.FAILED:
+                logger.warning(
+                    f"Task {task_id} encountered an error: {task.message}"
                 )
-                response.raise_for_status()
-                task_status = response.json().get('status')
-
-                if task_status == "Succeeded":
-                    logger.info(f"Task {task_id} completed successfully.")
-                    return self._pretty_print_response(response.json())
-                else:
-                    logger.info(
-                        f"Task {task_id} is still {task_status}. Retrying "
-                        "in 5 seconds..."
-                    )
-            except Exception as e:
-                logger.error(f"Failed to retrieve task status: {e}")
-                raise ValueError(f"Failed to retrieve task status: {e}") from e
-
-            attempts += 1
-            time.sleep(5)
-
-        logger.error(f"Max retries reached for task {task_id}.")
-        raise RuntimeError(f"Max retries reached for task {task_id}.")
+                return None
+            else:
+                logger.warning(f"Task {task_id} was manually cancelled.")
+                return None
+        except Exception as e:
+            logger.error(f"Failed to retrieve task status: {e}")
+            raise ValueError(f"Failed to retrieve task status: {e}") from e
 
     def _pretty_print_response(self, response_json: dict) -> str:
         r"""Pretty prints the JSON response.
@@ -164,4 +153,41 @@ class ChunkrReader:
         Returns:
             str: Formatted JSON as a string.
         """
-        return json.dumps(response_json, indent=4, ensure_ascii=False)
+        from datetime import datetime
+
+        return json.dumps(
+            response_json,
+            default=lambda o: o.isoformat()
+            if isinstance(o, datetime)
+            else None,
+            indent=4,
+        )
+
+    def _to_chunkr_configuration(
+        self, chunkr_config: ChunkrReaderConfig
+    ) -> "Configuration":
+        r"""Converts the ChunkrReaderConfig to Chunkr Configuration.
+
+        Args:
+            chunkr_config (ChunkrReaderConfig):
+                The ChunkrReaderConfig to convert.
+
+        Returns:
+            Configuration: Chunkr SDK configuration.
+        """
+        from chunkr_ai.models import (
+            ChunkProcessing,
+            Configuration,
+            OcrStrategy,
+        )
+
+        return Configuration(
+            chunk_processing=ChunkProcessing(
+                target_length=chunkr_config.chunk_processing
+            ),
+            high_resolution=chunkr_config.high_resolution,
+            ocr_strategy={
+                "Auto": OcrStrategy.AUTO,
+                "All": OcrStrategy.ALL,
+            }.get(chunkr_config.ocr_strategy, OcrStrategy.ALL),
+        )

camel/loaders/mistral_reader.py

@@ -0,0 +1,148 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    from mistralai.models import OCRResponse
+
+from camel.logger import get_logger
+from camel.utils import api_keys_required
+
+logger = get_logger(__name__)
+
+
+class MistralReader:
+    r"""Mistral Document Loader."""
+
+    @api_keys_required(
+        [
+            ("api_key", "MISTRAL_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: Optional[str] = "mistral-ocr-latest",
+    ) -> None:
+        r"""Initialize the MistralReader.
+
+        Args:
+            api_key (Optional[str]): The API key for the Mistral API.
+                (default: :obj:`None`)
+            model (Optional[str]): The model to use for OCR.
+                (default: :obj:`"mistral-ocr-latest"`)
+        """
+        from mistralai import Mistral
+
+        self._api_key = api_key or os.environ.get("MISTRAL_API_KEY")
+        self.client = Mistral(api_key=self._api_key)
+        self.model = model
+
+    def _encode_file(self, file_path: str) -> str:
+        r"""Encode the pdf to base64.
+
+        Args:
+            file_path (str): Path to the input file.
+
+        Returns:
+            str: base64 version of the file.
+        """
+
+        import base64
+
+        try:
+            with open(file_path, "rb") as pdf_file:
+                return base64.b64encode(pdf_file.read()).decode('utf-8')
+        except FileNotFoundError:
+            logger.error(f"Error: The file {file_path} was not found.")
+            return ""
+        except Exception as e:
+            logger.error(f"Error: {e}")
+            return ""
+
+    def extract_text(
+        self,
+        file_path: str,
+        is_image: bool = False,
+        pages: Optional[List[int]] = None,
+        include_image_base64: Optional[bool] = None,
+    ) -> "OCRResponse":
+        r"""Converts the given file to Markdown format.
+
+        Args:
+            file_path (str): Path to the input file or a remote URL.
+            is_image (bool): Whether the file or URL is an image. If True,
+                uses image_url type instead of document_url.
+                (default: :obj:`False`)
+            pages (Optional[List[int]]): Specific pages user wants to process
+                in various formats: single number, range, or list of both.
+                Starts from 0. (default: :obj:`None`)
+            include_image_base64 (Optional[bool]): Whether to include image
+                URLs in response. (default: :obj:`None`)
+
+        Returns:
+            OCRResponse: page wise extractions.
+
+        Raises:
+            FileNotFoundError: If the specified local file does not exist.
+            ValueError: If the file format is not supported.
+            Exception: For other errors during conversion.
+        """
+        # Check if the input is a URL (starts with http:// or https://)
+        is_url = file_path.startswith(('http://', 'https://'))
+
+        if not is_url and not os.path.isfile(file_path):
+            logger.error(f"File not found: {file_path}")
+            raise FileNotFoundError(f"File not found: {file_path}")
+        try:
+            if is_url:
+                logger.info(f"Processing URL: {file_path}")
+                if is_image:
+                    document_config = {
+                        "type": "image_url",
+                        "image_url": file_path,
+                    }
+                else:
+                    document_config = {
+                        "type": "document_url",
+                        "document_url": file_path,
+                    }
+            else:
+                logger.info(f"Converting local file: {file_path}")
+                base64_file = self._encode_file(file_path)
+                if is_image:
+                    document_config = {
+                        "type": "image_url",
+                        "image_url": f"data:image/jpeg;base64,{base64_file}",
+                    }
+                else:
+                    document_config = {
+                        "type": "document_url",
+                        "document_url": f"data:application/"
+                        f"pdf;base64,{base64_file}",
+                    }
+
+            ocr_response = self.client.ocr.process(
+                model=self.model,
+                document=document_config,  # type: ignore[arg-type]
+                pages=None if is_image else pages,
+                include_image_base64=include_image_base64,
+            )
+
+            logger.info(f"Processing completed successfully for: {file_path}")
+            return ocr_response
+        except Exception as e:
+            logger.error(f"Error processing '{file_path}': {e}")
+            raise ValueError(f"Error processing '{file_path}': {e}")

camel/memories/blocks/chat_history_block.py

@@ -73,7 +73,6 @@ class ChatHistoryBlock(MemoryBlock):
             warnings.warn("The `ChatHistoryMemory` is empty.")
             return list()
 
-        chat_records: List[MemoryRecord] = []
         if window_size is not None and window_size >= 0:
             # Initial preserved index: Keep first message
             # if it's SYSTEM/DEVELOPER (index 0)
@@ -117,7 +116,7 @@ class ChatHistoryBlock(MemoryBlock):
             # Return full records when no window restriction
             final_records = record_dicts
 
-        chat_records = [
+        chat_records: List[MemoryRecord] = [
             MemoryRecord.from_dict(record) for record in final_records
         ]
 

camel/models/model_manager.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
+import asyncio
 import logging
 from itertools import cycle
 from random import choice
@@ -69,6 +70,7 @@ class ModelManager:
             self.models = [models]
         self.models_cycle = cycle(self.models)
         self.current_model = self.models[0]
+        self.lock = asyncio.Lock()
 
         # Set the scheduling strategy; default is round-robin
         try:
@@ -246,7 +248,8 @@ class ModelManager:
                 `ChatCompletion` in the non-stream mode, or
                 `AsyncStream[ChatCompletionChunk]` in the stream mode.
         """
-        self.current_model = self.scheduling_strategy()
+        async with self.lock:
+            self.current_model = self.scheduling_strategy()
 
         # Pass all messages to the selected model and get the response
         try:
@@ -260,7 +263,8 @@ class ModelManager:
                 logger.warning(
                     "The scheduling strategy has been changed to 'round_robin'"
                 )
-                # Skip already used one
-                self.current_model = self.scheduling_strategy()
+                async with self.lock:
+                    # Skip already used one
+                    self.current_model = self.scheduling_strategy()
             raise exc
         return response

camel/societies/workforce/workforce.py

@@ -48,29 +48,58 @@ logger = get_logger(__name__)
 
 
 class Workforce(BaseNode):
-    r"""A system where multiple workder nodes (agents) cooperate together
-    to solve tasks. It can assign tasks to workder nodes and also take
+    r"""A system where multiple worker nodes (agents) cooperate together
+    to solve tasks. It can assign tasks to worker nodes and also take
     strategies such as create new worker, decompose tasks, etc. to handle
     situations when the task fails.
 
+    The workforce uses three specialized ChatAgents internally:
+    - Coordinator Agent: Assigns tasks to workers based on their
+      capabilities
+    - Task Planner Agent: Decomposes complex tasks and composes results
+    - Dynamic Workers: Created at runtime when tasks fail repeatedly
+
     Args:
-        description (str): Description of the node.
+        description (str): Description of the workforce.
         children (Optional[List[BaseNode]], optional): List of child nodes
             under this node. Each child node can be a worker node or
             another workforce node. (default: :obj:`None`)
         coordinator_agent_kwargs (Optional[Dict], optional): Keyword
-            arguments for the coordinator agent, e.g. `model`, `api_key`,
-            `tools`, etc. If not provided, default model settings will be used.
-            (default: :obj:`None`)
-        task_agent_kwargs (Optional[Dict], optional): Keyword arguments for
-            the task agent, e.g. `model`, `api_key`, `tools`, etc.
-            If not provided, default model settings will be used.
-            (default: :obj:`None`)
-        new_worker_agent_kwargs (Optional[Dict]): Default keyword arguments
-            for the worker agent that will be created during runtime to
-            handle failed tasks, e.g. `model`, `api_key`, `tools`, etc.
-            If not provided, default model settings will be used.
-            (default: :obj:`None`)
+            arguments passed directly to the coordinator :obj:`ChatAgent`
+            constructor. The coordinator manages task assignment and failure
+            handling strategies. See :obj:`ChatAgent` documentation
+            for all available parameters.
+            (default: :obj:`None` - uses ModelPlatformType.DEFAULT,
+            ModelType.DEFAULT)
+        task_agent_kwargs (Optional[Dict], optional): Keyword arguments
+            passed directly to the task planning :obj:`ChatAgent` constructor.
+            The task agent handles task decomposition into subtasks and result
+            composition. See :obj:`ChatAgent` documentation for all
+            available parameters.
+            (default: :obj:`None` - uses ModelPlatformType.DEFAULT,
+            ModelType.DEFAULT)
+        new_worker_agent_kwargs (Optional[Dict], optional): Default keyword
+            arguments passed to :obj:`ChatAgent` constructor for workers
+            created dynamically at runtime when existing workers cannot handle
+            failed tasks. See :obj:`ChatAgent` documentation for all
+            available parameters.
+            (default: :obj:`None` - creates workers with SearchToolkit,
+            CodeExecutionToolkit, and ThinkingToolkit)
+
+    Example:
+        >>> # Configure with custom model
+        >>> model = ModelFactory.create(
+        ...     ModelPlatformType.OPENAI, ModelType.GPT_4O
+        ... )
+        >>> workforce = Workforce(
+        ...     "Research Team",
+        ...     coordinator_agent_kwargs={"model": model, "token_limit": 4000},
+        ...     task_agent_kwargs={"model": model, "token_limit": 8000}
+        ... )
+        >>>
+        >>> # Process a task
+        >>> task = Task(content="Research AI trends", id="1")
+        >>> result = workforce.process_task(task)
     """
 
     def __init__(
@@ -89,21 +118,33 @@ class Workforce(BaseNode):
         # Warning messages for default model usage
         if coordinator_agent_kwargs is None:
             logger.warning(
-                "No coordinator_agent_kwargs provided. "
-                "Using `ModelPlatformType.DEFAULT` and `ModelType.DEFAULT` "
-                "for coordinator agent."
+                "No coordinator_agent_kwargs provided. Using default "
+                "ChatAgent settings (ModelPlatformType.DEFAULT, "
+                "ModelType.DEFAULT). To customize the coordinator agent "
+                "that assigns tasks and handles failures, pass a dictionary "
+                "with ChatAgent parameters, e.g.: {'model': your_model, "
+                "'tools': your_tools, 'token_limit': 8000}. See ChatAgent "
+                "documentation for all available options."
             )
         if task_agent_kwargs is None:
             logger.warning(
-                "No task_agent_kwargs provided. "
-                "Using `ModelPlatformType.DEFAULT` and `ModelType.DEFAULT` "
-                "for task agent."
+                "No task_agent_kwargs provided. Using default ChatAgent "
+                "settings (ModelPlatformType.DEFAULT, ModelType.DEFAULT). "
+                "To customize the task planning agent that "
+                "decomposes/composes tasks, pass a dictionary with "
+                "ChatAgent parameters, e.g.: {'model': your_model, "
+                "'token_limit': 16000}. See ChatAgent documentation for "
+                "all available options."
             )
         if new_worker_agent_kwargs is None:
             logger.warning(
-                "No new_worker_agent_kwargs provided. "
-                "Using `ModelPlatformType.DEFAULT` and `ModelType.DEFAULT` "
-                "for worker agents created during runtime."
+                "No new_worker_agent_kwargs provided. Workers created at "
+                "runtime will use default ChatAgent settings with "
+                "SearchToolkit, CodeExecutionToolkit, and ThinkingToolkit. "
+                "To customize runtime worker creation, pass a dictionary "
+                "with ChatAgent parameters, e.g.: {'model': your_model, "
+                "'tools': your_tools}. See ChatAgent documentation for all "
+                "available options."
             )
 
         coord_agent_sys_msg = BaseMessage.make_assistant_message(

camel/storages/__init__.py

@@ -26,6 +26,7 @@ from .vectordb_storages.base import (
     VectorDBQueryResult,
     VectorRecord,
 )
+from .vectordb_storages.faiss import FaissStorage
 from .vectordb_storages.milvus import MilvusStorage
 from .vectordb_storages.oceanbase import OceanBaseStorage
 from .vectordb_storages.qdrant import QdrantStorage
@@ -43,6 +44,7 @@ __all__ = [
     'QdrantStorage',
     'MilvusStorage',
     "TiDBStorage",
+    "FaissStorage",
     'BaseGraphStorage',
     'Neo4jGraph',
     'NebulaGraph',

camel/storages/vectordb_storages/__init__.py

@@ -19,6 +19,7 @@ from .base import (
     VectorDBStatus,
     VectorRecord,
 )
+from .faiss import FaissStorage
 from .milvus import MilvusStorage
 from .oceanbase import OceanBaseStorage
 from .qdrant import QdrantStorage
@@ -31,6 +32,7 @@ __all__ = [
     'QdrantStorage',
     'MilvusStorage',
     "TiDBStorage",
+    'FaissStorage',
     'OceanBaseStorage',
     'VectorRecord',
     'VectorDBStatus',