camel-ai 0.2.60__py3-none-any.whl → 0.2.62__py3-none-any.whl

camel/loaders/chunkr_reader.py

@@ -13,16 +13,38 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
 import json
-import logging
 import os
-import time
-from typing import IO, Any, Optional, Union
+from typing import TYPE_CHECKING, Optional
 
-import requests
+if TYPE_CHECKING:
+    from chunkr_ai.models import Configuration
 
+from camel.logger import get_logger
 from camel.utils import api_keys_required
 
-logger = logging.getLogger(__name__)
+logger = get_logger(__name__)
+
+
+class ChunkrReaderConfig:
+    r"""Defines the parameters for configuring the task.
+
+    Args:
+        chunk_processing (int, optional): The target chunk length.
+            (default: :obj:`512`)
+        high_resolution (bool, optional): Whether to use high resolution OCR.
+            (default: :obj:`True`)
+        ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
+    """
+
+    def __init__(
+        self,
+        chunk_processing: int = 512,
+        high_resolution: bool = True,
+        ocr_strategy: str = "Auto",
+    ):
+        self.chunk_processing = chunk_processing
+        self.high_resolution = high_resolution
+        self.ocr_strategy = ocr_strategy
 
 
 class ChunkrReader:
@@ -35,8 +57,6 @@ class ChunkrReader:
             `CHUNKR_API_KEY`. (default: :obj:`None`)
         url (Optional[str], optional): The url to the Chunkr service.
             (default: :obj:`https://api.chunkr.ai/api/v1/task`)
-        timeout (int, optional): The maximum time in seconds to wait for the
-            API responses. (default: :obj:`30`)
         **kwargs (Any): Additional keyword arguments for request headers.
     """
 
@@ -49,111 +69,80 @@ class ChunkrReader:
         self,
         api_key: Optional[str] = None,
         url: Optional[str] = "https://api.chunkr.ai/api/v1/task",
-        timeout: int = 30,
-        **kwargs: Any,
     ) -> None:
+        from chunkr_ai import Chunkr
+
         self._api_key = api_key or os.getenv('CHUNKR_API_KEY')
-        self._url = os.getenv('CHUNKR_API_URL') or url
-        self._headers = {
-            "Authorization": f"{self._api_key}",
-            **kwargs,
-        }
-        self.timeout = timeout
-
-    def submit_task(
+        self._chunkr = Chunkr(api_key=self._api_key)
+
+    async def submit_task(
         self,
         file_path: str,
-        model: str = "Fast",
-        ocr_strategy: str = "Auto",
-        target_chunk_length: str = "512",
+        chunkr_config: Optional[ChunkrReaderConfig] = None,
     ) -> str:
         r"""Submits a file to the Chunkr API and returns the task ID.
 
         Args:
             file_path (str): The path to the file to be uploaded.
-            model (str, optional): The model to be used for the task.
-                (default: :obj:`Fast`)
-            ocr_strategy (str, optional): The OCR strategy. Defaults to 'Auto'.
-            target_chunk_length (str, optional): The target chunk length.
-                (default: :obj:`512`)
+            chunkr_config (ChunkrReaderConfig, optional): The configuration
+                for the Chunkr API. Defaults to None.
 
         Returns:
             str: The task ID.
         """
-        with open(file_path, 'rb') as file:
-            files: dict[
-                str, Union[tuple[None, IO[bytes]], tuple[None, str]]
-            ] = {
-                'file': (
-                    None,
-                    file,
-                ),  # Properly pass the file as a binary stream
-                'model': (None, model),
-                'ocr_strategy': (None, ocr_strategy),
-                'target_chunk_length': (None, target_chunk_length),
-            }
-            try:
-                response = requests.post(
-                    self._url,  # type: ignore[arg-type]
-                    headers=self._headers,
-                    files=files,
-                    timeout=self.timeout,
-                )
-                response.raise_for_status()
-                task_id = response.json().get('task_id')
-                if not task_id:
-                    raise ValueError("Task ID not returned in the response.")
-                logger.info(f"Task submitted successfully. Task ID: {task_id}")
-                return task_id
-            except Exception as e:
-                logger.error(f"Failed to submit task: {e}")
-                raise ValueError(f"Failed to submit task: {e}") from e
-
-    def get_task_output(self, task_id: str, max_retries: int = 5) -> str:
+        chunkr_config = self._to_chunkr_configuration(
+            chunkr_config or ChunkrReaderConfig()
+        )
+
+        try:
+            task = await self._chunkr.create_task(
+                file=file_path, config=chunkr_config
+            )
+            logger.info(
+                f"Task submitted successfully. Task ID: {task.task_id}"
+            )
+            return task.task_id
+        except Exception as e:
+            logger.error(f"Failed to submit task: {e}")
+            raise ValueError(f"Failed to submit task: {e}") from e
+
+    async def get_task_output(self, task_id: str) -> str | None:
         r"""Polls the Chunkr API to check the task status and returns the task
         result.
 
         Args:
             task_id (str): The task ID to check the status for.
-            max_retries (int, optional): Maximum number of retry attempts.
-                (default: :obj:`5`)
 
         Returns:
-            str: The formatted task result in JSON format.
-
-        Raises:
-            ValueError: If the task status cannot be retrieved.
-            RuntimeError: If the maximum number of retries is reached without
-                a successful task completion.
+            Optional[str]: The formatted task result in JSON format, or `None`
+                if the task fails or is canceld.
         """
-        url_get = f"{self._url}/{task_id}"
-        attempts = 0
-
-        while attempts < max_retries:
-            try:
-                response = requests.get(
-                    url_get, headers=self._headers, timeout=self.timeout
+        from chunkr_ai.models import Status
+
+        try:
+            task = await self._chunkr.get_task(task_id)
+        except Exception as e:
+            logger.error(f"Failed to get task by task id: {task_id}: {e}")
+            raise ValueError(
+                f"Failed to get task by task id: {task_id}: {e}"
+            ) from e
+
+        try:
+            await task.poll()
+            if task.status == Status.SUCCEEDED:
+                logger.info(f"Task {task_id} completed successfully.")
+                return self._pretty_print_response(task.json())
+            elif task.status == Status.FAILED:
+                logger.warning(
+                    f"Task {task_id} encountered an error: {task.message}"
                 )
-                response.raise_for_status()
-                task_status = response.json().get('status')
-
-                if task_status == "Succeeded":
-                    logger.info(f"Task {task_id} completed successfully.")
-                    return self._pretty_print_response(response.json())
-                else:
-                    logger.info(
-                        f"Task {task_id} is still {task_status}. Retrying "
-                        "in 5 seconds..."
-                    )
-            except Exception as e:
-                logger.error(f"Failed to retrieve task status: {e}")
-                raise ValueError(f"Failed to retrieve task status: {e}") from e
-
-            attempts += 1
-            time.sleep(5)
-
-        logger.error(f"Max retries reached for task {task_id}.")
-        raise RuntimeError(f"Max retries reached for task {task_id}.")
+                return None
+            else:
+                logger.warning(f"Task {task_id} was manually cancelled.")
+                return None
+        except Exception as e:
+            logger.error(f"Failed to retrieve task status: {e}")
+            raise ValueError(f"Failed to retrieve task status: {e}") from e
 
     def _pretty_print_response(self, response_json: dict) -> str:
         r"""Pretty prints the JSON response.
@@ -164,4 +153,41 @@ class ChunkrReader:
         Returns:
             str: Formatted JSON as a string.
         """
-        return json.dumps(response_json, indent=4, ensure_ascii=False)
+        from datetime import datetime
+
+        return json.dumps(
+            response_json,
+            default=lambda o: o.isoformat()
+            if isinstance(o, datetime)
+            else None,
+            indent=4,
+        )
+
+    def _to_chunkr_configuration(
+        self, chunkr_config: ChunkrReaderConfig
+    ) -> "Configuration":
+        r"""Converts the ChunkrReaderConfig to Chunkr Configuration.
+
+        Args:
+            chunkr_config (ChunkrReaderConfig):
+                The ChunkrReaderConfig to convert.
+
+        Returns:
+            Configuration: Chunkr SDK configuration.
+        """
+        from chunkr_ai.models import (
+            ChunkProcessing,
+            Configuration,
+            OcrStrategy,
+        )
+
+        return Configuration(
+            chunk_processing=ChunkProcessing(
+                target_length=chunkr_config.chunk_processing
+            ),
+            high_resolution=chunkr_config.high_resolution,
+            ocr_strategy={
+                "Auto": OcrStrategy.AUTO,
+                "All": OcrStrategy.ALL,
+            }.get(chunkr_config.ocr_strategy, OcrStrategy.ALL),
+        )

camel/loaders/mistral_reader.py

@@ -0,0 +1,148 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import os
+from typing import TYPE_CHECKING, List, Optional
+
+if TYPE_CHECKING:
+    from mistralai.models import OCRResponse
+
+from camel.logger import get_logger
+from camel.utils import api_keys_required
+
+logger = get_logger(__name__)
+
+
+class MistralReader:
+    r"""Mistral Document Loader."""
+
+    @api_keys_required(
+        [
+            ("api_key", "MISTRAL_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: Optional[str] = "mistral-ocr-latest",
+    ) -> None:
+        r"""Initialize the MistralReader.
+
+        Args:
+            api_key (Optional[str]): The API key for the Mistral API.
+                (default: :obj:`None`)
+            model (Optional[str]): The model to use for OCR.
+                (default: :obj:`"mistral-ocr-latest"`)
+        """
+        from mistralai import Mistral
+
+        self._api_key = api_key or os.environ.get("MISTRAL_API_KEY")
+        self.client = Mistral(api_key=self._api_key)
+        self.model = model
+
+    def _encode_file(self, file_path: str) -> str:
+        r"""Encode the pdf to base64.
+
+        Args:
+            file_path (str): Path to the input file.
+
+        Returns:
+            str: base64 version of the file.
+        """
+
+        import base64
+
+        try:
+            with open(file_path, "rb") as pdf_file:
+                return base64.b64encode(pdf_file.read()).decode('utf-8')
+        except FileNotFoundError:
+            logger.error(f"Error: The file {file_path} was not found.")
+            return ""
+        except Exception as e:
+            logger.error(f"Error: {e}")
+            return ""
+
+    def extract_text(
+        self,
+        file_path: str,
+        is_image: bool = False,
+        pages: Optional[List[int]] = None,
+        include_image_base64: Optional[bool] = None,
+    ) -> "OCRResponse":
+        r"""Converts the given file to Markdown format.
+
+        Args:
+            file_path (str): Path to the input file or a remote URL.
+            is_image (bool): Whether the file or URL is an image. If True,
+                uses image_url type instead of document_url.
+                (default: :obj:`False`)
+            pages (Optional[List[int]]): Specific pages user wants to process
+                in various formats: single number, range, or list of both.
+                Starts from 0. (default: :obj:`None`)
+            include_image_base64 (Optional[bool]): Whether to include image
+                URLs in response. (default: :obj:`None`)
+
+        Returns:
+            OCRResponse: page wise extractions.
+
+        Raises:
+            FileNotFoundError: If the specified local file does not exist.
+            ValueError: If the file format is not supported.
+            Exception: For other errors during conversion.
+        """
+        # Check if the input is a URL (starts with http:// or https://)
+        is_url = file_path.startswith(('http://', 'https://'))
+
+        if not is_url and not os.path.isfile(file_path):
+            logger.error(f"File not found: {file_path}")
+            raise FileNotFoundError(f"File not found: {file_path}")
+        try:
+            if is_url:
+                logger.info(f"Processing URL: {file_path}")
+                if is_image:
+                    document_config = {
+                        "type": "image_url",
+                        "image_url": file_path,
+                    }
+                else:
+                    document_config = {
+                        "type": "document_url",
+                        "document_url": file_path,
+                    }
+            else:
+                logger.info(f"Converting local file: {file_path}")
+                base64_file = self._encode_file(file_path)
+                if is_image:
+                    document_config = {
+                        "type": "image_url",
+                        "image_url": f"data:image/jpeg;base64,{base64_file}",
+                    }
+                else:
+                    document_config = {
+                        "type": "document_url",
+                        "document_url": f"data:application/"
+                        f"pdf;base64,{base64_file}",
+                    }
+
+            ocr_response = self.client.ocr.process(
+                model=self.model,
+                document=document_config,  # type: ignore[arg-type]
+                pages=None if is_image else pages,
+                include_image_base64=include_image_base64,
+            )
+
+            logger.info(f"Processing completed successfully for: {file_path}")
+            return ocr_response
+        except Exception as e:
+            logger.error(f"Error processing '{file_path}': {e}")
+            raise ValueError(f"Error processing '{file_path}': {e}")

camel/memories/blocks/chat_history_block.py

@@ -73,7 +73,6 @@ class ChatHistoryBlock(MemoryBlock):
             warnings.warn("The `ChatHistoryMemory` is empty.")
             return list()
 
-        chat_records: List[MemoryRecord] = []
         if window_size is not None and window_size >= 0:
             # Initial preserved index: Keep first message
             # if it's SYSTEM/DEVELOPER (index 0)
@@ -117,7 +116,7 @@ class ChatHistoryBlock(MemoryBlock):
             # Return full records when no window restriction
             final_records = record_dicts
 
-        chat_records = [
+        chat_records: List[MemoryRecord] = [
             MemoryRecord.from_dict(record) for record in final_records
         ]
 

camel/models/model_manager.py

@@ -12,6 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
+import asyncio
 import logging
 from itertools import cycle
 from random import choice
@@ -69,6 +70,7 @@ class ModelManager:
             self.models = [models]
         self.models_cycle = cycle(self.models)
         self.current_model = self.models[0]
+        self.lock = asyncio.Lock()
 
         # Set the scheduling strategy; default is round-robin
         try:
@@ -246,7 +248,8 @@ class ModelManager:
                 `ChatCompletion` in the non-stream mode, or
                 `AsyncStream[ChatCompletionChunk]` in the stream mode.
         """
-        self.current_model = self.scheduling_strategy()
+        async with self.lock:
+            self.current_model = self.scheduling_strategy()
 
         # Pass all messages to the selected model and get the response
         try:
@@ -260,7 +263,8 @@ class ModelManager:
                 logger.warning(
                     "The scheduling strategy has been changed to 'round_robin'"
                 )
-                # Skip already used one
-                self.current_model = self.scheduling_strategy()
+                async with self.lock:
+                    # Skip already used one
+                    self.current_model = self.scheduling_strategy()
             raise exc
         return response

camel/retrievers/auto_retriever.py

@@ -128,12 +128,31 @@ class AutoRetriever:
         Returns:
             str: A sanitized, valid collection name suitable for use.
         """
+        import hashlib
+        import os
+
         from unstructured.documents.elements import Element
 
         if isinstance(content, Element):
             content = content.metadata.file_directory or str(uuid.uuid4())
 
-        collection_name = re.sub(r'[^a-zA-Z0-9]', '', content)[:20]
+        # For file paths, use a combination of directory hash and filename
+        if os.path.isfile(content):
+            # Get directory and filename
+            directory = os.path.dirname(content)
+            filename = os.path.basename(content)
+            # Create a short hash of the directory path
+            dir_hash = hashlib.md5(directory.encode()).hexdigest()[:6]
+            # Get filename without extension and remove special chars
+            base_name = os.path.splitext(filename)[0]
+            clean_name = re.sub(r'[^a-zA-Z0-9]', '', base_name)[:10]
+            # Combine for a unique name
+            collection_name = f"{clean_name}_{dir_hash}"
+        else:
+            # For URL content
+            content_hash = hashlib.md5(content.encode()).hexdigest()[:6]
+            clean_content = re.sub(r'[^a-zA-Z0-9]', '', content)[-10:]
+            collection_name = f"{clean_content}_{content_hash}"
 
         # Ensure the first character is either an underscore or a letter for
         # Milvus

camel/{runtime → runtimes}/daytona_runtime.py

@@ -21,7 +21,7 @@ from typing import Any, Dict, List, Optional, Union
 from pydantic import BaseModel
 
 from camel.logger import get_logger
-from camel.runtime import BaseRuntime
+from camel.runtimes import BaseRuntime
 from camel.toolkits.function_tool import FunctionTool
 
 logger = get_logger(__name__)

camel/{runtime → runtimes}/docker_runtime.py

@@ -26,7 +26,7 @@ import requests
 from pydantic import BaseModel
 from tqdm import tqdm
 
-from camel.runtime import BaseRuntime, TaskConfig
+from camel.runtimes import BaseRuntime, TaskConfig
 from camel.toolkits import FunctionTool
 
 if TYPE_CHECKING:

camel/{runtime → runtimes}/llm_guard_runtime.py

@@ -19,8 +19,8 @@ from typing import List, Optional, Union
 from camel.agents import ChatAgent
 from camel.configs import ChatGPTConfig
 from camel.models import BaseModelBackend, ModelFactory
-from camel.runtime import BaseRuntime
-from camel.runtime.utils import FunctionRiskToolkit, IgnoreRiskToolkit
+from camel.runtimes import BaseRuntime
+from camel.runtimes.utils import FunctionRiskToolkit, IgnoreRiskToolkit
 from camel.toolkits import FunctionTool
 from camel.types import ModelPlatformType, ModelType
 

camel/{runtime → runtimes}/remote_http_runtime.py

@@ -24,7 +24,7 @@ from typing import Any, Dict, List, Optional, Union
 import requests
 from pydantic import BaseModel
 
-from camel.runtime import BaseRuntime
+from camel.runtimes import BaseRuntime
 from camel.toolkits.function_tool import FunctionTool
 
 logger = logging.getLogger(__name__)

camel/{runtime → runtimes}/ubuntu_docker_runtime.py

@@ -18,7 +18,7 @@ import time
 from pathlib import Path
 from typing import Callable, List, Optional, Union
 
-from camel.runtime.docker_runtime import DockerRuntime
+from camel.runtimes.docker_runtime import DockerRuntime
 from camel.toolkits import FunctionTool
 
 logger = logging.getLogger(__name__)

camel/societies/workforce/base.py

@@ -12,7 +12,7 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 from abc import ABC, abstractmethod
-from typing import Any
+from typing import Any, Optional
 
 from camel.societies.workforce.task_channel import TaskChannel
 from camel.societies.workforce.utils import check_if_running
@@ -23,10 +23,14 @@ class BaseNode(ABC):
 
     Args:
         description (str): Description of the node.
+        node_id (Optional[str]): ID of the node. If not provided, it will
+            be generated automatically. (default: :obj:`None`)
     """
 
-    def __init__(self, description: str) -> None:
-        self.node_id = str(id(self))
+    def __init__(
+        self, description: str, node_id: Optional[str] = None
+    ) -> None:
+        self.node_id = node_id if node_id is not None else str(id(self))
         self.description = description
         self._channel: TaskChannel = TaskChannel()
         self._running = False

camel/societies/workforce/single_agent_worker.py

@@ -39,7 +39,8 @@ class SingleAgentWorker(Worker):
         description: str,
         worker: ChatAgent,
     ) -> None:
-        super().__init__(description)
+        node_id = worker.agent_id
+        super().__init__(description, node_id=node_id)
         self.worker = worker
 
     def reset(self) -> Any:

camel/societies/workforce/worker.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 import logging
 from abc import ABC, abstractmethod
-from typing import List
+from typing import List, Optional
 
 from colorama import Fore
 
@@ -33,14 +33,16 @@ class Worker(BaseNode, ABC):
 
     Args:
         description (str): Description of the node.
-
+        node_id (Optional[str]): ID of the node. If not provided, it will
+            be generated automatically. (default: :obj:`None`)
     """
 
     def __init__(
         self,
         description: str,
+        node_id: Optional[str] = None,
     ) -> None:
-        super().__init__(description)
+        super().__init__(description, node_id=node_id)
 
     def __repr__(self):
         return f"Worker node {self.node_id} ({self.description})"