camel-ai 0.2.20a1__py3-none-any.whl → 0.2.22__py3-none-any.whl

camel/models/base_model.py

@@ -11,6 +11,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+import abc
+import re
 from abc import ABC, abstractmethod
 from typing import Any, Dict, List, Optional, Union
 
@@ -27,7 +29,30 @@ from camel.types import (
 from camel.utils import BaseTokenCounter
 
 
-class BaseModelBackend(ABC):
+class ModelBackendMeta(abc.ABCMeta):
+    r"""Metaclass that automatically preprocesses messages in run method.
+
+    Automatically wraps the run method of any class inheriting from
+    BaseModelBackend to preprocess messages (remove <think> tags) before they
+    are sent to the model.
+    """
+
+    def __new__(mcs, name, bases, namespace):
+        r"""Wraps run method with preprocessing if it exists in the class."""
+        if 'run' in namespace:
+            original_run = namespace['run']
+
+            def wrapped_run(
+                self, messages: List[OpenAIMessage], *args, **kwargs
+            ):
+                messages = self.preprocess_messages(messages)
+                return original_run(self, messages, *args, **kwargs)
+
+            namespace['run'] = wrapped_run
+        return super().__new__(mcs, name, bases, namespace)
+
+
+class BaseModelBackend(ABC, metaclass=ModelBackendMeta):
     r"""Base class for different model backends.
     It may be OpenAI API, a local LLM, a stub for unit tests, etc.
 
@@ -73,6 +98,34 @@ class BaseModelBackend(ABC):
         """
         pass
 
+    def preprocess_messages(
+        self, messages: List[OpenAIMessage]
+    ) -> List[OpenAIMessage]:
+        r"""Preprocess messages before sending to model API.
+        Removes thinking content and other model-specific preprocessing.
+
+        Args:
+            messages (List[OpenAIMessage]): Original messages
+
+        Returns:
+            List[OpenAIMessage]: Preprocessed messages
+        """
+        # Remove thinking content from messages before sending to API
+        # This ensures only the final response is sent, excluding
+        # intermediate thought processes
+        return [
+            {  # type: ignore[misc]
+                **msg,
+                'content': re.sub(
+                    r'<think>.*?</think>',
+                    '',
+                    msg['content'],  # type: ignore[arg-type]
+                    flags=re.DOTALL,
+                ).strip(),
+            }
+            for msg in messages
+        ]
+
     @abstractmethod
     def run(
         self,

camel/models/deepseek_model.py

@@ -118,8 +118,6 @@ class DeepSeekModel(BaseModelBackend):
         if self.model_type in [
             ModelType.DEEPSEEK_REASONER,
         ]:
-            import re
-
             logger.warning(
                 "You are using a DeepSeek Reasoner model, "
                 "which has certain limitations, reference: "
@@ -141,22 +139,6 @@ class DeepSeekModel(BaseModelBackend):
                 if key in self.model_config_dict:
                     del self.model_config_dict[key]
 
-            # Remove thinking content from messages before sending to API
-            # This ensures only the final response is sent, excluding
-            # intermediate thought processes
-            messages = [
-                {  # type: ignore[misc]
-                    **msg,
-                    'content': re.sub(
-                        r'<think>.*?</think>',
-                        '',
-                        msg['content'],  # type: ignore[arg-type]
-                        flags=re.DOTALL,
-                    ).strip(),
-                }
-                for msg in messages
-            ]
-
         response = self._client.chat.completions.create(
             messages=messages,
             model=self.model_type,

camel/models/model_factory.py

@@ -13,6 +13,7 @@
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 from typing import Dict, Optional, Type, Union
 
+from camel.models.aiml_model import AIMLModel
 from camel.models.anthropic_model import AnthropicModel
 from camel.models.azure_openai_model import AzureOpenAIModel
 from camel.models.base_model import BaseModelBackend
@@ -104,6 +105,8 @@ class ModelFactory:
             model_class = NvidiaModel
         elif model_platform.is_siliconflow:
             model_class = SiliconFlowModel
+        elif model_platform.is_aiml:
+            model_class = AIMLModel
 
         elif model_platform.is_openai and model_type.is_openai:
             model_class = OpenAIModel

camel/models/siliconflow_model.py

@@ -18,7 +18,7 @@ from openai import OpenAI, Stream
 
 from camel.configs import SILICONFLOW_API_PARAMS, SiliconFlowConfig
 from camel.messages import OpenAIMessage
-from camel.models import BaseModelBackend
+from camel.models.base_model import BaseModelBackend
 from camel.types import (
     ChatCompletion,
     ChatCompletionChunk,

camel/societies/workforce/role_playing_worker.py

@@ -168,11 +168,9 @@ class RolePlayingWorker(Worker):
             chat_history=chat_history_str,
             additional_info=task.additional_info,
         )
-        req = BaseMessage.make_user_message(
-            role_name="User",
-            content=prompt,
+        response = self.summarize_agent.step(
+            prompt, response_format=TaskResult
         )
-        response = self.summarize_agent.step(req, response_format=TaskResult)
         result_dict = json.loads(response.msg.content)
         task_result = TaskResult(**result_dict)
         task.result = task_result.content

camel/societies/workforce/single_agent_worker.py

@@ -19,7 +19,6 @@ from typing import Any, List
 from colorama import Fore
 
 from camel.agents import ChatAgent
-from camel.messages.base import BaseMessage
 from camel.societies.workforce.prompts import PROCESS_TASK_PROMPT
 from camel.societies.workforce.utils import TaskResult
 from camel.societies.workforce.worker import Worker
@@ -72,12 +71,8 @@ class SingleAgentWorker(Worker):
             dependency_tasks_info=dependency_tasks_info,
             additional_info=task.additional_info,
         )
-        req = BaseMessage.make_user_message(
-            role_name="User",
-            content=prompt,
-        )
         try:
-            response = self.worker.step(req, response_format=TaskResult)
+            response = self.worker.step(prompt, response_format=TaskResult)
         except Exception as e:
             print(
                 f"{Fore.RED}Error occurred while processing task {task.id}:"

camel/societies/workforce/workforce.py

@@ -281,13 +281,9 @@ class Workforce(BaseNode):
             child_nodes_info=self._get_child_nodes_info(),
             additional_info=task.additional_info,
         )
-        req = BaseMessage.make_user_message(
-            role_name="User",
-            content=prompt,
-        )
 
         response = self.coordinator_agent.step(
-            req, response_format=TaskAssignResult
+            prompt, response_format=TaskAssignResult
         )
         result_dict = json.loads(response.msg.content)
         task_assign_result = TaskAssignResult(**result_dict)
@@ -315,11 +311,9 @@ class Workforce(BaseNode):
             child_nodes_info=self._get_child_nodes_info(),
             additional_info=task.additional_info,
         )
-        req = BaseMessage.make_user_message(
-            role_name="User",
-            content=prompt,
+        response = self.coordinator_agent.step(
+            prompt, response_format=WorkerConf
         )
-        response = self.coordinator_agent.step(req, response_format=WorkerConf)
         result_dict = json.loads(response.msg.content)
         new_node_conf = WorkerConf(**result_dict)
 

camel/toolkits/__init__.py

@@ -46,6 +46,9 @@ from .stripe_toolkit import StripeToolkit
 from .video_toolkit import VideoDownloaderToolkit
 from .dappier_toolkit import DappierToolkit
 from .semantic_scholar_toolkit import SemanticScholarToolkit
+from .sympy_toolkit import SymPyToolkit
+from .mineru_toolkit import MinerUToolkit
+
 
 __all__ = [
     'BaseToolkit',
@@ -79,4 +82,6 @@ __all__ = [
     'OpenBBToolkit',
     'DappierToolkit',
     'SemanticScholarToolkit',
+    'SymPyToolkit',
+    'MinerUToolkit',
 ]

camel/toolkits/mineru_toolkit.py

@@ -0,0 +1,178 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from typing import Dict, List, Optional
+
+from camel.loaders.mineru_extractor import MinerU
+from camel.toolkits.base import BaseToolkit
+from camel.toolkits.function_tool import FunctionTool
+from camel.utils import api_keys_required
+
+
+class MinerUToolkit(BaseToolkit):
+    r"""Toolkit for extracting and processing document content
+        using MinerU API.
+
+    Provides comprehensive document processing capabilities including content
+    extraction from URLs and files, with support for OCR, formula recognition,
+    and table detection through the MinerU API service.
+
+    Note:
+        - Maximum file size: 200MB per file
+        - Maximum pages: 600 pages per file
+        - Daily quota: 2000 pages for high-priority parsing
+        - Network restrictions may affect certain URLs (e.g., GitHub, AWS)
+    """
+
+    @api_keys_required(
+        [
+            (None, "MINERU_API_KEY"),
+        ]
+    )
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = "https://mineru.net/api/v4",
+        is_ocr: bool = False,
+        enable_formula: bool = False,
+        enable_table: bool = True,
+        layout_model: str = "doclayout_yolo",
+        language: str = "en",
+        wait: bool = True,
+        timeout: float = 300,
+    ) -> None:
+        r"""Initialize the MinerU document processing toolkit.
+
+        Args:
+            api_key (Optional[str]): Authentication key for MinerU API access.
+                If not provided, uses MINERU_API_KEY environment variable.
+                (default: :obj:`None`)
+            api_url (Optional[str]): Base endpoint URL for MinerU API service.
+                (default: :obj:`"https://mineru.net/api/v4"`)
+            is_ocr (bool): Enable Optical Character Recognition for image-based
+                text extraction. (default: :obj:`False`)
+            enable_formula (bool): Enable mathematical formula detection and
+                recognition. (default: :obj:`False`)
+            enable_table (bool): Enable table structure detection and
+                extraction. (default: :obj:`True`)
+            layout_model (str): Document layout analysis model selection.
+                Available options: 'doclayout_yolo', 'layoutlmv3'.
+                (default: :obj:`"doclayout_yolo"`)
+            language (str): Primary language of the document for processing.
+                (default: :obj:`"en"`)
+            wait (bool): Block execution until processing completion.
+                (default: :obj:`True`)
+            timeout (float): Maximum duration in seconds to wait for task
+                completion. (default: :obj:`300`)
+        """
+        self.client = MinerU(
+            api_key=api_key,
+            api_url=api_url,
+            is_ocr=is_ocr,
+            enable_formula=enable_formula,
+            enable_table=enable_table,
+            layout_model=layout_model,
+            language=language,
+        )
+        self.wait = wait
+        self.timeout = timeout
+
+    def extract_from_urls(
+        self,
+        urls: str | List[str],
+    ) -> Dict:
+        r"""Process and extract content from one or multiple URLs.
+
+        Args:
+            urls (str | List[str]): Target URL or list of URLs for content
+                extraction. Supports both single URL string and multiple URLs
+                in a list.
+
+        Returns:
+            Dict: Response containing either completed task results when wait
+                is True, or task/batch identifiers for status tracking when
+                wait is False.
+        """
+        if isinstance(urls, str):
+            # Single URL case
+            response = self.client.extract_url(url=urls)
+
+            if self.wait:
+                return self.client.wait_for_completion(
+                    response['task_id'],
+                    timeout=self.timeout,
+                )
+            return response
+        else:
+            # Multiple URLs case
+            files: List[Dict[str, str | bool]] = [
+                {"url": str(url)} for url in urls
+            ]
+            batch_id = self.client.batch_extract_urls(files=files)
+
+            if self.wait:
+                return self.client.wait_for_completion(
+                    batch_id,
+                    is_batch=True,
+                    timeout=self.timeout if self.timeout > 300 else 600,
+                )
+            return {"batch_id": batch_id}
+
+    def get_task_status(self, task_id: str) -> Dict:
+        r"""Retrieve current status of an individual extraction task.
+
+        Args:
+            task_id (str): Unique identifier for the extraction task to check.
+
+        Returns:
+            Dict: Status information and results (if task is completed) for
+                the specified task.
+
+        Note:
+            This is a low-level status checking method. For most use cases,
+            prefer using extract_from_url with wait=True for automatic
+            completion handling.
+        """
+        return self.client.get_task_status(task_id)
+
+    def get_batch_status(self, batch_id: str) -> Dict:
+        r"""Retrieve current status of a batch extraction task.
+
+        Args:
+            batch_id (str): Unique identifier for the batch extraction task
+                to check.
+
+        Returns:
+            Dict: Comprehensive status information and results for all files
+                in the batch task.
+
+        Note:
+            This is a low-level status checking method. For most use cases,
+            prefer using batch_extract_from_urls with wait=True for automatic
+            completion handling.
+        """
+        return self.client.get_batch_status(batch_id)
+
+    def get_tools(self) -> List[FunctionTool]:
+        r"""Retrieve available toolkit functions as FunctionTool objects.
+
+        Returns:
+            List[FunctionTool]: Collection of FunctionTool objects representing
+                the available document processing functions in this toolkit.
+        """
+        return [
+            FunctionTool(self.extract_from_urls),
+            FunctionTool(self.get_task_status),
+            FunctionTool(self.get_batch_status),
+        ]

camel/toolkits/reddit_toolkit.py

@@ -16,10 +16,9 @@ import os
 import time
 from typing import Any, Dict, List, Union
 
-from requests.exceptions import RequestException
-
 from camel.toolkits import FunctionTool
 from camel.toolkits.base import BaseToolkit
+from camel.utils import retry_on_error
 
 
 class RedditToolkit(BaseToolkit):
@@ -61,30 +60,7 @@ class RedditToolkit(BaseToolkit):
             request_timeout=30,  # Set a timeout to handle delays
         )
 
-    def _retry_request(self, func, *args, **kwargs):
-        r"""Retries a function in case of network-related errors.
-
-        Args:
-            func (callable): The function to be retried.
-            *args: Arguments to pass to the function.
-            **kwargs: Keyword arguments to pass to the function.
-
-        Returns:
-            Any: The result of the function call if successful.
-
-        Raises:
-            RequestException: If all retry attempts fail.
-        """
-        for attempt in range(self.retries):
-            try:
-                return func(*args, **kwargs)
-            except RequestException as e:
-                print(f"Attempt {attempt + 1}/{self.retries} failed: {e}")
-                if attempt < self.retries - 1:
-                    time.sleep(self.delay)
-                else:
-                    raise
-
+    @retry_on_error()
     def collect_top_posts(
         self,
         subreddit_name: str,
@@ -113,8 +89,8 @@ class RedditToolkit(BaseToolkit):
                 "Please set the environment variables."
             )
 
-        subreddit = self._retry_request(self.reddit.subreddit, subreddit_name)
-        top_posts = self._retry_request(subreddit.top, limit=post_limit)
+        subreddit = self.reddit.subreddit(subreddit_name)
+        top_posts = subreddit.top(limit=post_limit)
         data = []
 
         for post in top_posts:
@@ -122,9 +98,7 @@ class RedditToolkit(BaseToolkit):
                 "Post Title": post.title,
                 "Comments": [
                     {"Comment Body": comment.body, "Upvotes": comment.score}
-                    for comment in self._retry_request(
-                        lambda post=post: list(post.comments)
-                    )[:comment_limit]
+                    for comment in list(post.comments)[:comment_limit]
                 ],
             }
             data.append(post_data)
@@ -192,15 +166,11 @@ class RedditToolkit(BaseToolkit):
         data = []
 
         for subreddit_name in subreddits:
-            subreddit = self._retry_request(
-                self.reddit.subreddit, subreddit_name
-            )
-            top_posts = self._retry_request(subreddit.top, limit=post_limit)
+            subreddit = self.reddit.subreddit(subreddit_name)
+            top_posts = subreddit.top(limit=post_limit)
 
             for post in top_posts:
-                for comment in self._retry_request(
-                    lambda post=post: list(post.comments)
-                )[:comment_limit]:
+                for comment in list(post.comments)[:comment_limit]:
                     # Print comment body for debugging
                     if any(
                         keyword.lower() in comment.body.lower()