camel-ai 0.2.41__py3-none-any.whl → 0.2.43__py3-none-any.whl

camel/loaders/crawl4ai_reader.py

@@ -0,0 +1,230 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import asyncio
+import logging
+from typing import Any, Dict, List, Optional, Set
+
+from pydantic import BaseModel, ValidationError
+
+logger = logging.getLogger(__name__)
+
+
+class Crawl4AI:
+    r"""Class for converting websites into LLM-ready data.
+
+    This class uses asynchronous crawling with CSS selectors or LLM-based
+    extraction to convert entire websites into structured data.
+
+    References:
+        https://docs.crawl4ai.com/
+    """
+
+    def __init__(self) -> None:
+        from crawl4ai import AsyncWebCrawler
+
+        self.crawler_class = AsyncWebCrawler
+
+    async def _run_crawler(self, url: str, **kwargs) -> Any:
+        r"""Run the asynchronous web crawler on a given URL.
+
+        Args:
+            url (str): URL to crawl or scrape.
+            **kwargs: Additional keyword arguments for crawler configuration.
+
+        Returns:
+            Any: The result from the crawler.
+
+        Raises:
+            RuntimeError: If crawler execution fails.
+        """
+
+        try:
+            async with self.crawler_class() as c:
+                return await c.arun(url, **kwargs)
+        except Exception as e:
+            logger.error("Crawler run failed: %s", e)
+            raise RuntimeError(f"Crawler run failed: {e}") from e
+
+    async def crawl(
+        self,
+        start_url: str,
+        max_depth: int = 1,
+        extraction_strategy=None,
+        **kwargs,
+    ) -> List[Dict[str, Any]]:
+        r"""Crawl a URL and its subpages using breadth-first search.
+
+        Args:
+            start_url (str): URL to start crawling from.
+            max_depth (int, optional): Maximum depth of links to follow
+                (default: :obj:`1`)
+            extraction_strategy (ExtractionStrategy, optional): Strategy
+                for data extraction. (default: :obj:`None`)
+            **kwargs: Additional arguments for crawler configuration.
+
+        Returns:
+            List[Dict[str, Any]]: List of crawled page results.
+
+        Raises:
+            RuntimeError: If an error occurs during crawling.
+        """
+
+        all_results: List[Dict[str, Any]] = []
+        visited_urls: Set[str] = set()
+        queue: asyncio.Queue = asyncio.Queue()
+
+        await queue.put((start_url, 1))
+        visited_urls.add(start_url)
+
+        while not queue.empty():
+            url, depth = await queue.get()
+            try:
+                result = await self._run_crawler(
+                    url, extraction_strategy=extraction_strategy, **kwargs
+                )
+                all_results.append(
+                    {
+                        "url": url,
+                        "raw_result": result,
+                        "markdown": result.markdown,
+                        "cleaned_html": result.cleaned_html,
+                        "links": result.links,
+                    }
+                )
+
+                if depth < max_depth and result.links:
+                    for _, links in result.links.items():
+                        for link in links:
+                            if (
+                                'href' in link
+                                and link['href'] not in visited_urls
+                            ):
+                                visited_urls.add(link['href'])
+                                await queue.put((link['href'], depth + 1))
+
+            except Exception as e:
+                logger.error("Error crawling %s: %s", url, e)
+                raise RuntimeError(f"Error crawling {url}: {e}") from e
+
+            queue.task_done()
+
+        await queue.join()
+
+        return all_results
+
+    async def scrape(
+        self,
+        url: str,
+        extraction_strategy=None,
+        **kwargs,
+    ) -> Dict[str, Any]:
+        r"""Scrape a single URL using CSS or LLM-based extraction.
+
+        Args:
+            url (str): URL to scrape.
+            extraction_strategy (ExtractionStrategy, optional): Extraction
+                strategy to use. (default: :obj:`None`)
+            **kwargs: Additional arguments for crawler configuration.
+
+        Returns:
+            Dict[str, Any]: Dictionary containing scraped data such as markdown
+                and HTML content.
+
+        Raises:
+            RuntimeError: If scraping fails.
+        """
+
+        result = await self._run_crawler(
+            url, extraction_strategy=extraction_strategy, **kwargs
+        )
+        return {
+            "url": url,
+            "raw_result": result,
+            "markdown": result.markdown,
+            "cleaned_html": result.cleaned_html,
+            "links": result.links,
+        }
+
+    async def structured_scrape(
+        self,
+        url: str,
+        response_format: BaseModel,
+        api_key: Optional[str] = None,
+        llm_provider: str = 'ollama/llama3',
+        **kwargs,
+    ) -> Any:
+        r"""Extract structured data from a URL using an LLM.
+
+        Args:
+            url (str): URL to scrape.
+            response_format (BaseModel): Model defining the expected output
+                schema.
+            api_key (str, optional): API key for the LLM provider
+                (default: :obj:`None`).
+            llm_provider (str, optional): Identifier for the LLM provider
+                (default: :obj:`'ollama/llama3'`).
+            **kwargs: Additional arguments for crawler configuration.
+
+        Returns:
+            Any: Crawl result containing the extracted data
+                structured according to the schema.
+
+        Raises:
+            ValidationError: If extracted data does not match the schema.
+            RuntimeError: If extraction fails.
+        """
+
+        from crawl4ai.extraction_strategy import (
+            LLMExtractionStrategy,
+        )
+
+        extraction_strategy = LLMExtractionStrategy(
+            provider=llm_provider,
+            api_token=api_key,
+            schema=response_format.model_json_schema(),
+            extraction_type="schema",
+            instruction="Extract the data according to the schema.",
+        )
+
+        try:
+            return await self._run_crawler(
+                url, extraction_strategy=extraction_strategy, **kwargs
+            )
+        except ValidationError as e:
+            raise ValidationError(
+                f"Extracted data does not match schema: {e}"
+            ) from e
+        except Exception as e:
+            raise RuntimeError(e) from e
+
+    async def map_site(self, start_url: str, **kwargs) -> List[str]:
+        r"""Map a website by extracting all accessible URLs.
+
+        Args:
+            start_url (str): Starting URL to map.
+            **kwargs: Additional configuration arguments.
+
+        Returns:
+            List[str]: List of URLs discovered on the website.
+
+        Raises:
+            RuntimeError: If mapping fails.
+        """
+
+        try:
+            result = await self.crawl(start_url, **kwargs)
+            return [page["url"] for page in result]
+        except Exception as e:
+            raise RuntimeError(f"Failed to map url: {e}") from e

camel/models/__init__.py

@@ -35,6 +35,7 @@ from .openai_audio_models import OpenAIAudioModels
 from .openai_compatible_model import OpenAICompatibleModel
 from .openai_model import OpenAIModel
 from .openrouter_model import OpenRouterModel
+from .ppio_model import PPIOModel
 from .qwen_model import QwenModel
 from .reka_model import RekaModel
 from .samba_model import SambaModel
@@ -72,6 +73,7 @@ __all__ = [
     'RekaModel',
     'SambaModel',
     'TogetherAIModel',
+    'PPIOModel',
     'YiModel',
     'QwenModel',
     'ModelProcessingError',

camel/models/azure_openai_model.py

@@ -219,9 +219,13 @@ class AzureOpenAIModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
 
         request_config["response_format"] = response_format
+        # Remove stream from request config since OpenAI does not support it
+        # with structured response
         request_config.pop("stream", None)
         if tools is not None:
             request_config["tools"] = tools
@@ -238,9 +242,13 @@ class AzureOpenAIModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
 
         request_config["response_format"] = response_format
+        # Remove stream from request config since OpenAI does not support it
+        # with structured response
         request_config.pop("stream", None)
         if tools is not None:
             request_config["tools"] = tools

camel/models/base_model.py

@@ -108,42 +108,125 @@ class BaseModelBackend(ABC, metaclass=ModelBackendMeta):
     ) -> List[OpenAIMessage]:
         r"""Preprocess messages before sending to model API.
         Removes thinking content from assistant and user messages.
+        Automatically formats messages for parallel tool calls if tools are
+        detected.
 
         Args:
-            messages (List[OpenAIMessage]): Original messages
+            messages (List[OpenAIMessage]): Original messages.
 
         Returns:
             List[OpenAIMessage]: Preprocessed messages
         """
-
-        def should_process_thinking(msg: OpenAIMessage) -> bool:
-            # Only process thinking content for assistant and user messages
-            return msg['role'] in ['assistant', 'user'] and isinstance(
-                msg['content'], str
+        # Process all messages in a single pass
+        processed_messages = []
+        tool_calls_buffer: List[OpenAIMessage] = []
+        tool_responses_buffer: Dict[str, OpenAIMessage] = {}
+        has_tool_calls = False
+
+        for msg in messages:
+            # Remove thinking content if needed
+            role = msg.get('role')
+            content = msg.get('content')
+            if role in ['assistant', 'user'] and isinstance(content, str):
+                if '<think>' in content and '</think>' in content:
+                    content = re.sub(
+                        r'<think>.*?</think>', '', content, flags=re.DOTALL
+                    ).strip()
+                processed_msg = dict(msg)
+                processed_msg['content'] = content
+            else:
+                processed_msg = dict(msg)
+
+            # Check and track tool calls/responses
+            is_tool_call = (
+                processed_msg.get("role") == "assistant"
+                and "tool_calls" in processed_msg
+            )
+            is_tool_response = (
+                processed_msg.get("role") == "tool"
+                and "tool_call_id" in processed_msg
             )
 
-        def remove_thinking(content: str) -> str:
-            # Only remove thinking content if the tags are present
-            if '<think>' in content and '</think>' in content:
-                return re.sub(
-                    r'<think>.*?</think>',
-                    '',
-                    content,
-                    flags=re.DOTALL,
-                ).strip()
-            return content
-
-        return [
-            {  # type: ignore[misc]
-                **msg,
-                'content': (
-                    remove_thinking(msg['content'])  # type: ignore[arg-type]
-                    if should_process_thinking(msg)
-                    else msg['content']
-                ),
-            }
-            for msg in messages
-        ]
+            if is_tool_call or is_tool_response:
+                has_tool_calls = True
+
+            # Store the processed message for later formatting if needed
+            processed_messages.append(processed_msg)
+
+        # If no tool calls detected, return the processed messages
+        if not has_tool_calls:
+            return processed_messages  # type: ignore[return-value]
+
+        # Format messages for parallel tool calls
+        formatted_messages = []
+        tool_calls_buffer = []
+        tool_responses_buffer = {}
+
+        for msg in processed_messages:  # type: ignore[assignment]
+            # If this is an assistant message with tool calls, add it to the
+            # buffer
+            if msg.get("role") == "assistant" and "tool_calls" in msg:
+                tool_calls_buffer.append(msg)
+                continue
+
+            # If this is a tool response, add it to the responses buffer
+            if msg.get("role") == "tool" and "tool_call_id" in msg:
+                tool_call_id = msg.get("tool_call_id")
+                if isinstance(tool_call_id, str):
+                    tool_responses_buffer[tool_call_id] = msg
+                continue
+
+            # Process any complete tool call + responses before adding regular
+            # messages
+            if tool_calls_buffer and tool_responses_buffer:
+                # Add the assistant message with tool calls
+                assistant_msg = tool_calls_buffer[0]
+                formatted_messages.append(assistant_msg)
+
+                # Add all matching tool responses for this assistant message
+                tool_calls = assistant_msg.get("tool_calls", [])
+                if isinstance(tool_calls, list):
+                    for tool_call in tool_calls:
+                        tool_call_id = tool_call.get("id")
+                        if (
+                            isinstance(tool_call_id, str)
+                            and tool_call_id in tool_responses_buffer
+                        ):
+                            formatted_messages.append(
+                                tool_responses_buffer[tool_call_id]
+                            )
+                            del tool_responses_buffer[tool_call_id]
+
+                tool_calls_buffer.pop(0)
+
+            # Add the current regular message
+            formatted_messages.append(msg)
+
+        # Process any remaining buffered tool calls and responses
+        while tool_calls_buffer:
+            assistant_msg = tool_calls_buffer[0]
+            formatted_messages.append(assistant_msg)
+
+            tool_calls = assistant_msg.get("tool_calls", [])
+            if isinstance(tool_calls, list):
+                for tool_call in tool_calls:
+                    tool_call_id = tool_call.get("id")
+                    if (
+                        isinstance(tool_call_id, str)
+                        and tool_call_id in tool_responses_buffer
+                    ):
+                        formatted_messages.append(
+                            tool_responses_buffer[tool_call_id]
+                        )
+                        del tool_responses_buffer[tool_call_id]
+
+            tool_calls_buffer.pop(0)
+
+        # Add any remaining tool responses
+        for response in tool_responses_buffer.values():
+            formatted_messages.append(response)
+
+        return formatted_messages
 
     @abstractmethod
     def _run(

camel/models/cohere_model.py

@@ -227,7 +227,11 @@ class CohereModel(BaseModelBackend):
         response_format: Optional[Type[BaseModel]] = None,
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Dict[str, Any]:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove strict from each tool's function parameters since Cohere does
+        # not support them
         if tools:
             for tool in tools:
                 function_dict = tool.get('function', {})

camel/models/deepseek_model.py

@@ -145,7 +145,11 @@ class DeepSeekModel(BaseModelBackend):
                 for key, value in request_config.items()
                 if key not in REASONSER_UNSUPPORTED_PARAMS
             }
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove strict from each tool's function parameters since DeepSeek
+        # does not support them
         if tools:
             for tool in tools:
                 function_dict = tool.get('function', {})

camel/models/gemini_model.py

@@ -172,8 +172,11 @@ class GeminiModel(BaseModelBackend):
         messages: List[OpenAIMessage],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
-        request_config = self.model_config_dict.copy()
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove strict and anyOf from each tool's function parameters since
+        # Gemini does not support them
         if tools:
             for tool in tools:
                 function_dict = tool.get('function', {})
@@ -209,8 +212,11 @@ class GeminiModel(BaseModelBackend):
         messages: List[OpenAIMessage],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
-        request_config = self.model_config_dict.copy()
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove strict and anyOf from each tool's function parameters since
+        # Gemini does not support them
         if tools:
             for tool in tools:
                 function_dict = tool.get('function', {})

camel/models/model_factory.py

@@ -34,6 +34,7 @@ from camel.models.ollama_model import OllamaModel
 from camel.models.openai_compatible_model import OpenAICompatibleModel
 from camel.models.openai_model import OpenAIModel
 from camel.models.openrouter_model import OpenRouterModel
+from camel.models.ppio_model import PPIOModel
 from camel.models.qwen_model import QwenModel
 from camel.models.reka_model import RekaModel
 from camel.models.samba_model import SambaModel
@@ -145,6 +146,8 @@ class ModelFactory:
             model_class = QwenModel
         elif model_platform.is_deepseek:
             model_class = DeepSeekModel
+        elif model_platform.is_ppio:
+            model_class = PPIOModel
         elif model_platform.is_internlm and model_type.is_internlm:
             model_class = InternLMModel
         elif model_platform.is_moonshot and model_type.is_moonshot:

camel/models/ollama_model.py

@@ -250,8 +250,11 @@ class OllamaModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove stream from request_config since Ollama does not support it
+        # when structured response is used
         request_config["response_format"] = response_format
         request_config.pop("stream", None)
         if tools is not None:
@@ -269,8 +272,11 @@ class OllamaModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove stream from request_config since Ollama does not support it
+        # when structured response is used
         request_config["response_format"] = response_format
         request_config.pop("stream", None)
         if tools is not None:

camel/models/openai_compatible_model.py

@@ -177,8 +177,11 @@ class OpenAICompatibleModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove stream from request_config since OpenAI does not support it
+        # when structured response is used
         request_config["response_format"] = response_format
         request_config.pop("stream", None)
         if tools is not None:
@@ -196,8 +199,11 @@ class OpenAICompatibleModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
 
+        request_config = copy.deepcopy(self.model_config_dict)
+        # Remove stream from request_config since OpenAI does not support it
+        # when structured response is used
         request_config["response_format"] = response_format
         request_config.pop("stream", None)
         if tools is not None:

camel/models/openai_model.py

@@ -247,7 +247,9 @@ class OpenAIModel(BaseModelBackend):
         messages: List[OpenAIMessage],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
 
         if tools:
             request_config["tools"] = tools
@@ -265,7 +267,9 @@ class OpenAIModel(BaseModelBackend):
         messages: List[OpenAIMessage],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> Union[ChatCompletion, AsyncStream[ChatCompletionChunk]]:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
 
         if tools:
             request_config["tools"] = tools
@@ -284,9 +288,13 @@ class OpenAIModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
 
         request_config["response_format"] = response_format
+        # Remove stream from request config since OpenAI does not support it
+        # with structured response
         request_config.pop("stream", None)
         if tools is not None:
             request_config["tools"] = tools
@@ -305,9 +313,13 @@ class OpenAIModel(BaseModelBackend):
         response_format: Type[BaseModel],
         tools: Optional[List[Dict[str, Any]]] = None,
     ) -> ChatCompletion:
-        request_config = self.model_config_dict.copy()
+        import copy
+
+        request_config = copy.deepcopy(self.model_config_dict)
 
         request_config["response_format"] = response_format
+        # Remove stream from request config since OpenAI does not support it
+        # with structured response
         request_config.pop("stream", None)
         if tools is not None:
             request_config["tools"] = tools