camel-ai 0.1.5.6__py3-none-any.whl → 0.1.6.1__py3-none-any.whl

camel/loaders/firecrawl_reader.py

@@ -0,0 +1,213 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel
+
+
+class Firecrawl:
+    r"""Firecrawl allows you to turn entire websites into LLM-ready markdown.
+
+    Args:
+        api_key (Optional[str]): API key for authenticating with the Firecrawl
+            API.
+        api_url (Optional[str]): Base URL for the Firecrawl API.
+
+    References:
+        https://docs.firecrawl.dev/introduction
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = None,
+    ) -> None:
+        from firecrawl import FirecrawlApp
+
+        self._api_key = api_key or os.environ.get("FIRECRAWL_API_KEY")
+        self._api_url = api_url or os.environ.get("FIRECRAWL_API_URL")
+
+        self.app = FirecrawlApp(api_key=self._api_key, api_url=self._api_url)
+
+    def crawl(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        wait_until_done: bool = True,
+        **kwargs: Any,
+    ) -> Any:
+        r"""Crawl a URL and all accessible subpages. Customize the crawl by
+        setting different parameters, and receive the full response or a job
+        ID based on the specified options.
+
+        Args:
+            url (str): The URL to crawl.
+            params (Optional[Dict[str, Any]]): Additional parameters for the
+                crawl request. Defaults to `None`.
+            wait_until_done (bool): Whether to wait until the crawl job is
+                completed. Defaults to `True`.
+            **kwargs (Any): Additional keyword arguments, such as
+                `poll_interval`, `idempotency_key`, etc.
+
+        Returns:
+            Any: The list content of the URL if `wait_until_done` is True;
+                otherwise, a string job ID.
+
+        Raises:
+            RuntimeError: If the crawling process fails.
+        """
+
+        try:
+            crawl_response = self.app.crawl_url(
+                url=url,
+                params=params,
+                **kwargs,
+                wait_until_done=wait_until_done,
+            )
+            return (
+                crawl_response
+                if wait_until_done
+                else crawl_response.get("jobId")
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to crawl the URL: {e}")
+
+    def markdown_crawl(self, url: str) -> str:
+        r"""Crawl a URL and all accessible subpages and return the content in
+        Markdown format.
+
+        Args:
+            url (str): The URL to crawl.
+
+        Returns:
+            str: The content of the URL in Markdown format.
+
+        Raises:
+            RuntimeError: If the crawling process fails.
+        """
+
+        try:
+            crawl_result = self.app.crawl_url(url=url)
+            if not isinstance(crawl_result, list):
+                raise ValueError("Unexpected response format")
+            markdown_contents = [
+                result.get('markdown', '') for result in crawl_result
+            ]
+            return '\n'.join(markdown_contents)
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to crawl the URL and retrieve markdown: {e}"
+            )
+
+    def check_crawl_job(self, job_id: str) -> Dict:
+        r"""Check the status of a crawl job.
+
+        Args:
+            job_id (str): The ID of the crawl job.
+
+        Returns:
+            Dict: The response including status of the crawl job.
+
+        Raises:
+            RuntimeError: If the check process fails.
+        """
+
+        try:
+            return self.app.check_crawl_status(job_id)
+        except Exception as e:
+            raise RuntimeError(f"Failed to check the crawl job status: {e}")
+
+    def scrape(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Dict:
+        r"""To scrape a single URL. This function supports advanced scraping
+        by setting different parameters and returns the full scraped data as a
+        dictionary.
+
+        Reference: https://docs.firecrawl.dev/advanced-scraping-guide
+
+        Args:
+            url (str): The URL to read.
+            params (Optional[Dict[str, Any]]): Additional parameters for the
+                scrape request.
+
+        Returns:
+            Dict: The scraped data.
+
+        Raises:
+            RuntimeError: If the scrape process fails.
+        """
+        try:
+            return self.app.scrape_url(url=url, params=params)
+        except Exception as e:
+            raise RuntimeError(f"Failed to scrape the URL: {e}")
+
+    def structured_scrape(self, url: str, output_schema: BaseModel) -> Dict:
+        r"""Use LLM to extract structured data from given URL.
+
+        Args:
+            url (str): The URL to read.
+            output_schema (BaseModel): A pydantic model
+                that includes value types and field descriptions used to
+                generate a structured response by LLM. This schema helps
+                in defining the expected output format.
+
+        Returns:
+            Dict: The content of the URL.
+
+        Raises:
+            RuntimeError: If the scrape process fails.
+        """
+        try:
+            data = self.app.scrape_url(
+                url,
+                {
+                    'extractorOptions': {
+                        "mode": "llm-extraction",
+                        "extractionPrompt": "Based on the information on "
+                        "the page, extract the information from the schema.",
+                        'extractionSchema': output_schema.model_json_schema(),
+                    },
+                    'pageOptions': {'onlyMainContent': True},
+                },
+            )
+            return data.get("llm_extraction", {})
+        except Exception as e:
+            raise RuntimeError(f"Failed to perform structured scrape: {e}")
+
+    def tidy_scrape(self, url: str) -> str:
+        r"""Only return the main content of the page, excluding headers,
+        navigation bars, footers, etc. in Markdown format.
+
+        Args:
+            url (str): The URL to read.
+
+        Returns:
+            str: The markdown content of the URL.
+
+        Raises:
+            RuntimeError: If the scrape process fails.
+        """
+
+        try:
+            scrape_result = self.app.scrape_url(
+                url, {'pageOptions': {'onlyMainContent': True}}
+            )
+            return scrape_result.get("markdown", "")
+        except Exception as e:
+            raise RuntimeError(f"Failed to perform tidy scrape: {e}")

camel/memories/agent_memories.py

@@ -16,10 +16,7 @@ from typing import List, Optional
 
 from camel.memories.base import AgentMemory, BaseContextCreator
 from camel.memories.blocks import ChatHistoryBlock, VectorDBBlock
-from camel.memories.records import (
-    ContextRecord,
-    MemoryRecord,
-)
+from camel.memories.records import ContextRecord, MemoryRecord
 from camel.storages import BaseKeyValueStorage, BaseVectorStorage
 from camel.types import OpenAIBackendRole
 

camel/memories/blocks/chat_history_block.py

@@ -84,11 +84,15 @@ class ChatHistoryBlock(MemoryBlock):
         for record in reversed(chat_records):
             if record.role_at_backend == OpenAIBackendRole.SYSTEM:
                 # System messages are always kept.
-                output_records.append(ContextRecord(record, 1.0))
+                output_records.append(
+                    ContextRecord(memory_record=record, score=1.0)
+                )
             else:
                 # Other messages' score drops down gradually
                 score *= self.keep_rate
-                output_records.append(ContextRecord(record, score))
+                output_records.append(
+                    ContextRecord(memory_record=record, score=score)
+                )
 
         output_records.reverse()
         return output_records

camel/memories/blocks/vectordb_block.py

@@ -67,7 +67,9 @@ class VectorDBBlock(MemoryBlock):
                 vector database based on similarity to :obj:`current_state`.
         """
         query_vector = self.embedding.embed(keyword)
-        results = self.storage.query(VectorDBQuery(query_vector, top_k=limit))
+        results = self.storage.query(
+            VectorDBQuery(query_vector=query_vector, top_k=limit)
+        )
         return [
             ContextRecord(
                 memory_record=MemoryRecord.from_dict(result.record.payload),

camel/memories/context_creators/score_based.py

@@ -11,17 +11,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from dataclasses import dataclass
 from typing import List, Tuple
 
+from pydantic import BaseModel
+
 from camel.memories.base import BaseContextCreator
 from camel.memories.records import ContextRecord
 from camel.messages import OpenAIMessage
 from camel.utils import BaseTokenCounter
 
 
-@dataclass(frozen=True)
-class _ContextUnit:
+class _ContextUnit(BaseModel):
     idx: int
     record: ContextRecord
     num_tokens: int
@@ -88,9 +88,9 @@ class ScoreBasedContextCreator(BaseContextCreator):
                 uuid_set.add(record.memory_record.uuid)
                 context_units.append(
                     _ContextUnit(
-                        idx,
-                        record,
-                        self.token_counter.count_tokens_from_messages(
+                        idx=idx,
+                        record=record,
+                        num_tokens=self.token_counter.count_tokens_from_messages(
                             [record.memory_record.to_openai_message()]
                         ),
                     )

camel/memories/records.py

@@ -12,16 +12,17 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict
 from typing import Any, ClassVar, Dict
 from uuid import UUID, uuid4
 
+from pydantic import BaseModel, ConfigDict, Field
+
 from camel.messages import BaseMessage, FunctionCallingMessage, OpenAIMessage
 from camel.types import OpenAIBackendRole
 
 
-@dataclass(frozen=True)
-class MemoryRecord:
+class MemoryRecord(BaseModel):
     r"""The basic message storing unit in the CAMEL memory system.
 
     Attributes:
@@ -38,10 +39,12 @@ class MemoryRecord:
             will be an empty `Dict`.
     """
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     message: BaseMessage
     role_at_backend: OpenAIBackendRole
-    uuid: UUID = field(default_factory=uuid4)
-    extra_info: Dict[str, str] = field(default_factory=dict)
+    uuid: UUID = Field(default_factory=uuid4)
+    extra_info: Dict[str, str] = Field(default_factory=dict)
 
     _MESSAGE_TYPES: ClassVar[dict] = {
         "BaseMessage": BaseMessage,
@@ -85,8 +88,7 @@ class MemoryRecord:
         return self.message.to_openai_message(self.role_at_backend)
 
 
-@dataclass(frozen=True)
-class ContextRecord:
+class ContextRecord(BaseModel):
     r"""The result of memory retrieving."""
 
     memory_record: MemoryRecord

camel/messages/base.py

@@ -60,6 +60,7 @@ class BaseMessage:
     role_type: RoleType
     meta_dict: Optional[Dict[str, str]]
     content: str
+
     video_bytes: Optional[bytes] = None
     image_list: Optional[List[Image.Image]] = None
     image_detail: Literal["auto", "low", "high"] = "auto"

camel/models/__init__.py

@@ -12,9 +12,12 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from .anthropic_model import AnthropicModel
+from .azure_openai_model import AzureOpenAIModel
 from .base_model import BaseModelBackend
 from .gemini_model import GeminiModel
+from .groq_model import GroqModel
 from .litellm_model import LiteLLMModel
+from .mistral_model import MistralModel
 from .model_factory import ModelFactory
 from .nemotron_model import NemotronModel
 from .ollama_model import OllamaModel
@@ -22,12 +25,16 @@ from .open_source_model import OpenSourceModel
 from .openai_audio_models import OpenAIAudioModels
 from .openai_model import OpenAIModel
 from .stub_model import StubModel
+from .vllm_model import VLLMModel
 from .zhipuai_model import ZhipuAIModel
 
 __all__ = [
     'BaseModelBackend',
     'OpenAIModel',
+    'AzureOpenAIModel',
     'AnthropicModel',
+    'MistralModel',
+    'GroqModel',
     'StubModel',
     'ZhipuAIModel',
     'OpenSourceModel',
@@ -36,5 +43,6 @@ __all__ = [
     'OpenAIAudioModels',
     'NemotronModel',
     'OllamaModel',
+    'VLLMModel',
     'GeminiModel',
 ]

camel/models/anthropic_model.py

@@ -36,6 +36,7 @@ class AnthropicModel(BaseModelBackend):
         model_config_dict: Dict[str, Any],
         api_key: Optional[str] = None,
         url: Optional[str] = None,
+        token_counter: Optional[BaseTokenCounter] = None,
     ) -> None:
         r"""Constructor for Anthropic backend.
 
@@ -48,12 +49,16 @@ class AnthropicModel(BaseModelBackend):
                 Anthropic service. (default: :obj:`None`)
             url (Optional[str]): The url to the Anthropic service. (default:
                 :obj:`None`)
+            token_counter (Optional[BaseTokenCounter]): Token counter to use
+                for the model. If not provided, `AnthropicTokenCounter` will
+                be used.
         """
-        super().__init__(model_type, model_config_dict, api_key, url)
+        super().__init__(
+            model_type, model_config_dict, api_key, url, token_counter
+        )
         self._api_key = api_key or os.environ.get("ANTHROPIC_API_KEY")
         self._url = url or os.environ.get("ANTHROPIC_API_BASE_URL")
         self.client = Anthropic(api_key=self._api_key, base_url=self._url)
-        self._token_counter: Optional[BaseTokenCounter] = None
 
     def _convert_response_from_anthropic_to_openai(self, response):
         # openai ^1.0.0 format, reference openai/types/chat/chat_completion.py

camel/models/azure_openai_model.py

@@ -0,0 +1,152 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+import os
+from typing import Any, Dict, List, Optional, Union
+
+from openai import AzureOpenAI, Stream
+
+from camel.configs import OPENAI_API_PARAMS
+from camel.messages import OpenAIMessage
+from camel.models.base_model import BaseModelBackend
+from camel.types import ChatCompletion, ChatCompletionChunk, ModelType
+from camel.utils import BaseTokenCounter, OpenAITokenCounter, api_keys_required
+
+
+class AzureOpenAIModel(BaseModelBackend):
+    r"""Azure OpenAI API in a unified BaseModelBackend interface.
+    Doc: https://learn.microsoft.com/en-us/azure/ai-services/openai/
+    """
+
+    def __init__(
+        self,
+        model_type: ModelType,
+        model_config_dict: Dict[str, Any],
+        api_key: Optional[str] = None,
+        url: Optional[str] = None,
+        api_version: Optional[str] = None,
+        azure_deployment_name: Optional[str] = None,
+    ) -> None:
+        r"""Constructor for OpenAI backend.
+
+        Args:
+            model_type (ModelType): Model for which a backend is created,
+                one of GPT_* series.
+            model_config_dict (Dict[str, Any]): A dictionary that will
+                be fed into openai.ChatCompletion.create().
+            api_key (Optional[str]): The API key for authenticating with the
+                OpenAI service. (default: :obj:`None`)
+            url (Optional[str]): The url to the OpenAI service. (default:
+                :obj:`None`)
+            api_version (Optional[str]): The api version for the model.
+            azure_deployment_name (Optional[str]): The deployment name you
+                chose when you deployed an azure model. (default: :obj:`None`)
+        """
+        super().__init__(model_type, model_config_dict, api_key, url)
+        self._url = url or os.environ.get("AZURE_OPENAI_ENDPOINT")
+        self._api_key = api_key or os.environ.get("AZURE_OPENAI_API_KEY")
+        self.api_version = api_version or os.environ.get("AZURE_API_VERSION")
+        self.azure_deployment_name = azure_deployment_name or os.environ.get(
+            "AZURE_DEPLOYMENT_NAME"
+        )
+
+        if self._url is None:
+            raise ValueError(
+                "Must provide either the `url` argument "
+                "or `AZURE_OPENAI_ENDPOINT` environment variable."
+            )
+        if self._api_key is None:
+            raise ValueError(
+                "Must provide either the `api_key` argument "
+                "or `AZURE_OPENAI_API_KEY` environment variable."
+            )
+        if self.api_version is None:
+            raise ValueError(
+                "Must provide either the `api_version` argument "
+                "or `AZURE_API_VERSION` environment variable."
+            )
+        if self.azure_deployment_name is None:
+            raise ValueError(
+                "Must provide either the `azure_deployment_name` argument "
+                "or `AZURE_DEPLOYMENT_NAME` environment variable."
+            )
+        self.model = str(self.azure_deployment_name)
+
+        self._client = AzureOpenAI(
+            azure_endpoint=str(self._url),
+            azure_deployment=self.azure_deployment_name,
+            api_version=self.api_version,
+            api_key=self._api_key,
+            timeout=60,
+            max_retries=3,
+        )
+        self._token_counter: Optional[BaseTokenCounter] = None
+
+    @property
+    def token_counter(self) -> BaseTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            BaseTokenCounter: The token counter following the model's
+                tokenization style.
+        """
+        if not self._token_counter:
+            self._token_counter = OpenAITokenCounter(self.model_type)
+        return self._token_counter
+
+    @api_keys_required("AZURE_OPENAI_API_KEY", "AZURE_API_VERSION")
+    def run(
+        self,
+        messages: List[OpenAIMessage],
+    ) -> Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+        r"""Runs inference of Azure OpenAI chat completion.
+
+        Args:
+            messages (List[OpenAIMessage]): Message list with the chat history
+                in OpenAI API format.
+
+        Returns:
+            Union[ChatCompletion, Stream[ChatCompletionChunk]]:
+                `ChatCompletion` in the non-stream mode, or
+                `Stream[ChatCompletionChunk]` in the stream mode.
+        """
+        response = self._client.chat.completions.create(
+            messages=messages,
+            model=self.model,
+            **self.model_config_dict,
+        )
+        return response
+
+    def check_model_config(self):
+        r"""Check whether the model configuration contains any
+        unexpected arguments to Azure OpenAI API.
+
+        Raises:
+            ValueError: If the model configuration dictionary contains any
+                unexpected arguments to Azure OpenAI API.
+        """
+        for param in self.model_config_dict:
+            if param not in OPENAI_API_PARAMS:
+                raise ValueError(
+                    f"Unexpected argument `{param}` is "
+                    "input into Azure OpenAI model backend."
+                )
+
+    @property
+    def stream(self) -> bool:
+        r"""Returns whether the model is in stream mode,
+            which sends partial results each time.
+        Returns:
+            bool: Whether the model is in stream mode.
+        """
+        return self.model_config_dict.get("stream", False)

camel/models/base_model.py

@@ -32,6 +32,7 @@ class BaseModelBackend(ABC):
         model_config_dict: Dict[str, Any],
         api_key: Optional[str] = None,
         url: Optional[str] = None,
+        token_counter: Optional[BaseTokenCounter] = None,
     ) -> None:
         r"""Constructor for the model backend.
 
@@ -41,13 +42,16 @@ class BaseModelBackend(ABC):
             api_key (Optional[str]): The API key for authenticating with the
                 model service.
             url (Optional[str]): The url to the model service.
+            token_counter (Optional[BaseTokenCounter]): Token counter to use
+                for the model. If not provided, `OpenAITokenCounter` will
+                be used.
         """
         self.model_type = model_type
-
         self.model_config_dict = model_config_dict
         self._api_key = api_key
         self._url = url
         self.check_model_config()
+        self._token_counter = token_counter
 
     @property
     @abstractmethod
@@ -109,7 +113,10 @@ class BaseModelBackend(ABC):
         Returns:
             int: The maximum token limit for the given model.
         """
-        return self.model_type.token_limit
+        return (
+            self.model_config_dict.get("max_tokens")
+            or self.model_type.token_limit
+        )
 
     @property
     def stream(self) -> bool:

camel/models/gemini_model.py

@@ -44,6 +44,7 @@ class GeminiModel(BaseModelBackend):
         model_config_dict: Dict[str, Any],
         api_key: Optional[str] = None,
         url: Optional[str] = None,
+        token_counter: Optional[BaseTokenCounter] = None,
     ) -> None:
         r"""Constructor for Gemini backend.
 
@@ -54,17 +55,22 @@ class GeminiModel(BaseModelBackend):
             api_key (Optional[str]): The API key for authenticating with the
                 gemini service. (default: :obj:`None`)
             url (Optional[str]): The url to the gemini service.
+            token_counter (Optional[BaseTokenCounter]): Token counter to use
+                for the model. If not provided, `GeminiTokenCounter` will be
+                used.
         """
         import os
 
         import google.generativeai as genai
         from google.generativeai.types.generation_types import GenerationConfig
 
-        super().__init__(model_type, model_config_dict, api_key, url)
+        super().__init__(
+            model_type, model_config_dict, api_key, url, token_counter
+        )
         self._api_key = api_key or os.environ.get("GOOGLE_API_KEY")
         genai.configure(api_key=self._api_key)
         self._client = genai.GenerativeModel(self.model_type.value)
-        self._token_counter: Optional[BaseTokenCounter] = None
+
         keys = list(self.model_config_dict.keys())
         generation_config_dict = {
             k: self.model_config_dict.pop(k)
@@ -78,6 +84,12 @@ class GeminiModel(BaseModelBackend):
 
     @property
     def token_counter(self) -> BaseTokenCounter:
+        r"""Initialize the token counter for the model backend.
+
+        Returns:
+            BaseTokenCounter: The token counter following the model's
+                tokenization style.
+        """
         if not self._token_counter:
             self._token_counter = GeminiTokenCounter(self.model_type)
         return self._token_counter