camel-ai 0.1.5.9__py3-none-any.whl → 0.1.6.2__py3-none-any.whl

camel/loaders/firecrawl_reader.py

@@ -0,0 +1,213 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+
+import os
+from typing import Any, Dict, Optional
+
+from pydantic import BaseModel
+
+
+class Firecrawl:
+    r"""Firecrawl allows you to turn entire websites into LLM-ready markdown.
+
+    Args:
+        api_key (Optional[str]): API key for authenticating with the Firecrawl
+            API.
+        api_url (Optional[str]): Base URL for the Firecrawl API.
+
+    References:
+        https://docs.firecrawl.dev/introduction
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_url: Optional[str] = None,
+    ) -> None:
+        from firecrawl import FirecrawlApp
+
+        self._api_key = api_key or os.environ.get("FIRECRAWL_API_KEY")
+        self._api_url = api_url or os.environ.get("FIRECRAWL_API_URL")
+
+        self.app = FirecrawlApp(api_key=self._api_key, api_url=self._api_url)
+
+    def crawl(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+        wait_until_done: bool = True,
+        **kwargs: Any,
+    ) -> Any:
+        r"""Crawl a URL and all accessible subpages. Customize the crawl by
+        setting different parameters, and receive the full response or a job
+        ID based on the specified options.
+
+        Args:
+            url (str): The URL to crawl.
+            params (Optional[Dict[str, Any]]): Additional parameters for the
+                crawl request. Defaults to `None`.
+            wait_until_done (bool): Whether to wait until the crawl job is
+                completed. Defaults to `True`.
+            **kwargs (Any): Additional keyword arguments, such as
+                `poll_interval`, `idempotency_key`, etc.
+
+        Returns:
+            Any: The list content of the URL if `wait_until_done` is True;
+                otherwise, a string job ID.
+
+        Raises:
+            RuntimeError: If the crawling process fails.
+        """
+
+        try:
+            crawl_response = self.app.crawl_url(
+                url=url,
+                params=params,
+                **kwargs,
+                wait_until_done=wait_until_done,
+            )
+            return (
+                crawl_response
+                if wait_until_done
+                else crawl_response.get("jobId")
+            )
+        except Exception as e:
+            raise RuntimeError(f"Failed to crawl the URL: {e}")
+
+    def markdown_crawl(self, url: str) -> str:
+        r"""Crawl a URL and all accessible subpages and return the content in
+        Markdown format.
+
+        Args:
+            url (str): The URL to crawl.
+
+        Returns:
+            str: The content of the URL in Markdown format.
+
+        Raises:
+            RuntimeError: If the crawling process fails.
+        """
+
+        try:
+            crawl_result = self.app.crawl_url(url=url)
+            if not isinstance(crawl_result, list):
+                raise ValueError("Unexpected response format")
+            markdown_contents = [
+                result.get('markdown', '') for result in crawl_result
+            ]
+            return '\n'.join(markdown_contents)
+        except Exception as e:
+            raise RuntimeError(
+                f"Failed to crawl the URL and retrieve markdown: {e}"
+            )
+
+    def check_crawl_job(self, job_id: str) -> Dict:
+        r"""Check the status of a crawl job.
+
+        Args:
+            job_id (str): The ID of the crawl job.
+
+        Returns:
+            Dict: The response including status of the crawl job.
+
+        Raises:
+            RuntimeError: If the check process fails.
+        """
+
+        try:
+            return self.app.check_crawl_status(job_id)
+        except Exception as e:
+            raise RuntimeError(f"Failed to check the crawl job status: {e}")
+
+    def scrape(
+        self,
+        url: str,
+        params: Optional[Dict[str, Any]] = None,
+    ) -> Dict:
+        r"""To scrape a single URL. This function supports advanced scraping
+        by setting different parameters and returns the full scraped data as a
+        dictionary.
+
+        Reference: https://docs.firecrawl.dev/advanced-scraping-guide
+
+        Args:
+            url (str): The URL to read.
+            params (Optional[Dict[str, Any]]): Additional parameters for the
+                scrape request.
+
+        Returns:
+            Dict: The scraped data.
+
+        Raises:
+            RuntimeError: If the scrape process fails.
+        """
+        try:
+            return self.app.scrape_url(url=url, params=params)
+        except Exception as e:
+            raise RuntimeError(f"Failed to scrape the URL: {e}")
+
+    def structured_scrape(self, url: str, output_schema: BaseModel) -> Dict:
+        r"""Use LLM to extract structured data from given URL.
+
+        Args:
+            url (str): The URL to read.
+            output_schema (BaseModel): A pydantic model
+                that includes value types and field descriptions used to
+                generate a structured response by LLM. This schema helps
+                in defining the expected output format.
+
+        Returns:
+            Dict: The content of the URL.
+
+        Raises:
+            RuntimeError: If the scrape process fails.
+        """
+        try:
+            data = self.app.scrape_url(
+                url,
+                {
+                    'extractorOptions': {
+                        "mode": "llm-extraction",
+                        "extractionPrompt": "Based on the information on "
+                        "the page, extract the information from the schema.",
+                        'extractionSchema': output_schema.model_json_schema(),
+                    },
+                    'pageOptions': {'onlyMainContent': True},
+                },
+            )
+            return data.get("llm_extraction", {})
+        except Exception as e:
+            raise RuntimeError(f"Failed to perform structured scrape: {e}")
+
+    def tidy_scrape(self, url: str) -> str:
+        r"""Only return the main content of the page, excluding headers,
+        navigation bars, footers, etc. in Markdown format.
+
+        Args:
+            url (str): The URL to read.
+
+        Returns:
+            str: The markdown content of the URL.
+
+        Raises:
+            RuntimeError: If the scrape process fails.
+        """
+
+        try:
+            scrape_result = self.app.scrape_url(
+                url, {'pageOptions': {'onlyMainContent': True}}
+            )
+            return scrape_result.get("markdown", "")
+        except Exception as e:
+            raise RuntimeError(f"Failed to perform tidy scrape: {e}")

camel/memories/agent_memories.py

@@ -16,10 +16,7 @@ from typing import List, Optional
 
 from camel.memories.base import AgentMemory, BaseContextCreator
 from camel.memories.blocks import ChatHistoryBlock, VectorDBBlock
-from camel.memories.records import (
-    ContextRecord,
-    MemoryRecord,
-)
+from camel.memories.records import ContextRecord, MemoryRecord
 from camel.storages import BaseKeyValueStorage, BaseVectorStorage
 from camel.types import OpenAIBackendRole
 

camel/memories/blocks/chat_history_block.py

@@ -84,11 +84,15 @@ class ChatHistoryBlock(MemoryBlock):
         for record in reversed(chat_records):
             if record.role_at_backend == OpenAIBackendRole.SYSTEM:
                 # System messages are always kept.
-                output_records.append(ContextRecord(record, 1.0))
+                output_records.append(
+                    ContextRecord(memory_record=record, score=1.0)
+                )
             else:
                 # Other messages' score drops down gradually
                 score *= self.keep_rate
-                output_records.append(ContextRecord(record, score))
+                output_records.append(
+                    ContextRecord(memory_record=record, score=score)
+                )
 
         output_records.reverse()
         return output_records

camel/memories/blocks/vectordb_block.py

@@ -67,7 +67,9 @@ class VectorDBBlock(MemoryBlock):
                 vector database based on similarity to :obj:`current_state`.
         """
         query_vector = self.embedding.embed(keyword)
-        results = self.storage.query(VectorDBQuery(query_vector, top_k=limit))
+        results = self.storage.query(
+            VectorDBQuery(query_vector=query_vector, top_k=limit)
+        )
         return [
             ContextRecord(
                 memory_record=MemoryRecord.from_dict(result.record.payload),

camel/memories/context_creators/score_based.py

@@ -11,17 +11,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from dataclasses import dataclass
 from typing import List, Tuple
 
+from pydantic import BaseModel
+
 from camel.memories.base import BaseContextCreator
 from camel.memories.records import ContextRecord
 from camel.messages import OpenAIMessage
 from camel.utils import BaseTokenCounter
 
 
-@dataclass(frozen=True)
-class _ContextUnit:
+class _ContextUnit(BaseModel):
     idx: int
     record: ContextRecord
     num_tokens: int
@@ -88,9 +88,9 @@ class ScoreBasedContextCreator(BaseContextCreator):
                 uuid_set.add(record.memory_record.uuid)
                 context_units.append(
                     _ContextUnit(
-                        idx,
-                        record,
-                        self.token_counter.count_tokens_from_messages(
+                        idx=idx,
+                        record=record,
+                        num_tokens=self.token_counter.count_tokens_from_messages(
                             [record.memory_record.to_openai_message()]
                         ),
                     )

camel/memories/records.py

@@ -12,16 +12,17 @@
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 
-from dataclasses import asdict, dataclass, field
+from dataclasses import asdict
 from typing import Any, ClassVar, Dict
 from uuid import UUID, uuid4
 
+from pydantic import BaseModel, ConfigDict, Field
+
 from camel.messages import BaseMessage, FunctionCallingMessage, OpenAIMessage
 from camel.types import OpenAIBackendRole
 
 
-@dataclass(frozen=True)
-class MemoryRecord:
+class MemoryRecord(BaseModel):
     r"""The basic message storing unit in the CAMEL memory system.
 
     Attributes:
@@ -38,10 +39,12 @@ class MemoryRecord:
             will be an empty `Dict`.
     """
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     message: BaseMessage
     role_at_backend: OpenAIBackendRole
-    uuid: UUID = field(default_factory=uuid4)
-    extra_info: Dict[str, str] = field(default_factory=dict)
+    uuid: UUID = Field(default_factory=uuid4)
+    extra_info: Dict[str, str] = Field(default_factory=dict)
 
     _MESSAGE_TYPES: ClassVar[dict] = {
         "BaseMessage": BaseMessage,
@@ -85,8 +88,7 @@ class MemoryRecord:
         return self.message.to_openai_message(self.role_at_backend)
 
 
-@dataclass(frozen=True)
-class ContextRecord:
+class ContextRecord(BaseModel):
     r"""The result of memory retrieving."""
 
     memory_record: MemoryRecord

camel/messages/base.py

@@ -60,6 +60,7 @@ class BaseMessage:
     role_type: RoleType
     meta_dict: Optional[Dict[str, str]]
     content: str
+
     video_bytes: Optional[bytes] = None
     image_list: Optional[List[Image.Image]] = None
     image_detail: Literal["auto", "low", "high"] = "auto"

camel/models/open_source_model.py

@@ -95,7 +95,7 @@ class OpenSourceModel(BaseModelBackend):
 
         # Replace `model_config_dict` with only the params to be
         # passed to OpenAI API
-        self.model_config_dict = self.model_config_dict["api_params"].__dict__
+        self.model_config_dict = self.model_config_dict["api_params"]
 
     @property
     def token_counter(self) -> BaseTokenCounter:
@@ -152,7 +152,7 @@ class OpenSourceModel(BaseModelBackend):
                 ":obj:`model_path` or :obj:`server_url` missing."
             )
 
-        for param in self.model_config_dict["api_params"].__dict__:
+        for param in self.model_config_dict["api_params"]:
             if param not in OPENAI_API_PARAMS:
                 raise ValueError(
                     f"Unexpected argument `{param}` is "

camel/prompts/__init__.py

@@ -18,7 +18,11 @@ from .evaluation import EvaluationPromptTemplateDict
 from .generate_text_embedding_data import (
     GenerateTextEmbeddingDataPromptTemplateDict,
 )
+from .image_craft import ImageCraftPromptTemplateDict
 from .misalignment import MisalignmentPromptTemplateDict
+from .multi_condition_image_craft import (
+    MultiConditionImageCraftPromptTemplateDict,
+)
 from .object_recognition import ObjectRecognitionPromptTemplateDict
 from .prompt_templates import PromptTemplateGenerator
 from .role_description_prompt_template import RoleDescriptionPromptTemplateDict
@@ -42,5 +46,8 @@ __all__ = [
     'SolutionExtractionPromptTemplateDict',
     'GenerateTextEmbeddingDataPromptTemplateDict',
     'ObjectRecognitionPromptTemplateDict',
+    'ImageCraftPromptTemplateDict',
+    'MultiConditionImageCraftPromptTemplateDict',
+    'DescriptionVideoPromptTemplateDict',
     'VideoDescriptionPromptTemplateDict',
 ]

camel/prompts/image_craft.py

@@ -0,0 +1,34 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Any
+
+from camel.prompts import TextPrompt, TextPromptDict
+from camel.types import RoleType
+
+
+class ImageCraftPromptTemplateDict(TextPromptDict):
+    ASSISTANT_PROMPT = TextPrompt(
+        """You are tasked with creating an original image based on
+        the provided descriptive captions. Use your imagination
+        and artistic skills to visualize and draw the images and
+        explain your thought process."""
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.update(
+            {
+                RoleType.ASSISTANT: self.ASSISTANT_PROMPT,
+            }
+        )

camel/prompts/multi_condition_image_craft.py

@@ -0,0 +1,34 @@
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
+from typing import Any
+
+from camel.prompts import TextPrompt, TextPromptDict
+from camel.types import RoleType
+
+
+class MultiConditionImageCraftPromptTemplateDict(TextPromptDict):
+    ASSISTANT_PROMPT = TextPrompt(
+        """You are tasked with creating an image based on
+        the provided text and images conditions. Please use your
+        imagination and artistic capabilities to visualize and
+        draw the images and explain what you are thinking about."""
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.update(
+            {
+                RoleType.ASSISTANT: self.ASSISTANT_PROMPT,
+            }
+        )

camel/prompts/task_prompt_template.py

@@ -24,7 +24,11 @@ from camel.prompts.evaluation import (
 from camel.prompts.generate_text_embedding_data import (
     GenerateTextEmbeddingDataPromptTemplateDict,
 )
+from camel.prompts.image_craft import ImageCraftPromptTemplateDict
 from camel.prompts.misalignment import MisalignmentPromptTemplateDict
+from camel.prompts.multi_condition_image_craft import (
+    MultiConditionImageCraftPromptTemplateDict,
+)
 from camel.prompts.object_recognition import (
     ObjectRecognitionPromptTemplateDict,
 )
@@ -64,6 +68,8 @@ class TaskPromptTemplateDict(Dict[Any, TextPromptDict]):
                 TaskType.ROLE_DESCRIPTION: RoleDescriptionPromptTemplateDict(),
                 TaskType.OBJECT_RECOGNITION: ObjectRecognitionPromptTemplateDict(),  # noqa: E501
                 TaskType.GENERATE_TEXT_EMBEDDING_DATA: GenerateTextEmbeddingDataPromptTemplateDict(),  # noqa: E501
+                TaskType.IMAGE_CRAFT: ImageCraftPromptTemplateDict(),
+                TaskType.MULTI_CONDITION_IMAGE_CRAFT: MultiConditionImageCraftPromptTemplateDict(),  # noqa: E501
                 TaskType.VIDEO_DESCRIPTION: VideoDescriptionPromptTemplateDict(),  # noqa: E501
             }
         )

camel/responses/agent_responses.py

@@ -11,14 +11,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
-from dataclasses import dataclass
 from typing import Any, Dict, List
 
+from pydantic import BaseModel, ConfigDict
+
 from camel.messages import BaseMessage
 
 
-@dataclass(frozen=True)
-class ChatAgentResponse:
+class ChatAgentResponse(BaseModel):
     r"""Response of a ChatAgent.
 
     Attributes:
@@ -31,6 +31,7 @@ class ChatAgentResponse:
         info (Dict[str, Any]): Extra information about the chat message.
     """
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
     msgs: List[BaseMessage]
     terminated: bool
     info: Dict[str, Any]

camel/retrievers/auto_retriever.py

@@ -235,8 +235,6 @@ class AutoRetriever:
             else content_input_paths
         )
 
-        vr = VectorRetriever()
-
         all_retrieved_info = []
         for content_input_path in content_input_paths:
             # Generate a valid collection name

camel/societies/babyagi_playing.py

@@ -270,10 +270,12 @@ class BabyAGI:
                 "All tasks are solved"
             )
             return ChatAgentResponse(
-                [assistant_msg], terminated, assistant_response.info
+                msgs=[assistant_msg],
+                terminated=terminated,
+                info=assistant_response.info,
             )
         return ChatAgentResponse(
-            [assistant_msg],
-            assistant_response.terminated,
-            assistant_response.info,
+            msgs=[assistant_msg],
+            terminated=assistant_response.terminated,
+            info=assistant_response.info,
         )

camel/societies/role_playing.py

@@ -478,9 +478,11 @@ class RolePlaying:
         user_response = self.user_agent.step(assistant_msg)
         if user_response.terminated or user_response.msgs is None:
             return (
-                ChatAgentResponse([], False, {}),
+                ChatAgentResponse(msgs=[], terminated=False, info={}),
                 ChatAgentResponse(
-                    [], user_response.terminated, user_response.info
+                    msgs=[],
+                    terminated=user_response.terminated,
+                    info=user_response.info,
                 ),
             )
         user_msg = self._reduce_message_options(user_response.msgs)
@@ -490,20 +492,26 @@ class RolePlaying:
         if assistant_response.terminated or assistant_response.msgs is None:
             return (
                 ChatAgentResponse(
-                    [], assistant_response.terminated, assistant_response.info
+                    msgs=[],
+                    terminated=assistant_response.terminated,
+                    info=assistant_response.info,
+                ),
+                ChatAgentResponse(
+                    msgs=[user_msg], terminated=False, info=user_response.info
                 ),
-                ChatAgentResponse([user_msg], False, user_response.info),
             )
         assistant_msg = self._reduce_message_options(assistant_response.msgs)
         self.assistant_agent.record_message(assistant_msg)
 
         return (
             ChatAgentResponse(
-                [assistant_msg],
-                assistant_response.terminated,
-                assistant_response.info,
+                msgs=[assistant_msg],
+                terminated=assistant_response.terminated,
+                info=assistant_response.info,
             ),
             ChatAgentResponse(
-                [user_msg], user_response.terminated, user_response.info
+                msgs=[user_msg],
+                terminated=user_response.terminated,
+                info=user_response.info,
             ),
         )

camel/storages/graph_storages/graph_element.py

@@ -13,17 +13,17 @@
 # =========== Copyright 2023 @ CAMEL-AI.org. All Rights Reserved. ===========
 from __future__ import annotations
 
-from dataclasses import dataclass, field
 from typing import List, Union
 
+from pydantic import BaseModel, ConfigDict, Field
+
 try:
     from unstructured.documents.elements import Element
 except ImportError:
     Element = None
 
 
-@dataclass
-class Node:
+class Node(BaseModel):
     r"""Represents a node in a graph with associated properties.
 
     Attributes:
@@ -35,11 +35,10 @@ class Node:
 
     id: Union[str, int]
     type: str = "Node"
-    properties: dict = field(default_factory=dict)
+    properties: dict = Field(default_factory=dict)
 
 
-@dataclass
-class Relationship:
+class Relationship(BaseModel):
     r"""Represents a directed relationship between two nodes in a graph.
 
     Attributes:
@@ -53,24 +52,21 @@ class Relationship:
     subj: Node
     obj: Node
     type: str = "Relationship"
-    properties: dict = field(default_factory=dict)
+    properties: dict = Field(default_factory=dict)
 
 
-@dataclass
-class GraphElement:
+class GraphElement(BaseModel):
     r"""A graph element with lists of nodes and relationships.
 
     Attributes:
         nodes (List[Node]): A list of nodes in the graph.
         relationships (List[Relationship]): A list of relationships in the
-        graph.
+            graph.
         source (Element): The element from which the graph information is
-        derived.
+            derived.
     """
 
-    # Allow arbitrary types for Element
-    class Config:
-        arbitrary_types_allowed = True
+    model_config = ConfigDict(arbitrary_types_allowed=True)
 
     nodes: List[Node]
     relationships: List[Relationship]