camel-ai 0.2.10__py3-none-any.whl → 0.2.12__py3-none-any.whl

camel/__init__.py

@@ -12,9 +12,14 @@
 # limitations under the License.
 # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
 
-__version__ = '0.2.10'
+from camel.logger import disable_logging, enable_logging, set_log_level
+
+__version__ = '0.2.12'
 
 __all__ = [
     '__version__',
     'camel',
+    'disable_logging',
+    'enable_logging',
+    'set_log_level',
 ]

camel/agents/chat_agent.py

@@ -21,6 +21,7 @@ from collections import defaultdict
 from typing import (
     TYPE_CHECKING,
     Any,
+    Callable,
     Dict,
     List,
     Optional,
@@ -41,7 +42,12 @@ from camel.memories import (
     ScoreBasedContextCreator,
 )
 from camel.messages import BaseMessage, FunctionCallingMessage, OpenAIMessage
-from camel.models import BaseModelBackend, ModelFactory
+from camel.models import (
+    BaseModelBackend,
+    ModelFactory,
+    ModelManager,
+    ModelProcessingError,
+)
 from camel.responses import ChatAgentResponse
 from camel.types import (
     ChatCompletion,
@@ -145,12 +151,16 @@ class ChatAgent(BaseAgent):
         response_terminators (List[ResponseTerminator], optional): List of
             :obj:`ResponseTerminator` bind to one chat agent.
             (default: :obj:`None`)
+        scheduling_strategy (str): name of function that defines how to select
+            the next model in ModelManager. (default: :str:`round_robin`)
     """
 
     def __init__(
         self,
         system_message: Optional[Union[BaseMessage, str]] = None,
-        model: Optional[BaseModelBackend] = None,
+        model: Optional[
+            Union[BaseModelBackend, List[BaseModelBackend]]
+        ] = None,
         memory: Optional[AgentMemory] = None,
         message_window_size: Optional[int] = None,
         token_limit: Optional[int] = None,
@@ -158,6 +168,7 @@ class ChatAgent(BaseAgent):
         tools: Optional[List[FunctionTool]] = None,
         external_tools: Optional[List[FunctionTool]] = None,
         response_terminators: Optional[List[ResponseTerminator]] = None,
+        scheduling_strategy: str = "round_robin",
     ) -> None:
         if isinstance(system_message, str):
             system_message = BaseMessage.make_assistant_message(
@@ -172,13 +183,14 @@ class ChatAgent(BaseAgent):
         self.role_type: RoleType = (
             getattr(system_message, 'role_type', None) or RoleType.ASSISTANT
         )
-        self.model_backend: BaseModelBackend = (
+        self.model_backend = ModelManager(
             model
             if model is not None
             else ModelFactory.create(
                 model_platform=ModelPlatformType.DEFAULT,
                 model_type=ModelType.DEFAULT,
-            )
+            ),
+            scheduling_strategy=scheduling_strategy,
         )
 
         self.model_type = self.model_backend.model_type
@@ -487,6 +499,13 @@ class ChatAgent(BaseAgent):
                 "the model configuration and in the ChatAgent step."
             )
 
+        original_model_dict = self.model_backend.model_config_dict
+        if response_format and self.model_type in {"gpt-4o", "gpt-4o-mini"}:
+            self.model_backend.model_config_dict = original_model_dict.copy()
+            self.model_backend.model_config_dict["response_format"] = (
+                response_format
+            )
+
         if isinstance(input_message, str):
             input_message = BaseMessage.make_user_message(
                 role_name='User', content=input_message
@@ -620,6 +639,7 @@ class ChatAgent(BaseAgent):
                 try:
                     openai_messages, num_tokens = self.memory.get_context()
                 except RuntimeError as e:
+                    self.model_backend.model_config_dict = original_model_dict
                     return self._step_token_exceed(
                         e.args[1], tool_call_records, "max_tokens_exceeded"
                     )
@@ -655,6 +675,8 @@ class ChatAgent(BaseAgent):
                         num_tokens,
                         tool_call_request,
                     )
+
+                    self.model_backend.model_config_dict = original_model_dict
                     return ChatAgentResponse(
                         msgs=output_messages,
                         terminated=self.terminated,
@@ -669,6 +691,7 @@ class ChatAgent(BaseAgent):
             if (
                 response_format is not None
                 and self.model_type.support_native_tool_calling
+                and self.model_type not in {"gpt-4o", "gpt-4o-mini"}
             ):
                 (
                     output_messages,
@@ -699,6 +722,7 @@ class ChatAgent(BaseAgent):
                     "to record the selected message manually."
                 )
 
+            self.model_backend.model_config_dict = original_model_dict
             return ChatAgentResponse(
                 msgs=output_messages, terminated=self.terminated, info=info
             )
@@ -918,7 +942,7 @@ class ChatAgent(BaseAgent):
             )
 
         for base_message_item in output_messages:
-            base_message_item.content = str(tool_call_record.result)
+            base_message_item.content = json.dumps(tool_call_record.result)
 
         # Recover the original tools
         self.func_dict = original_func_dict
@@ -945,8 +969,32 @@ class ChatAgent(BaseAgent):
         str,
     ]:
         r"""Internal function for agent step model response."""
+
+        response = None
         # Obtain the model's response
-        response = self.model_backend.run(openai_messages)
+        for _ in range(len(self.model_backend.models)):
+            try:
+                response = self.model_backend.run(openai_messages)
+                break
+            except Exception as exc:
+                logger.error(
+                    f"An error occurred while running model "
+                    f"{self.model_backend.model_type}, "
+                    f"index: {self.model_backend.current_model_index}",
+                    exc_info=exc,
+                )
+                continue
+        if not response:
+            raise ModelProcessingError(
+                "Unable to process messages: none of the provided models "
+                "run succesfully."
+            )
+
+        logger.info(
+            f"Model {self.model_backend.model_type}, "
+            f"index {self.model_backend.current_model_index}, "
+            f"processed these messages: {openai_messages}"
+        )
 
         if isinstance(response, ChatCompletion):
             output_messages, finish_reasons, usage_dict, response_id = (
@@ -1054,8 +1102,32 @@ class ChatAgent(BaseAgent):
                 role_type=self.role_type,
                 meta_dict=dict(),
                 content=choice.message.content or "",
+                parsed=getattr(choice.message, 'parsed', None),
             )
+            # Process log probabilities and append to the message meta information
+            if choice.logprobs is not None:
+                tokens_logprobs = choice.logprobs.content
+
+                if tokens_logprobs is not None:
+                    # Extract and structure logprob information
+                    logprobs_info = [
+                        {
+                            "token": token_logprob.token,
+                            "logprob": token_logprob.logprob,
+                            "top_logprobs": [
+                                (top_logprob.token, top_logprob.logprob)
+                                for top_logprob in token_logprob.top_logprobs
+                            ],
+                        }
+                        for token_logprob in tokens_logprobs
+                    ]
+                # Ensure meta_dict exists before adding logprobs info
+                if chat_message.meta_dict is None:
+                    chat_message.meta_dict = {}
+                chat_message.meta_dict["logprobs_info"] = logprobs_info
+            # Append the processed chat message to output
             output_messages.append(chat_message)
+
         finish_reasons = [
             str(choice.finish_reason) for choice in response.choices
         ]
@@ -1298,6 +1370,15 @@ class ChatAgent(BaseAgent):
         )
         return usage_dict
 
+    def add_model_scheduling_strategy(self, name: str, strategy_fn: Callable):
+        r"""Add a scheduling strategy method provided by user to ModelManger.
+
+        Args:
+            name (str): The name of the strategy.
+            strategy_fn (Callable): The scheduling strategy function.
+        """
+        self.model_backend.add_strategy(name, strategy_fn)
+
     def __repr__(self) -> str:
         r"""Returns a string representation of the :obj:`ChatAgent`.
 

camel/agents/deductive_reasoner_agent.py

@@ -15,11 +15,14 @@ import re
 from typing import Dict, List, Optional, Union
 
 from camel.agents.chat_agent import ChatAgent
+from camel.logger import get_logger
 from camel.messages import BaseMessage
 from camel.models import BaseModelBackend
 from camel.prompts import TextPrompt
 from camel.types import RoleType
 
+logger = get_logger(__name__)
+
 # AgentOps decorator setting
 try:
     import os
@@ -253,7 +256,7 @@ square brackets)
                 "Deduction failed. Error:\n" + f"{response.info}"
             )
         msg: BaseMessage = response.msg
-        print(f"Message content:\n{msg.content}")
+        logger.info(f"Message content:\n{msg.content}")
 
         # Extract the conditions from the message
         conditions_dict = {

camel/benchmarks/__init__.py

@@ -0,0 +1,18 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+from .base import BaseBenchmark
+from .gaia import DefaultGAIARetriever, GAIABenchmark
+
+__all__ = ["BaseBenchmark", "GAIABenchmark", "DefaultGAIARetriever"]

camel/benchmarks/base.py

@@ -0,0 +1,152 @@
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. =========
+
+import logging
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Any, Dict, List, Literal, Optional
+
+from camel.agents import ChatAgent
+
+logger = logging.getLogger(__name__)
+
+
+class BaseBenchmark(ABC):
+    r"""Base class for benchmarks.
+
+    Attributes:
+        name (str): Name of the benchmark.
+        data_dir (str): Path to the data directory.
+        save_to (str): Path to save the results.
+        processes (int): Number of processes to use for parallel
+            processing. :(default: :obj:`1`)
+    """
+
+    def __init__(
+        self, name: str, data_dir: str, save_to: str, processes: int = 1
+    ):
+        r"""Initialize the benchmark.
+
+        Args:
+            name (str): Name of the benchmark.
+            data_dir (str): Path to the data directory.
+            save_to (str): Path to save the results.
+            processes (int): Number of processes to use for parallel
+                processing. :(default: :obj:`1`)
+
+        """
+        self.name = name
+        self.data_dir = Path(data_dir)
+        self.processes = processes
+        self.save_to = save_to
+        if not self.data_dir.exists():
+            logger.info(
+                f"Data directory {data_dir} does not exist. Creating it."
+            )
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+        if not self.data_dir.is_dir():
+            raise NotADirectoryError(
+                f"Data directory {data_dir} is not a directory"
+            )
+        self._data: Dict[str, List[Dict[str, Any]]] = dict()
+        self._results: List[Dict[str, Any]] = []
+
+    @abstractmethod
+    def download(self) -> "BaseBenchmark":
+        r"""Download the benchmark data.
+
+        Returns:
+            BaseBenchmark: The benchmark instance.
+        """
+        pass
+
+    @abstractmethod
+    def load(self, force_download: bool = False) -> "BaseBenchmark":
+        r"""Load the benchmark data.
+
+        Args:
+            force_download (bool): Whether to force download the data.
+
+        Returns:
+            BaseBenchmark: The benchmark instance.
+        """
+        pass
+
+    @property
+    def train(self) -> List[Dict[str, Any]]:
+        r"""Get the training data.
+
+        Returns:
+            List[Dict[str, Any]]: The training data.
+        """
+        if not self._data:
+            logger.info("Data not loaded. Loading data.")
+            self.load()
+        return self._data["train"]
+
+    @property
+    def valid(self) -> List[Dict[str, Any]]:
+        r"""Get the validation data.
+
+        Returns:
+            List[Dict[str, Any]]: The validation data.
+        """
+        if not self._data:
+            logger.info("Data not loaded. Loading data.")
+            self.load()
+        return self._data["valid"]
+
+    @property
+    def test(self) -> List[Dict[str, Any]]:
+        r"""Get the test data.
+
+        Returns:
+            List[Dict[str, Any]]: The test data.
+        """
+        if not self._data:
+            logger.info("Data not loaded. Loading data.")
+            self.load()
+        return self._data["test"]
+
+    @abstractmethod
+    def run(
+        self,
+        agent: ChatAgent,
+        on: Literal["train", "valid", "test"],
+        randomize: bool = False,
+        subset: Optional[int] = None,
+        *args,
+        **kwargs,
+    ) -> "BaseBenchmark":
+        r"""Run the benchmark.
+
+        Args:
+            agent (ChatAgent): The chat agent.
+            on (str): The data split to run the benchmark on.
+            randomize (bool): Whether to randomize the data.
+            subset (int): The subset of the data to run the benchmark on.
+
+        Returns:
+            BaseBenchmark: The benchmark instance.
+        """
+        pass
+
+    @property
+    def results(self) -> List[Dict[str, Any]]:
+        r"""Get the results.
+
+        Returns:
+            List[Dict[str, Any]]: The results.
+        """
+        return self._results