camel-ai 0.2.72a10__py3-none-any.whl → 0.2.73a0__py3-none-any.whl

camel/__init__.py

@@ -14,7 +14,7 @@
 
 from camel.logger import disable_logging, enable_logging, set_log_level
 
-__version__ = '0.2.72a10'
+__version__ = '0.2.73a0'
 
 __all__ = [
     '__version__',

camel/agents/chat_agent.py

@@ -74,7 +74,7 @@ from camel.models import (
 from camel.prompts import TextPrompt
 from camel.responses import ChatAgentResponse
 from camel.storages import JsonStorage
-from camel.toolkits import FunctionTool
+from camel.toolkits import FunctionTool, RegisteredAgentToolkit
 from camel.types import (
     ChatCompletion,
     ChatCompletionChunk,
@@ -89,7 +89,6 @@ from camel.utils import (
     model_from_json_schema,
 )
 from camel.utils.commons import dependencies_required
-from camel.utils.tool_result import ToolResult
 
 if TYPE_CHECKING:
     from camel.terminators import ResponseTerminator
@@ -348,6 +347,13 @@ class ChatAgent(BaseAgent):
         tools (Optional[List[Union[FunctionTool, Callable]]], optional): List
             of available :obj:`FunctionTool` or :obj:`Callable`. (default:
             :obj:`None`)
+        toolkits_to_register_agent (Optional[List[RegisteredAgentToolkit]],
+            optional): List of toolkit instances that inherit from
+            :obj:`RegisteredAgentToolkit`. The agent will register itself with
+            these toolkits, allowing them to access the agent instance. Note:
+            This does NOT add the toolkit's tools to the agent. To use tools
+            from these toolkits, pass them explicitly via the `tools`
+            parameter. (default: :obj:`None`)
         external_tools (Optional[List[Union[FunctionTool, Callable,
             Dict[str, Any]]]], optional): List of external tools
             (:obj:`FunctionTool` or :obj:`Callable` or :obj:`Dict[str, Any]`)
@@ -405,6 +411,9 @@ class ChatAgent(BaseAgent):
         token_limit: Optional[int] = None,
         output_language: Optional[str] = None,
         tools: Optional[List[Union[FunctionTool, Callable]]] = None,
+        toolkits_to_register_agent: Optional[
+            List[RegisteredAgentToolkit]
+        ] = None,
         external_tools: Optional[
             List[Union[FunctionTool, Callable, Dict[str, Any]]]
         ] = None,
@@ -438,7 +447,7 @@ class ChatAgent(BaseAgent):
             token_limit or self.model_backend.token_limit,
         )
 
-        self.memory: AgentMemory = memory or ChatHistoryMemory(
+        self._memory: AgentMemory = memory or ChatHistoryMemory(
             context_creator,
             window_size=message_window_size,
             agent_id=self.agent_id,
@@ -446,7 +455,7 @@ class ChatAgent(BaseAgent):
 
         # So we don't have to pass agent_id when we define memory
         if memory is not None:
-            memory.agent_id = self.agent_id
+            self._memory.agent_id = self.agent_id
 
         # Set up system message and initialize messages
         self._original_system_message = (
@@ -479,6 +488,12 @@ class ChatAgent(BaseAgent):
             ]
         }
 
+        # Register agent with toolkits that have RegisteredAgentToolkit mixin
+        if toolkits_to_register_agent:
+            for toolkit in toolkits_to_register_agent:
+                if isinstance(toolkit, RegisteredAgentToolkit):
+                    toolkit.register_agent(self)
+
         self._external_tool_schemas = {
             tool_schema["function"]["name"]: tool_schema
             for tool_schema in [
@@ -494,9 +509,6 @@ class ChatAgent(BaseAgent):
         self.tool_execution_timeout = tool_execution_timeout
         self.mask_tool_output = mask_tool_output
         self._secure_result_store: Dict[str, Any] = {}
-        self._pending_images: List[str] = []
-        self._image_retry_count: Dict[str, int] = {}
-        # Store images to attach to next user message
         self.pause_event = pause_event
         self.prune_tool_calls_from_memory = prune_tool_calls_from_memory
 
@@ -504,8 +516,6 @@ class ChatAgent(BaseAgent):
         r"""Resets the :obj:`ChatAgent` to its initial state."""
         self.terminated = False
         self.init_messages()
-        self._pending_images = []
-        self._image_retry_count = {}
         for terminator in self.response_terminators:
             terminator.reset()
 
@@ -670,6 +680,25 @@ class ChatAgent(BaseAgent):
         )
         self.init_messages()
 
+    @property
+    def memory(self) -> AgentMemory:
+        r"""Returns the agent memory."""
+        return self._memory
+
+    @memory.setter
+    def memory(self, value: AgentMemory) -> None:
+        r"""Set the agent memory.
+
+        When setting a new memory, the system message is automatically
+        re-added to ensure it's not lost.
+
+        Args:
+            value (AgentMemory): The new agent memory to use.
+        """
+        self._memory = value
+        # Ensure the new memory has the system message
+        self.init_messages()
+
     def _get_full_tool_schemas(self) -> List[Dict[str, Any]]:
         r"""Returns a list of tool schemas of all tools, including internal
         and external tools.
@@ -1367,16 +1396,6 @@ class ChatAgent(BaseAgent):
                 role_name="User", content=input_message
             )
 
-        # Attach any pending images from previous tool calls
-        image_list = self._process_pending_images()
-        if image_list:
-            # Create new message with images attached
-            input_message = BaseMessage.make_user_message(
-                role_name="User",
-                content=input_message.content,
-                image_list=image_list,
-            )
-
         # Add user input to memory
         self.update_memory(input_message, OpenAIBackendRole.USER)
 
@@ -1571,16 +1590,6 @@ class ChatAgent(BaseAgent):
                 role_name="User", content=input_message
             )
 
-        # Attach any pending images from previous tool calls
-        image_list = self._process_pending_images()
-        if image_list:
-            # Create new message with images attached
-            input_message = BaseMessage.make_user_message(
-                role_name="User",
-                content=input_message.content,
-                image_list=image_list,
-            )
-
         self.update_memory(input_message, OpenAIBackendRole.USER)
 
         tool_call_records: List[ToolCallingRecord] = []
@@ -1634,7 +1643,6 @@ class ChatAgent(BaseAgent):
 
             if tool_call_requests := response.tool_call_requests:
                 # Process all tool calls
-                new_images_from_tools = []
                 for tool_call_request in tool_call_requests:
                     if (
                         tool_call_request.tool_name
@@ -1654,72 +1662,10 @@ class ChatAgent(BaseAgent):
                         )
                         tool_call_records.append(tool_call_record)
 
-                        # Check if this tool call produced images
-                        if (
-                            hasattr(tool_call_record, 'images')
-                            and tool_call_record.images
-                        ):
-                            new_images_from_tools.extend(
-                                tool_call_record.images
-                            )
-
                 # If we found an external tool call, break the loop
                 if external_tool_call_requests:
                     break
 
-                # If tools produced images
-                # send them to the model as a user message
-                if new_images_from_tools:
-                    # Convert base64 images to PIL Images
-                    image_list = []
-                    for img_data in new_images_from_tools:
-                        try:
-                            import base64
-                            import io
-
-                            from PIL import Image
-
-                            # Extract base64 data from data URL format
-                            if img_data.startswith("data:image"):
-                                # Format:
-                                # "data:image/png;base64,iVBORw0KGgo..."
-                                base64_data = img_data.split(',', 1)[1]
-                            else:
-                                # Raw base64 data
-                                base64_data = img_data
-
-                            # Decode and create PIL Image
-                            image_bytes = base64.b64decode(base64_data)
-                            pil_image = Image.open(io.BytesIO(image_bytes))
-                            # Convert to ensure proper
-                            # Image.Image type for compatibility
-                            pil_image_tool_result: Image.Image = (
-                                pil_image.convert('RGB')
-                            )
-                            image_list.append(pil_image_tool_result)
-
-                        except Exception as e:
-                            logger.warning(
-                                f"Failed to convert "
-                                f"base64 image to PIL for immediate use: {e}"
-                            )
-                            continue
-
-                    # If we have valid images
-                    # create a user message with images
-                    if image_list:
-                        # Create a user message with images
-                        # to provide visual context immediately
-                        image_message = BaseMessage.make_user_message(
-                            role_name="User",
-                            content="[Visual content from tool execution - please analyze and continue]",  # noqa: E501
-                            image_list=image_list,
-                        )
-
-                        self.update_memory(
-                            image_message, OpenAIBackendRole.USER
-                        )
-
                 if (
                     self.max_iteration is not None
                     and iteration_count >= self.max_iteration
@@ -1810,69 +1756,6 @@ class ChatAgent(BaseAgent):
             info=info,
         )
 
-    def _process_pending_images(self) -> List:
-        r"""Process pending images with retry logic and return PIL Image list.
-
-        Returns:
-            List: List of successfully converted PIL Images.
-        """
-        if not self._pending_images:
-            return []
-
-        image_list = []
-        successfully_processed = []
-        failed_images = []
-
-        for img_data in self._pending_images:
-            # Track retry count
-            retry_count = self._image_retry_count.get(img_data, 0)
-
-            # Remove images that have failed too many times (max 3 attempts)
-            if retry_count >= 3:
-                failed_images.append(img_data)
-                logger.warning(
-                    f"Removing image after {retry_count} failed attempts"
-                )
-                continue
-
-            try:
-                import base64
-                import io
-
-                from PIL import Image
-
-                # Extract base64 data from data URL format
-                if img_data.startswith("data:image"):
-                    # Format: "data:image/png;base64,iVBORw0KGgo..."
-                    base64_data = img_data.split(',', 1)[1]
-                else:
-                    # Raw base64 data
-                    base64_data = img_data
-
-                # Decode and create PIL Image
-                image_bytes = base64.b64decode(base64_data)
-                pil_image = Image.open(io.BytesIO(image_bytes))
-                pil_image_converted: Image.Image = pil_image.convert('RGB')
-                image_list.append(pil_image_converted)
-                successfully_processed.append(img_data)
-
-            except Exception as e:
-                # Increment retry count for failed conversion
-                self._image_retry_count[img_data] = retry_count + 1
-                logger.warning(
-                    f"Failed to convert base64 image to PIL "
-                    f"(attempt {retry_count + 1}/3): {e}"
-                )
-                continue
-
-        # Clean up processed and failed images
-        for img in successfully_processed + failed_images:
-            self._pending_images.remove(img)
-            # Clean up retry count for processed/removed images
-            self._image_retry_count.pop(img, None)
-
-        return image_list
-
     def _record_final_output(self, output_messages: List[BaseMessage]) -> None:
         r"""Log final messages or warnings about multiple responses."""
         if len(output_messages) == 1:
@@ -1883,61 +1766,6 @@ class ChatAgent(BaseAgent):
                 "selected message manually using `record_message()`."
             )
 
-    def _is_vision_error(self, exc: Exception) -> bool:
-        r"""Check if the exception is likely related to vision/image is not
-        supported by the model."""
-        # TODO: more robust vision error detection
-        error_msg = str(exc).lower()
-        vision_keywords = [
-            'vision',
-            'image',
-            'multimodal',
-            'unsupported',
-            'invalid content type',
-            'image_url',
-            'visual',
-        ]
-        return any(keyword in error_msg for keyword in vision_keywords)
-
-    def _has_images(self, messages: List[OpenAIMessage]) -> bool:
-        r"""Check if any message contains images."""
-        for msg in messages:
-            content = msg.get('content')
-            if isinstance(content, list):
-                for item in content:
-                    if (
-                        isinstance(item, dict)
-                        and item.get('type') == 'image_url'
-                    ):
-                        return True
-        return False
-
-    def _strip_images_from_messages(
-        self, messages: List[OpenAIMessage]
-    ) -> List[OpenAIMessage]:
-        r"""Remove images from messages, keeping only text content."""
-        stripped_messages = []
-        for msg in messages:
-            content = msg.get('content')
-            if isinstance(content, list):
-                # Extract only text content from multimodal messages
-                text_content = ""
-                for item in content:
-                    if isinstance(item, dict) and item.get('type') == 'text':
-                        text_content += item.get('text', '')
-
-                # Create new message with only text content
-                new_msg = msg.copy()
-                new_msg['content'] = (
-                    text_content
-                    or "[Image content removed - model doesn't support vision]"
-                )
-                stripped_messages.append(new_msg)
-            else:
-                # Regular text message, keep as is
-                stripped_messages.append(msg)
-        return stripped_messages
-
     @observe()
     def _get_model_response(
         self,
@@ -1971,35 +1799,13 @@ class ChatAgent(BaseAgent):
                 openai_messages, response_format, tool_schemas or None
             )
         except Exception as exc:
-            # Try again without images if the error might be vision-related
-            if self._is_vision_error(exc) and self._has_images(
-                openai_messages
-            ):
-                logger.warning(
-                    "Model appears to not support vision."
-                    "Retrying without images."
-                )
-                try:
-                    stripped_messages = self._strip_images_from_messages(
-                        openai_messages
-                    )
-                    response = self.model_backend.run(
-                        stripped_messages,
-                        response_format,
-                        tool_schemas or None,
-                    )
-                except Exception:
-                    pass  # Fall through to original error handling
-
-            if not response:
-                logger.error(
-                    f"An error occurred while running model "
-                    f"iteration {current_iteration}, "
-                    f"{self.model_backend.model_type}, "
-                    f"index: {self.model_backend.current_model_index}",
-                    exc_info=exc,
-                )
-                error_info = str(exc)
+            logger.error(
+                f"An error occurred while running model "
+                f"{self.model_backend.model_type}, "
+                f"index: {self.model_backend.current_model_index}",
+                exc_info=exc,
+            )
+            error_info = str(exc)
 
         if not response and self.model_backend.num_models > 1:
             raise ModelProcessingError(
@@ -2060,33 +1866,13 @@ class ChatAgent(BaseAgent):
                 openai_messages, response_format, tool_schemas or None
             )
         except Exception as exc:
-            # Try again without images if the error might be vision-related
-            if self._is_vision_error(exc) and self._has_images(
-                openai_messages
-            ):
-                logger.warning(
-                    "Model appears to not support vision. Retrying without images."  # noqa: E501
-                )
-                try:
-                    stripped_messages = self._strip_images_from_messages(
-                        openai_messages
-                    )
-                    response = await self.model_backend.arun(
-                        stripped_messages,
-                        response_format,
-                        tool_schemas or None,
-                    )
-                except Exception:
-                    pass  # Fall through to original error handling
-
-            if not response:
-                logger.error(
-                    f"An error occurred while running model "
-                    f"{self.model_backend.model_type}, "
-                    f"index: {self.model_backend.current_model_index}",
-                    exc_info=exc,
-                )
-                error_info = str(exc)
+            logger.error(
+                f"An error occurred while running model "
+                f"{self.model_backend.model_type}, "
+                f"index: {self.model_backend.current_model_index}",
+                exc_info=exc,
+            )
+            error_info = str(exc)
 
         if not response and self.model_backend.num_models > 1:
             raise ModelProcessingError(
@@ -2457,26 +2243,9 @@ class ChatAgent(BaseAgent):
             mask_flag = False
             logger.warning(f"{error_msg} with result: {result}")
 
-        # Check if result is a ToolResult with images
-        images_to_attach = None
-        if isinstance(result, ToolResult):
-            images_to_attach = result.images
-            result = str(result)  # Use string representation for storage
-
-        tool_record = self._record_tool_calling(
+        return self._record_tool_calling(
             func_name, args, result, tool_call_id, mask_output=mask_flag
         )
-        logger.info(f"Tool calling record:\n{tool_record}")
-
-        # Store images for later attachment to next user message
-        if images_to_attach:
-            tool_record.images = images_to_attach
-            # Add images with duplicate prevention
-            for img in images_to_attach:
-                if img not in self._pending_images:
-                    self._pending_images.append(img)
-
-        return tool_record
 
     async def _aexecute_tool(
         self,
@@ -2517,26 +2286,7 @@ class ChatAgent(BaseAgent):
             error_msg = f"Error executing async tool '{func_name}': {e!s}"
             result = f"Tool execution failed: {error_msg}"
             logging.warning(error_msg)
-
-        # Check if result is a ToolResult with images
-        images_to_attach = None
-        if isinstance(result, ToolResult):
-            images_to_attach = result.images
-            result = str(result)  # Use string representation for storage
-
-        tool_record = self._record_tool_calling(
-            func_name, args, result, tool_call_id
-        )
-
-        # Store images for later attachment to next user message
-        if images_to_attach:
-            tool_record.images = images_to_attach
-            # Add images with duplicate prevention
-            for img in images_to_attach:
-                if img not in self._pending_images:
-                    self._pending_images.append(img)
-
-        return tool_record
+        return self._record_tool_calling(func_name, args, result, tool_call_id)
 
     def _record_tool_calling(
         self,
@@ -3802,6 +3552,9 @@ class ChatAgent(BaseAgent):
         # To avoid duplicated system memory.
         system_message = None if with_memory else self._original_system_message
 
+        # Clone tools and collect toolkits that need registration
+        cloned_tools, toolkits_to_register = self._clone_tools()
+
         new_agent = ChatAgent(
             system_message=system_message,
             model=self.model_backend.models,  # Pass the existing model_backend
@@ -3811,7 +3564,8 @@ class ChatAgent(BaseAgent):
                 self.memory.get_context_creator(), "token_limit", None
             ),
             output_language=self._output_language,
-            tools=self._clone_tools(),
+            tools=cloned_tools,
+            toolkits_to_register_agent=toolkits_to_register,
             external_tools=[
                 schema for schema in self._external_tool_schemas.values()
             ],
@@ -3836,55 +3590,76 @@ class ChatAgent(BaseAgent):
 
         return new_agent
 
-    def _clone_tools(self) -> List[Union[FunctionTool, Callable]]:
-        r"""Clone tools for new agent instance,
-        handling stateful toolkits properly."""
+    def _clone_tools(
+        self,
+    ) -> Tuple[
+        List[Union[FunctionTool, Callable]], List[RegisteredAgentToolkit]
+    ]:
+        r"""Clone tools and return toolkits that need agent registration.
+
+        This method handles stateful toolkits by cloning them if they have
+        a clone_for_new_session method, and collecting RegisteredAgentToolkit
+        instances for later registration.
+
+        Returns:
+            Tuple containing:
+            - List of cloned tools/functions
+            - List of RegisteredAgentToolkit instances need registration
+        """
         cloned_tools = []
-        hybrid_browser_toolkits = {}  # Cache for created toolkits
+        toolkits_to_register = []
+        cloned_toolkits = {}
+        # Cache for cloned toolkits by original toolkit id
 
         for tool in self._internal_tools.values():
-            # Check if this is a HybridBrowserToolkit method
-            if (
-                hasattr(tool.func, '__self__')
-                and tool.func.__self__.__class__.__name__
-                == 'HybridBrowserToolkit'
-            ):
+            # Check if this tool is a method bound to a toolkit instance
+            if hasattr(tool.func, '__self__'):
                 toolkit_instance = tool.func.__self__
                 toolkit_id = id(toolkit_instance)
 
-                # Check if we already created a clone for this toolkit
-                if toolkit_id not in hybrid_browser_toolkits:
-                    try:
-                        import uuid
+                if toolkit_id not in cloned_toolkits:
+                    # Check if the toolkit has a clone method
+                    if hasattr(toolkit_instance, 'clone_for_new_session'):
+                        try:
+                            import uuid
 
-                        new_session_id = str(uuid.uuid4())[:8]
-                        new_toolkit = toolkit_instance.clone_for_new_session(
-                            new_session_id
-                        )
-                        hybrid_browser_toolkits[toolkit_id] = new_toolkit
-                    except Exception as e:
-                        logger.warning(
-                            f"Failed to clone HybridBrowserToolkit: {e}"
-                        )
-                        # Fallback to original function
-                        cloned_tools.append(tool.func)
-                        continue
+                            new_session_id = str(uuid.uuid4())[:8]
+                            new_toolkit = (
+                                toolkit_instance.clone_for_new_session(
+                                    new_session_id
+                                )
+                            )
+
+                            # If this is a RegisteredAgentToolkit,
+                            # add it to registration list
+                            if isinstance(new_toolkit, RegisteredAgentToolkit):
+                                toolkits_to_register.append(new_toolkit)
+
+                            cloned_toolkits[toolkit_id] = new_toolkit
+                        except Exception as e:
+                            logger.warning(
+                                f"Failed to clone toolkit {toolkit_instance.__class__.__name__}: {e}"  # noqa:E501
+                            )
+                            # Use original toolkit if cloning fails
+                            cloned_toolkits[toolkit_id] = toolkit_instance
+                    else:
+                        # Toolkit doesn't support cloning, use original
+                        cloned_toolkits[toolkit_id] = toolkit_instance
 
-                # Get the corresponding method from the cloned toolkit
-                new_toolkit = hybrid_browser_toolkits[toolkit_id]
+                # Get the method from the cloned (or original) toolkit
+                toolkit = cloned_toolkits[toolkit_id]
                 method_name = tool.func.__name__
-                if hasattr(new_toolkit, method_name):
-                    new_method = getattr(new_toolkit, method_name)
+                if hasattr(toolkit, method_name):
+                    new_method = getattr(toolkit, method_name)
                     cloned_tools.append(new_method)
                 else:
                     # Fallback to original function
                     cloned_tools.append(tool.func)
             else:
-                # Regular tool or other stateless toolkit
-                # just use the original function
+                # Not a toolkit method, just use the original function
                 cloned_tools.append(tool.func)
 
-        return cloned_tools
+        return cloned_tools, toolkits_to_register
 
     def __repr__(self) -> str:
         r"""Returns a string representation of the :obj:`ChatAgent`.

camel/memories/agent_memories.py

@@ -101,30 +101,31 @@ class ChatHistoryMemory(AgentMemory):
         if not record_dicts:
             return
 
-        # Filter out tool-related messages
-        cleaned_records = []
-        for record in record_dicts:
+        # Track indices to remove (reverse order for efficient deletion)
+        indices_to_remove = []
+
+        # Identify indices of tool-related messages
+        for i, record in enumerate(record_dicts):
             role = record.get('role_at_backend')
 
-            # Skip FUNCTION messages
+            # Mark FUNCTION messages for removal
             if role == OpenAIBackendRole.FUNCTION.value:
-                continue
-
-            # Skip TOOL messages
-            if role == OpenAIBackendRole.TOOL.value:
-                continue
-
-            # Skip ASSISTANT messages with tool_calls
-            if role == OpenAIBackendRole.ASSISTANT.value:
+                indices_to_remove.append(i)
+            # Mark TOOL messages for removal
+            elif role == OpenAIBackendRole.TOOL.value:
+                indices_to_remove.append(i)
+            # Mark ASSISTANT messages with tool_calls for removal
+            elif role == OpenAIBackendRole.ASSISTANT.value:
                 meta_dict = record.get('meta_dict', {})
                 if meta_dict and 'tool_calls' in meta_dict:
-                    continue
+                    indices_to_remove.append(i)
 
-            # Keep all other messages
-            cleaned_records.append(record)
+        # Remove records in-place
+        for i in reversed(indices_to_remove):
+            del record_dicts[i]
 
-        # Save the cleaned records back to storage
-        self._chat_history_block.storage.save(cleaned_records)
+        # Save the modified records back to storage
+        self._chat_history_block.storage.save(record_dicts)
 
 
 class VectorDBMemory(AgentMemory):