autobyteus 1.1.5__py3-none-any.whl → 1.1.7__py3-none-any.whl

autobyteus/agent/context/agent_config.py

@@ -37,6 +37,7 @@ class AgentConfig:
                  system_prompt: Optional[str] = None,
                  tools: Optional[List['BaseTool']] = None,
                  auto_execute_tools: bool = True,
+                 use_xml_tool_format: bool = False,
                  input_processors: Optional[List['BaseAgentUserInputMessageProcessor']] = None,
                  llm_response_processors: Optional[List['BaseLLMResponseProcessor']] = None,
                  system_prompt_processors: Optional[List['BaseSystemPromptProcessor']] = None,
@@ -57,6 +58,8 @@ class AgentConfig:
                            llm_instance's config will be used as the base.
             tools: An optional list of pre-initialized tool instances (subclasses of BaseTool).
             auto_execute_tools: If True, the agent will execute tools without approval.
+            use_xml_tool_format: If True, forces the agent to use XML format for tool
+                                 definitions and parsing, overriding provider defaults.
             input_processors: A list of input processor instances.
             llm_response_processors: A list of LLM response processor instances.
             system_prompt_processors: A list of system prompt processor instances.
@@ -74,6 +77,7 @@ class AgentConfig:
         self.tools = tools or []
         self.workspace = workspace
         self.auto_execute_tools = auto_execute_tools
+        self.use_xml_tool_format = use_xml_tool_format
         self.input_processors = input_processors or []
         self.llm_response_processors = llm_response_processors if llm_response_processors is not None else list(self.DEFAULT_LLM_RESPONSE_PROCESSORS)
         self.system_prompt_processors = system_prompt_processors if system_prompt_processors is not None else list(self.DEFAULT_SYSTEM_PROMPT_PROCESSORS)
@@ -81,7 +85,7 @@ class AgentConfig:
         self.phase_hooks = phase_hooks or []
         self.initial_custom_data = initial_custom_data
 
-        logger.debug(f"AgentConfig created for name '{self.name}', role '{self.role}'.")
+        logger.debug(f"AgentConfig created for name '{self.name}', role '{self.role}'. XML tool format override: {self.use_xml_tool_format}")
 
     def copy(self) -> 'AgentConfig':
         """
@@ -98,6 +102,7 @@ class AgentConfig:
             system_prompt=self.system_prompt,
             tools=self.tools.copy(),  # Shallow copy the list, but reference the original tool instances
             auto_execute_tools=self.auto_execute_tools,
+            use_xml_tool_format=self.use_xml_tool_format,
             input_processors=self.input_processors.copy(), # Shallow copy the list
             llm_response_processors=self.llm_response_processors.copy(), # Shallow copy the list
             system_prompt_processors=self.system_prompt_processors.copy(), # Shallow copy the list

autobyteus/agent/context/agent_runtime_state.py

@@ -16,6 +16,7 @@ from autobyteus.agent.tool_invocation import ToolInvocation
 if TYPE_CHECKING:
     from autobyteus.agent.phases import AgentPhaseManager 
     from autobyteus.tools.base_tool import BaseTool 
+    from autobyteus.agent.tool_invocation import ToolInvocationTurn
 
 logger = logging.getLogger(__name__)
 
@@ -48,6 +49,9 @@ class AgentRuntimeState:
         self.pending_tool_approvals: Dict[str, ToolInvocation] = {}
         self.custom_data: Dict[str, Any] = custom_data or {}
         
+        # NEW: State for multi-tool call invocation turns, with a very explicit name.
+        self.active_multi_tool_call_turn: Optional['ToolInvocationTurn'] = None
+        
         self.processed_system_prompt: Optional[str] = None
         # self.final_llm_config_for_creation removed
         
@@ -83,7 +87,9 @@ class AgentRuntimeState:
         tools_status = f"{len(self.tool_instances)} Initialized" if self.tool_instances is not None else "Not Initialized"
         input_queues_status = "Initialized" if self.input_event_queues else "Not Initialized"
         # REMOVED output_queues_status from repr
+        active_turn_status = "Active" if self.active_multi_tool_call_turn else "Inactive"
         return (f"AgentRuntimeState(agent_id='{self.agent_id}', current_phase='{phase_repr}', "
                 f"llm_status='{llm_status}', tools_status='{tools_status}', "
                 f"input_queues_status='{input_queues_status}', "
-                f"pending_approvals={len(self.pending_tool_approvals)}, history_len={len(self.conversation_history)})")
+                f"pending_approvals={len(self.pending_tool_approvals)}, history_len={len(self.conversation_history)}, "
+                f"multi_tool_call_turn='{active_turn_status}')")

autobyteus/agent/handlers/llm_user_message_ready_event_handler.py

@@ -1,7 +1,7 @@
 # file: autobyteus/autobyteus/agent/handlers/llm_user_message_ready_event_handler.py
 import logging
 import traceback
-from typing import TYPE_CHECKING, cast, Optional
+from typing import TYPE_CHECKING, cast, Optional, List
 
 from autobyteus.agent.handlers.base_event_handler import AgentEventHandler
 from autobyteus.agent.events import LLMUserMessageReadyEvent, LLMCompleteResponseReceivedEvent 
@@ -53,6 +53,9 @@ class LLMUserMessageReadyEventHandler(AgentEventHandler):
         complete_response_text = ""
         complete_reasoning_text = ""
         token_usage: Optional[TokenUsage] = None
+        complete_image_urls: List[str] = []
+        complete_audio_urls: List[str] = []
+        complete_video_urls: List[str] = []
         
         notifier: Optional['AgentExternalEventNotifier'] = None
         if context.phase_manager:
@@ -72,9 +75,19 @@ class LLMUserMessageReadyEventHandler(AgentEventHandler):
                 if chunk_response.reasoning:
                     complete_reasoning_text += chunk_response.reasoning
 
-                if chunk_response.is_complete and chunk_response.usage:
-                    token_usage = chunk_response.usage
-                    logger.debug(f"Agent '{agent_id}' received final chunk with token usage: {token_usage}")
+                if chunk_response.is_complete:
+                    if chunk_response.usage:
+                        token_usage = chunk_response.usage
+                        logger.debug(f"Agent '{agent_id}' received final chunk with token usage: {token_usage}")
+                    if chunk_response.image_urls:
+                        complete_image_urls.extend(chunk_response.image_urls)
+                        logger.debug(f"Agent '{agent_id}' received final chunk with {len(chunk_response.image_urls)} image URLs.")
+                    if chunk_response.audio_urls:
+                        complete_audio_urls.extend(chunk_response.audio_urls)
+                        logger.debug(f"Agent '{agent_id}' received final chunk with {len(chunk_response.audio_urls)} audio URLs.")
+                    if chunk_response.video_urls:
+                        complete_video_urls.extend(chunk_response.video_urls)
+                        logger.debug(f"Agent '{agent_id}' received final chunk with {len(chunk_response.video_urls)} video URLs.")
 
                 if notifier:
                     try:
@@ -121,20 +134,30 @@ class LLMUserMessageReadyEventHandler(AgentEventHandler):
             logger.info(f"Agent '{agent_id}' enqueued LLMCompleteResponseReceivedEvent with error details from LLMUserMessageReadyEventHandler.")
             return 
 
-        # Add message to history with reasoning
+        # Add message to history with reasoning and multimodal data
         history_entry = {"role": "assistant", "content": complete_response_text}
         if complete_reasoning_text:
             history_entry["reasoning"] = complete_reasoning_text
+        if complete_image_urls:
+            history_entry["image_urls"] = complete_image_urls
+        if complete_audio_urls:
+            history_entry["audio_urls"] = complete_audio_urls
+        if complete_video_urls:
+            history_entry["video_urls"] = complete_video_urls
         context.state.add_message_to_history(history_entry)
         
-        # Create complete response with reasoning
+        # Create complete response with reasoning and multimodal data
         complete_response_obj = CompleteResponse(
             content=complete_response_text,
             reasoning=complete_reasoning_text,
-            usage=token_usage
+            usage=token_usage,
+            image_urls=complete_image_urls,
+            audio_urls=complete_audio_urls,
+            video_urls=complete_video_urls
         )
         llm_complete_event = LLMCompleteResponseReceivedEvent(
             complete_response=complete_response_obj
         )
         await context.input_event_queues.enqueue_internal_system_event(llm_complete_event)
         logger.info(f"Agent '{agent_id}' enqueued LLMCompleteResponseReceivedEvent from LLMUserMessageReadyEventHandler.")
+

autobyteus/agent/handlers/tool_result_event_handler.py

@@ -1,7 +1,7 @@
 # file: autobyteus/autobyteus/agent/handlers/tool_result_event_handler.py
 import logging
 import json 
-from typing import TYPE_CHECKING, Optional 
+from typing import TYPE_CHECKING, Optional, List
 
 from autobyteus.agent.handlers.base_event_handler import AgentEventHandler 
 from autobyteus.agent.events import ToolResultEvent, LLMUserMessageReadyEvent 
@@ -16,13 +16,60 @@ logger = logging.getLogger(__name__)
 
 class ToolResultEventHandler(AgentEventHandler):
     """
-    Handles ToolResultEvents by formatting the tool's output (or error)
-    as a new LLMUserMessage, emitting AGENT_DATA_TOOL_LOG event for this outcome,
-    and enqueuing an LLMUserMessageReadyEvent for further LLM processing.
+    Handles ToolResultEvents. It immediately processes and notifies for each
+    individual tool result. If a multi-tool call turn is active, it accumulates
+    these processed results until the turn is complete, then sends a single
+    aggregated message to the LLM.
     """
     def __init__(self):
         logger.info("ToolResultEventHandler initialized.")
 
+    async def _dispatch_aggregated_results_to_llm(self,
+                                                  processed_events: List[ToolResultEvent],
+                                                  context: 'AgentContext'):
+        """
+        Aggregates a list of PRE-PROCESSED tool results into a single message and
+        dispatches it to the LLM.
+        """
+        agent_id = context.agent_id
+        
+        # --- Aggregate results into a single message ---
+        aggregated_content_parts = []
+        for p_event in processed_events:
+            tool_invocation_id = p_event.tool_invocation_id if p_event.tool_invocation_id else 'N/A'
+            content_part: str
+            if p_event.error:
+                content_part = (
+                    f"Tool: {p_event.tool_name} (ID: {tool_invocation_id})\n"
+                    f"Status: Error\n"
+                    f"Details: {p_event.error}"
+                )
+            else:
+                try:
+                    result_str = json.dumps(p_event.result, indent=2) if not isinstance(p_event.result, str) else p_event.result
+                except TypeError: # pragma: no cover
+                    result_str = str(p_event.result)
+                content_part = (
+                    f"Tool: {p_event.tool_name} (ID: {tool_invocation_id})\n"
+                    f"Status: Success\n"
+                    f"Result:\n{result_str}" 
+                )
+            aggregated_content_parts.append(content_part)
+
+        final_content_for_llm = (
+            "The following tool executions have completed. Please analyze their results and decide the next course of action.\n\n"
+            + "\n\n---\n\n".join(aggregated_content_parts)
+        )
+        
+        logger.debug(f"Agent '{agent_id}' preparing aggregated message for LLM:\n---\n{final_content_for_llm}\n---")
+        llm_user_message = LLMUserMessage(content=final_content_for_llm)
+        
+        next_event = LLMUserMessageReadyEvent(llm_user_message=llm_user_message) 
+        await context.input_event_queues.enqueue_internal_system_event(next_event)
+        
+        logger.info(f"Agent '{agent_id}' enqueued LLMUserMessageReadyEvent with aggregated results from {len(processed_events)} tool(s).")
+
+
     async def handle(self,
                      event: ToolResultEvent,
                      context: 'AgentContext') -> None:
@@ -31,99 +78,64 @@ class ToolResultEventHandler(AgentEventHandler):
             return
 
         agent_id = context.agent_id
-        processed_event = event
+        notifier: Optional['AgentExternalEventNotifier'] = context.phase_manager.notifier if context.phase_manager else None
 
-        # --- New: Apply Tool Execution Result Processors ---
+        # --- Step 1: Immediately process the incoming event ---
+        processed_event = event
         processor_instances = context.config.tool_execution_result_processors
         if processor_instances:
-            processor_names = [p.get_name() for p in processor_instances]
-            logger.debug(f"Agent '{agent_id}': Applying tool execution result processors: {processor_names}")
             for processor_instance in processor_instances:
-                processor_name_for_log = "unknown"
+                if not isinstance(processor_instance, BaseToolExecutionResultProcessor):
+                    logger.error(f"Agent '{agent_id}': Invalid tool result processor type: {type(processor_instance)}. Skipping.")
+                    continue
                 try:
-                    if not isinstance(processor_instance, BaseToolExecutionResultProcessor):
-                        logger.error(f"Agent '{agent_id}': Invalid tool result processor type: {type(processor_instance)}. Skipping.")
-                        continue
-                    
-                    processor_name_for_log = processor_instance.get_name()
-                    logger.debug(f"Agent '{agent_id}': Applying tool result processor '{processor_name_for_log}'.")
-                    
-                    event_before_proc = processed_event
-                    processed_event = await processor_instance.process(event_before_proc, context)
-                    logger.info(f"Agent '{agent_id}': Tool result processor '{processor_name_for_log}' applied successfully.")
-
+                    processed_event = await processor_instance.process(processed_event, context)
                 except Exception as e:
-                    logger.error(f"Agent '{agent_id}': Error applying tool result processor '{processor_name_for_log}': {e}. "
-                                 f"Skipping and continuing with result from before this processor.", exc_info=True)
-                    processed_event = event_before_proc
-        # --- End New ---
-
-        tool_invocation_id = processed_event.tool_invocation_id if processed_event.tool_invocation_id else 'N/A'
-
-        logger.info(f"Agent '{agent_id}' handling processed ToolResultEvent from tool: '{processed_event.tool_name}' (Invocation ID: {tool_invocation_id}). Error: {processed_event.error is not None}")
+                    logger.error(f"Agent '{agent_id}': Error applying tool result processor '{processor_instance.get_name()}': {e}", exc_info=True)
         
-        notifier: Optional['AgentExternalEventNotifier'] = None
-        if context.phase_manager:
-            notifier = context.phase_manager.notifier
-        
-        if not notifier: # pragma: no cover
-            logger.error(f"Agent '{agent_id}': Notifier not available in ToolResultEventHandler. Tool result processing logs will not be emitted.")
-
-        if processed_event.error:
-            logger.debug(f"Agent '{agent_id}' tool '{processed_event.tool_name}' (ID: {tool_invocation_id}) raw error details: {processed_event.error}")
-        else:
+        # --- Step 2: Immediately notify the result of this single tool call ---
+        tool_invocation_id = processed_event.tool_invocation_id if processed_event.tool_invocation_id else 'N/A'
+        if notifier:
+            log_message = ""
+            if processed_event.error:
+                log_message = f"[TOOL_RESULT_ERROR_PROCESSED] Agent_ID: {agent_id}, Tool: {processed_event.tool_name}, Invocation_ID: {tool_invocation_id}, Error: {processed_event.error}"
+            else:
+                log_message = f"[TOOL_RESULT_SUCCESS_PROCESSED] Agent_ID: {agent_id}, Tool: {processed_event.tool_name}, Invocation_ID: {tool_invocation_id}, Result: {str(processed_event.result)}"
+            
             try:
-                raw_result_str_for_debug_log = json.dumps(processed_event.result, indent=2)
-            except TypeError: # pragma: no cover
-                raw_result_str_for_debug_log = str(processed_event.result)
-            logger.debug(f"Agent '{agent_id}' tool '{processed_event.tool_name}' (ID: {tool_invocation_id}) raw result:\n---\n{raw_result_str_for_debug_log}\n---")
+                log_data = {
+                    "log_entry": log_message,
+                    "tool_invocation_id": tool_invocation_id,
+                    "tool_name": processed_event.tool_name,
+                }
+                notifier.notify_agent_data_tool_log(log_data)
+                logger.debug(f"Agent '{agent_id}': Notified individual tool result for '{processed_event.tool_name}'.")
+            except Exception as e_notify: 
+                logger.error(f"Agent '{agent_id}': Error notifying tool result log: {e_notify}", exc_info=True)
 
+        # --- Step 3: Manage the multi-tool call turn state ---
+        active_turn = context.state.active_multi_tool_call_turn
 
-        content_for_llm: str
-        if processed_event.error:
-            content_for_llm = (
-                f"The tool '{processed_event.tool_name}' (invocation ID: {tool_invocation_id}) encountered an error.\n"
-                f"Error details: {processed_event.error}\n"
-                f"Please analyze this error and decide the next course of action."
-            )
-            log_msg_error_processed = f"[TOOL_RESULT_ERROR_PROCESSED] Agent_ID: {agent_id}, Tool: {processed_event.tool_name}, Invocation_ID: {tool_invocation_id}, Error: {processed_event.error}"
-            if notifier:
-                try:
-                    log_data = {
-                        "log_entry": log_msg_error_processed,
-                        "tool_invocation_id": tool_invocation_id,
-                        "tool_name": processed_event.tool_name,
-                    }
-                    notifier.notify_agent_data_tool_log(log_data)
-                except Exception as e_notify: 
-                    logger.error(f"Agent '{agent_id}': Error notifying tool result error log: {e_notify}", exc_info=True)
-        else:
-            try:
-                result_str_for_llm = json.dumps(processed_event.result, indent=2) if not isinstance(processed_event.result, str) else processed_event.result
-            except TypeError: # pragma: no cover
-                result_str_for_llm = str(processed_event.result)
+        # Case 1: Not a multi-tool call turn, dispatch to LLM immediately.
+        if not active_turn:
+            logger.info(f"Agent '{agent_id}' handling single ToolResultEvent from tool: '{processed_event.tool_name}'.")
+            await self._dispatch_aggregated_results_to_llm([processed_event], context)
+            return
 
-            content_for_llm = (
-                f"The tool '{processed_event.tool_name}' (invocation ID: {tool_invocation_id}) has executed.\n"
-                f"Result:\n{result_str_for_llm}\n" 
-                f"Based on this result, what is the next step or final answer?"
-            )
-            log_msg_success_processed = f"[TOOL_RESULT_SUCCESS_PROCESSED] Agent_ID: {agent_id}, Tool: {processed_event.tool_name}, Invocation_ID: {tool_invocation_id}, Result: {str(processed_event.result)}"
-            if notifier:
-                try:
-                    log_data = {
-                        "log_entry": log_msg_success_processed,
-                        "tool_invocation_id": tool_invocation_id,
-                        "tool_name": processed_event.tool_name,
-                    }
-                    notifier.notify_agent_data_tool_log(log_data)
-                except Exception as e_notify: 
-                    logger.error(f"Agent '{agent_id}': Error notifying tool result success log: {e_notify}", exc_info=True)
-        
-        logger.debug(f"Agent '{agent_id}' preparing message for LLM based on tool '{processed_event.tool_name}' (ID: {tool_invocation_id}) result:\n---\n{content_for_llm}\n---")
-        llm_user_message = LLMUserMessage(content=content_for_llm)
-        
-        next_event = LLMUserMessageReadyEvent(llm_user_message=llm_user_message) 
-        await context.input_event_queues.enqueue_internal_system_event(next_event)
+        # Case 2: Multi-tool call turn is active, accumulate results.
+        active_turn.results.append(processed_event)
+        num_results = len(active_turn.results)
+        num_expected = len(active_turn.invocations)
+        logger.info(f"Agent '{agent_id}' handling ToolResultEvent for multi-tool call turn. "
+                    f"Collected {num_results}/{num_expected} results.")
+
+        # If not all results are in, just wait for the next ToolResultEvent.
+        if not active_turn.is_complete():
+            return
+            
+        # If all results are in, dispatch them to the LLM and clean up the turn state.
+        logger.info(f"Agent '{agent_id}': All tool results for the turn collected. Aggregating for LLM.")
+        await self._dispatch_aggregated_results_to_llm(active_turn.results, context)
         
-        logger.info(f"Agent '{agent_id}' enqueued LLMUserMessageReadyEvent for LLM based on tool '{processed_event.tool_name}' (ID: {tool_invocation_id}) result summary.")
+        context.state.active_multi_tool_call_turn = None
+        logger.info(f"Agent '{agent_id}': Multi-tool call turn state has been cleared.")

autobyteus/agent/handlers/user_input_message_event_handler.py

@@ -3,14 +3,14 @@ import logging
 from typing import TYPE_CHECKING
 
 from autobyteus.agent.handlers.base_event_handler import AgentEventHandler
-from autobyteus.agent.events import UserMessageReceivedEvent, LLMUserMessageReadyEvent 
-from autobyteus.agent.message.agent_input_user_message import AgentInputUserMessage 
+from autobyteus.agent.events import UserMessageReceivedEvent, LLMUserMessageReadyEvent
+from autobyteus.agent.message.agent_input_user_message import AgentInputUserMessage
 from autobyteus.agent.input_processor import BaseAgentUserInputMessageProcessor
-from autobyteus.llm.user_message import LLMUserMessage
+from autobyteus.agent.message.multimodal_message_builder import build_llm_user_message
 
 
 if TYPE_CHECKING:
-    from autobyteus.agent.context import AgentContext 
+    from autobyteus.agent.context import AgentContext
     from autobyteus.agent.events.notifiers import AgentExternalEventNotifier
 
 logger = logging.getLogger(__name__)
@@ -18,24 +18,23 @@ logger = logging.getLogger(__name__)
 class UserInputMessageEventHandler(AgentEventHandler):
     """
     Handles UserMessageReceivedEvents by first applying any configured
-    AgentUserInputMessageProcessors (provided as instances) to the AgentInputUserMessage,
-    then converting the processed message into an LLMUserMessage, and finally
+    AgentUserInputMessageProcessors, then using the multimodal_message_builder
+    to convert the processed message into an LLMUserMessage, and finally
     enqueuing an LLMUserMessageReadyEvent for further processing by the LLM.
-    It also checks for metadata to emit special notifications for system-generated tasks.
     """
 
     def __init__(self):
         logger.info("UserInputMessageEventHandler initialized.")
 
     async def handle(self,
-                     event: UserMessageReceivedEvent, 
+                     event: UserMessageReceivedEvent,
                      context: 'AgentContext') -> None:
-        if not isinstance(event, UserMessageReceivedEvent): 
+        if not isinstance(event, UserMessageReceivedEvent):
             logger.warning(f"UserInputMessageEventHandler received non-UserMessageReceivedEvent: {type(event)}. Skipping.")
             return
 
-        original_agent_input_user_msg: AgentInputUserMessage = event.agent_input_user_message 
-        
+        original_agent_input_user_msg: AgentInputUserMessage = event.agent_input_user_message
+
         # --- NEW LOGIC: Check metadata for system-generated tasks and notify TUI ---
         if original_agent_input_user_msg.metadata.get('source') == 'system_task_notifier':
             if context.phase_manager:
@@ -47,11 +46,11 @@ class UserInputMessageEventHandler(AgentEventHandler):
                 notifier.notify_agent_data_system_task_notification_received(notification_data)
                 logger.info(f"Agent '{context.agent_id}' emitted system task notification for TUI.")
         # --- END NEW LOGIC ---
-        
-        processed_agent_input_user_msg: AgentInputUserMessage = original_agent_input_user_msg 
-        
-        logger.info(f"Agent '{context.agent_id}' handling UserMessageReceivedEvent: '{original_agent_input_user_msg.content}'") 
-        
+
+        processed_agent_input_user_msg: AgentInputUserMessage = original_agent_input_user_msg
+
+        logger.info(f"Agent '{context.agent_id}' handling UserMessageReceivedEvent: '{original_agent_input_user_msg.content}'")
+
         processor_instances = context.config.input_processors
         if processor_instances:
             processor_names = [p.get_name() for p in processor_instances]
@@ -62,14 +61,14 @@ class UserInputMessageEventHandler(AgentEventHandler):
                     if not isinstance(processor_instance, BaseAgentUserInputMessageProcessor):
                         logger.error(f"Agent '{context.agent_id}': Invalid input processor type in config: {type(processor_instance)}. Skipping.")
                         continue
-                    
+
                     processor_name_for_log = processor_instance.get_name()
                     logger.debug(f"Agent '{context.agent_id}': Applying input processor '{processor_name_for_log}'.")
                     msg_before_this_processor = processed_agent_input_user_msg
                     # Pass the original event to the processor
                     processed_agent_input_user_msg = await processor_instance.process(
-                        message=msg_before_this_processor, 
-                        context=context, 
+                        message=msg_before_this_processor,
+                        context=context,
                         triggering_event=event
                     )
                     logger.info(f"Agent '{context.agent_id}': Input processor '{processor_name_for_log}' applied successfully.")
@@ -81,12 +80,10 @@ class UserInputMessageEventHandler(AgentEventHandler):
         else:
             logger.debug(f"Agent '{context.agent_id}': No input processors configured in agent config.")
 
-        llm_user_message = LLMUserMessage( 
-            content=processed_agent_input_user_msg.content,
-            image_urls=processed_agent_input_user_msg.image_urls 
-        )
+        # --- Refactored: Use the dedicated builder ---
+        llm_user_message = build_llm_user_message(processed_agent_input_user_msg)
         
-        llm_user_message_ready_event = LLMUserMessageReadyEvent(llm_user_message=llm_user_message) 
+        llm_user_message_ready_event = LLMUserMessageReadyEvent(llm_user_message=llm_user_message)
         await context.input_event_queues.enqueue_internal_system_event(llm_user_message_ready_event)
-        
+
         logger.info(f"Agent '{context.agent_id}' processed AgentInputUserMessage and enqueued LLMUserMessageReadyEvent.")

autobyteus/agent/llm_response_processor/provider_aware_tool_usage_processor.py

@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, List
 
 from .base_processor import BaseLLMResponseProcessor
 from autobyteus.agent.events import PendingToolInvocationEvent
-from autobyteus.agent.tool_invocation import ToolInvocation
+from autobyteus.agent.tool_invocation import ToolInvocation, ToolInvocationTurn
 from autobyteus.tools.usage.parsers import ProviderAwareToolUsageParser
 from autobyteus.tools.usage.parsers.exceptions import ToolUsageParseException
 
@@ -72,6 +72,12 @@ class ProviderAwareToolUsageProcessor(BaseLLMResponseProcessor):
             processed_invocations.append(invocation)
 
         # --- END NEW LOGIC ---
+        
+        # --- NEW: Initialize the multi-tool turn state ---
+        if len(processed_invocations) > 0:
+            logger.info(f"Agent '{context.agent_id}': Initializing multi-tool call turn with {len(processed_invocations)} invocations.")
+            context.state.active_multi_tool_call_turn = ToolInvocationTurn(invocations=processed_invocations)
+        # --- END NEW ---
 
         logger.info(f"Agent '{context.agent_id}': Parsed {len(processed_invocations)} tool invocations. Enqueuing events with unique IDs.")
         for invocation in processed_invocations:

autobyteus/agent/message/__init__.py

@@ -9,12 +9,14 @@ from .agent_input_user_message import AgentInputUserMessage
 from .send_message_to import SendMessageTo
 from .context_file import ContextFile
 from .context_file_type import ContextFileType
+from .multimodal_message_builder import build_llm_user_message
 
 __all__ = [
-    "InterAgentMessage", 
-    "InterAgentMessageType", 
-    "AgentInputUserMessage", 
+    "InterAgentMessage",
+    "InterAgentMessageType",
+    "AgentInputUserMessage",
     "SendMessageTo",
-    "ContextFile",        
-    "ContextFileType",    
+    "ContextFile",
+    "ContextFileType",
+    "build_llm_user_message",
 ]

autobyteus/agent/message/agent_input_user_message.py

@@ -8,21 +8,18 @@ from .context_file import ContextFile # Import the new ContextFile dataclass
 logger = logging.getLogger(__name__)
 
 @dataclass
-class AgentInputUserMessage: 
+class AgentInputUserMessage:
     """
     Represents a message received from an external user interacting with the agent system.
-    This is a simple dataclass. It includes support for a list of ContextFile objects, 
-    allowing users to provide various documents as context.
+    This is a simple dataclass. It includes support for a list of ContextFile objects,
+    allowing users to provide various documents and media as context via a single list.
     """
     content: str
-    image_urls: Optional[List[str]] = field(default=None) # Basic list of strings
     context_files: Optional[List[ContextFile]] = field(default=None)
     metadata: Dict[str, Any] = field(default_factory=dict)
 
     def __post_init__(self):
         # Basic type validation that dataclasses don't do automatically for mutable defaults or complex types
-        if self.image_urls is not None and not (isinstance(self.image_urls, list) and all(isinstance(url, str) for url in self.image_urls)):
-            raise TypeError("AgentInputUserMessage 'image_urls' must be a list of strings if provided.")
         if self.context_files is not None and not (isinstance(self.context_files, list) and all(isinstance(cf, ContextFile) for cf in self.context_files)):
             raise TypeError("AgentInputUserMessage 'context_files' must be a list of ContextFile objects if provided.")
         if not isinstance(self.metadata, dict): # Should be caught by default_factory, but good practice
@@ -34,7 +31,7 @@ class AgentInputUserMessage:
             num_context_files = len(self.context_files) if self.context_files else 0
             logger.debug(
                 f"AgentInputUserMessage initialized. Content: '{self.content[:50]}...', "
-                f"Image URLs: {self.image_urls}, Num ContextFiles: {num_context_files}, "
+                f"Num ContextFiles: {num_context_files}, "
                 f"Metadata keys: {list(self.metadata.keys())}"
             )
 
@@ -47,7 +44,6 @@ class AgentInputUserMessage:
 
         return {
             "content": self.content,
-            "image_urls": self.image_urls,
             "context_files": context_files_dict_list,
             "metadata": self.metadata,
         }
@@ -59,31 +55,25 @@ class AgentInputUserMessage:
         if not isinstance(content, str): # Ensure content is string
             raise ValueError("AgentInputUserMessage 'content' in dictionary must be a string.")
 
-        image_urls = data.get("image_urls")
-        if image_urls is not None and not (isinstance(image_urls, list) and all(isinstance(url, str) for url in image_urls)):
-            raise ValueError("AgentInputUserMessage 'image_urls' in dictionary must be a list of strings if provided.")
-
         context_files_data = data.get("context_files")
         context_files_list: Optional[List[ContextFile]] = None
         if context_files_data is not None:
             if not isinstance(context_files_data, list):
                 raise ValueError("AgentInputUserMessage 'context_files' in dictionary must be a list if provided.")
             context_files_list = [ContextFile.from_dict(cf_data) for cf_data in context_files_data]
-        
+
         metadata = data.get("metadata", {})
         if not isinstance(metadata, dict):
             raise ValueError("AgentInputUserMessage 'metadata' in dictionary must be a dict if provided.")
 
         return cls(
             content=content,
-            image_urls=image_urls,
             context_files=context_files_list,
             metadata=metadata
         )
 
     def __repr__(self) -> str:
         content_preview = f"{self.content[:100]}..." if len(self.content) > 100 else self.content
-        images_repr = f", image_urls={self.image_urls}" if self.image_urls else ""
         
         if self.context_files:
             context_repr = f", context_files=[{len(self.context_files)} ContextFile(s)]"
@@ -93,4 +83,4 @@ class AgentInputUserMessage:
         meta_repr = f", metadata_keys={list(self.metadata.keys())}" if self.metadata else ""
         
         return (f"AgentInputUserMessage(content='{content_preview}'"
-                f"{images_repr}{context_repr}{meta_repr})")
+                f"{context_repr}{meta_repr})")