solace-agent-mesh 1.6.1__py3-none-any.whl → 1.13.2__py3-none-any.whl

solace_agent_mesh/agent/adk/callbacks.py

@@ -10,7 +10,6 @@ import asyncio
 import uuid
 from typing import Any, Dict, Optional, TYPE_CHECKING, List
 from collections import defaultdict
-from datetime import datetime, timezone
 
 from google.adk.tools import BaseTool, ToolContext
 from google.adk.artifacts import BaseArtifactService
@@ -34,32 +33,31 @@ from ...agent.utils.context_helpers import (
     get_session_from_callback_context,
 )
 from ..tools.tool_definition import BuiltinTool
+from ..tools.peer_agent_tool import PEER_TOOL_PREFIX
 
 from ...common.utils.embeds import (
     EMBED_DELIMITER_OPEN,
     EMBED_DELIMITER_CLOSE,
-)
-
-from ...common.utils.embeds import (
     EMBED_CHAIN_DELIMITER,
+    EARLY_EMBED_TYPES,
+    evaluate_embed,
+    resolve_embeds_in_string,
 )
+from ...common.utils.embeds.types import ResolutionMode
 
 from ...common.utils.embeds.modifiers import MODIFIER_IMPLEMENTATIONS
 
 from ...common import a2a
-from ...common.a2a.types import ContentPart
+from ...common.a2a.types import ArtifactInfo
 from ...common.data_parts import (
     AgentProgressUpdateData,
     ArtifactCreationProgressData,
     LlmInvocationData,
     ToolInvocationStartData,
     ToolResultData,
+    TemplateBlockData,
 )
 
-from ...agent.utils.artifact_helpers import (
-    save_artifact_with_metadata,
-    DEFAULT_SCHEMA_MAX_KEYS,
-)
 
 METADATA_RESPONSE_KEY = "appended_artifact_metadata"
 from ..tools.builtin_artifact_tools import _internal_create_artifact
@@ -73,10 +71,14 @@ from ...agent.adk.stream_parser import (
     BlockProgressedEvent,
     BlockCompletedEvent,
     BlockInvalidatedEvent,
+    TemplateBlockStartedEvent,
+    TemplateBlockCompletedEvent,
     ARTIFACT_BLOCK_DELIMITER_OPEN,
     ARTIFACT_BLOCK_DELIMITER_CLOSE,
+    TEMPLATE_LIQUID_START_SEQUENCE,
 )
 
+
 log = logging.getLogger(__name__)
 
 A2A_LLM_STREAM_CHUNKS_PROCESSED_KEY = "temp:llm_stream_chunks_processed"
@@ -90,32 +92,95 @@ async def _publish_data_part_status_update(
     a2a_context: Dict[str, Any],
     data_part_model: BaseModel,
 ):
-    """Helper to construct and publish a TaskStatusUpdateEvent with a DataPart."""
-    logical_task_id = a2a_context.get("logical_task_id")
-    context_id = a2a_context.get("contextId")
+    """Helper to construct and publish a TaskStatusUpdateEvent with a DataPart.
 
-    status_update_event = a2a.create_data_signal_event(
-        task_id=logical_task_id,
-        context_id=context_id,
+    This function delegates to the host component's publish_data_signal_from_thread method,
+    which handles the async loop check and scheduling internally.
+    """
+    host_component.publish_data_signal_from_thread(
+        a2a_context=a2a_context,
         signal_data=data_part_model,
-        agent_name=host_component.agent_name,
+        skip_buffer_flush=False,
+        log_identifier=host_component.log_identifier,
     )
 
-    loop = host_component.get_async_loop()
-    if loop and loop.is_running():
-        asyncio.run_coroutine_threadsafe(
-            host_component._publish_status_update_with_buffer_flush(
-                status_update_event,
-                a2a_context,
-                skip_buffer_flush=False,
-            ),
-            loop,
+
+async def _resolve_early_embeds_in_chunk(
+    chunk: str,
+    callback_context: CallbackContext,
+    host_component: "SamAgentComponent",
+    log_identifier: str,
+) -> str:
+    """
+    Resolves early embeds in an artifact chunk before streaming to the browser.
+
+    Args:
+        chunk: The text chunk containing potential embeds
+        callback_context: The ADK callback context with services
+        host_component: The host component instance
+        log_identifier: Identifier for logging
+
+    Returns:
+        The chunk with early embeds resolved
+    """
+    if not chunk or EMBED_DELIMITER_OPEN not in chunk:
+        return chunk
+
+    try:
+        # Build resolution context from callback_context (pattern from EmbedResolvingMCPToolset)
+        invocation_context = callback_context._invocation_context
+        if not invocation_context:
+            log.warning(
+                "%s No invocation context available for embed resolution",
+                log_identifier,
+            )
+            return chunk
+
+        session_context = invocation_context.session
+        if not session_context:
+            log.warning(
+                "%s No session context available for embed resolution", log_identifier
+            )
+            return chunk
+
+        resolution_context = {
+            "artifact_service": invocation_context.artifact_service,
+            "session_context": {
+                "session_id": get_original_session_id(invocation_context),
+                "user_id": session_context.user_id,
+                "app_name": session_context.app_name,
+            },
+        }
+
+        # Resolve only early embeds (math, datetime, uuid, artifact_meta)
+        resolved_text, processed_until, _ = await resolve_embeds_in_string(
+            text=chunk,
+            context=resolution_context,
+            resolver_func=evaluate_embed,
+            types_to_resolve=EARLY_EMBED_TYPES,  # Only resolve early embeds
+            resolution_mode=ResolutionMode.ARTIFACT_STREAMING,  # New mode
+            log_identifier=log_identifier,
+            config=None,  # Could pass host_component config if needed
         )
-    else:
+
+        # SAFETY CHECK: If resolver buffered something, parser has a bug
+        if processed_until < len(chunk):
+            log.error(
+                "%s PARSER BUG DETECTED: Resolver buffered partial embed. "
+                "Chunk ends with: %r. Returning unresolved chunk to avoid corruption.",
+                log_identifier,
+                chunk[-50:] if len(chunk) > 50 else chunk,
+            )
+            # Fallback: return original unresolved chunk (degraded but not corrupted)
+            return chunk
+
+        return resolved_text
+
+    except Exception as e:
         log.error(
-            "%s Async loop not available. Cannot publish status update.",
-            host_component.log_identifier,
+            "%s Error resolving embeds in chunk: %s", log_identifier, e, exc_info=True
         )
+        return chunk  # Return original chunk on error
 
 
 async def process_artifact_blocks_callback(
@@ -135,9 +200,13 @@ async def process_artifact_blocks_callback(
     parser: FencedBlockStreamParser = session.state.get(parser_state_key)
     if parser is None:
         log.debug("%s New turn. Creating new FencedBlockStreamParser.", log_identifier)
-        parser = FencedBlockStreamParser(progress_update_interval_bytes=250)
+        parser = FencedBlockStreamParser(progress_update_interval_bytes=50)
         session.state[parser_state_key] = parser
         session.state["completed_artifact_blocks_list"] = []
+        session.state["completed_template_blocks_list"] = []
+        session.state["artifact_chars_sent"] = (
+            0  # Reset character tracking for new turn
+        )
 
     stream_chunks_were_processed = callback_context.state.get(
         A2A_LLM_STREAM_CHUNKS_PROCESSED_KEY, False
@@ -169,18 +238,49 @@ async def process_artifact_blocks_callback(
                             log_identifier,
                             event.params,
                         )
+                        # Reset character tracking for this new artifact block
+                        session.state["artifact_chars_sent"] = 0
+
                         filename = event.params.get("filename", "unknown_artifact")
+                        if filename == "unknown_artifact":
+                            log.warning(
+                                "%s Fenced artifact block started without a 'filename' parameter.",
+                                log_identifier,
+                            )
+                        description = event.params.get("description")
+                        if filename == "unknown_artifact":
+                            log.warning(
+                                "%s Fenced artifact block started without a 'filename' parameter.",
+                                log_identifier,
+                            )
                         if a2a_context:
+                            status_text = f"Receiving artifact `{filename}`..."
+                            if description:
+                                status_text = (
+                                    f"Receiving artifact `{filename}`: {description}"
+                                )
                             progress_data = AgentProgressUpdateData(
-                                status_text=f"Receiving artifact `{filename}`..."
+                                status_text=status_text
                             )
                             await _publish_data_part_status_update(
                                 host_component, a2a_context, progress_data
                             )
+                            # Also send an initial in-progress event to create the UI bubble
+                            artifact_progress_data = ArtifactCreationProgressData(
+                                filename=filename,
+                                description=description,
+                                status="in-progress",
+                                bytes_transferred=0,
+                                artifact_chunk=None,
+                            )
+
+                            await _publish_data_part_status_update(
+                                host_component, a2a_context, artifact_progress_data
+                            )
                         params_str = " ".join(
                             [f'{k}="{v}"' for k, v in event.params.items()]
                         )
-                        original_text = f"«««save_artifact: {params_str}\n"
+                        original_text = f"{ARTIFACT_BLOCK_DELIMITER_OPEN}save_artifact: {params_str}\n"
                         session.state["artifact_block_original_text"] = original_text
 
                     elif isinstance(event, BlockProgressedEvent):
@@ -189,20 +289,44 @@ async def process_artifact_blocks_callback(
                             log_identifier,
                             event.buffered_size,
                         )
-                        params = parser._block_params
+                        params = event.params
                         filename = params.get("filename", "unknown_artifact")
+                        if filename == "unknown_artifact":
+                            log.warning(
+                                "%s Fenced artifact block progressed without a 'filename' parameter.",
+                                log_identifier,
+                            )
                         if a2a_context:
+                            # Resolve early embeds in the chunk before streaming
+                            resolved_chunk = await _resolve_early_embeds_in_chunk(
+                                chunk=event.chunk,
+                                callback_context=callback_context,
+                                host_component=host_component,
+                                log_identifier=f"{log_identifier}[ResolveChunk]",
+                            )
+
                             progress_data = ArtifactCreationProgressData(
                                 filename=filename,
-                                bytes_saved=event.buffered_size,
-                                artifact_chunk=event.chunk,
+                                description=params.get("description"),
+                                status="in-progress",
+                                bytes_transferred=event.buffered_size,
+                                artifact_chunk=resolved_chunk,  # Resolved chunk
                             )
+
+                            # Track the cumulative character count of what we've sent
+                            # We need character count (not bytes) to slice correctly later
+                            previous_char_count = session.state.get(
+                                "artifact_chars_sent", 0
+                            )
+                            new_char_count = previous_char_count + len(event.chunk)
+                            session.state["artifact_chars_sent"] = new_char_count
+
                             await _publish_data_part_status_update(
                                 host_component, a2a_context, progress_data
                             )
 
                     elif isinstance(event, BlockCompletedEvent):
-                        log.info(
+                        log.debug(
                             "%s Event: BlockCompleted. Content length: %d",
                             log_identifier,
                             len(event.content),
@@ -236,6 +360,18 @@ async def process_artifact_blocks_callback(
                                     "original_text": original_text,
                                 }
                             )
+                            if a2a_context:
+                                if not filename or not filename.strip():
+                                    filename = "unknown_artifact"
+                                progress_data = ArtifactCreationProgressData(
+                                    filename=filename or "unknown_artifact",
+                                    description=params.get("description"),
+                                    status="failed",
+                                    bytes_transferred=0,
+                                )
+                                await _publish_data_part_status_update(
+                                    host_component, a2a_context, progress_data
+                                )
                             continue
 
                         kwargs_for_call = {
@@ -257,7 +393,6 @@ async def process_artifact_blocks_callback(
                                     log_identifier,
                                     params["schema_max_keys"],
                                 )
-
                         wrapped_creator = ADKToolWrapper(
                             original_func=_internal_create_artifact,
                             tool_config=None,  # No specific config for this internal tool
@@ -288,6 +423,13 @@ async def process_artifact_blocks_callback(
                                             version_for_tool,
                                             logical_task_id,
                                         )
+                                    else:
+                                        log.warning(
+                                            "%s TaskExecutionContext not found for task %s, cannot register inline artifact '%s'.",
+                                            log_identifier,
+                                            logical_task_id,
+                                            filename,
+                                        )
                                 else:
                                     log.warning(
                                         "%s No logical_task_id, cannot register inline artifact.",
@@ -299,19 +441,172 @@ async def process_artifact_blocks_callback(
                                     log_identifier,
                                     e_track,
                                 )
+
+                            # Send final progress update with any remaining content not yet sent
+                            if a2a_context:
+                                # Check if there's unsent content (content after last progress event)
+                                total_bytes = len(event.content.encode("utf-8"))
+                                chars_already_sent = session.state.get(
+                                    "artifact_chars_sent", 0
+                                )
+
+                                if chars_already_sent < len(event.content):
+                                    # There's unsent content - send it as a final progress update
+                                    final_chunk = event.content[chars_already_sent:]
+
+                                    # Resolve embeds in final chunk
+                                    resolved_final_chunk = await _resolve_early_embeds_in_chunk(
+                                        chunk=final_chunk,
+                                        callback_context=callback_context,
+                                        host_component=host_component,
+                                        log_identifier=f"{log_identifier}[ResolveFinalChunk]",
+                                    )
+
+                                    final_progress_data = ArtifactCreationProgressData(
+                                        filename=filename,
+                                        description=params.get("description"),
+                                        status="in-progress",
+                                        bytes_transferred=total_bytes,
+                                        artifact_chunk=resolved_final_chunk,  # Resolved final chunk
+                                    )
+
+                                    await _publish_data_part_status_update(
+                                        host_component, a2a_context, final_progress_data
+                                    )
+
+                            # Publish completion status immediately via SSE
+                            if a2a_context:
+                                progress_data = ArtifactCreationProgressData(
+                                    filename=filename,
+                                    description=params.get("description"),
+                                    status="completed",
+                                    bytes_transferred=len(event.content),
+                                    mime_type=params.get("mime_type"),
+                                    version=version_for_tool,
+                                )
+                                await _publish_data_part_status_update(
+                                    host_component, a2a_context, progress_data
+                                )
                         else:
                             status_for_tool = "error"
                             version_for_tool = 0
+                            # Publish failure status immediately via SSE
+                            if a2a_context:
+                                progress_data = ArtifactCreationProgressData(
+                                    filename=filename,
+                                    description=params.get("description"),
+                                    status="failed",
+                                    bytes_transferred=len(event.content),
+                                )
+                                await _publish_data_part_status_update(
+                                    host_component, a2a_context, progress_data
+                                )
 
                         session.state["completed_artifact_blocks_list"].append(
                             {
                                 "filename": filename,
                                 "version": version_for_tool,
                                 "status": status_for_tool,
+                                "description": params.get("description"),
+                                "mime_type": params.get("mime_type"),
+                                "bytes_transferred": len(event.content),
                                 "original_text": original_text,
                             }
                         )
 
+                    elif isinstance(event, TemplateBlockStartedEvent):
+                        log.debug(
+                            "%s Event: TemplateBlockStarted. Params: %s",
+                            log_identifier,
+                            event.params,
+                        )
+
+                    elif isinstance(event, TemplateBlockCompletedEvent):
+                        log.debug(
+                            "%s Event: TemplateBlockCompleted. Template length: %d",
+                            log_identifier,
+                            len(event.template_content),
+                        )
+
+                        # Create a TemplateBlockData message to send to the gateway
+                        template_id = str(uuid.uuid4())
+                        params = event.params
+
+                        data_artifact = params.get("data")
+                        if not data_artifact:
+                            log.warning(
+                                "%s Template block is missing 'data' parameter. Skipping.",
+                                log_identifier,
+                            )
+                            continue
+
+                        template_data = TemplateBlockData(
+                            template_id=template_id,
+                            data_artifact=data_artifact,
+                            jsonpath=params.get("jsonpath"),
+                            limit=(
+                                int(params.get("limit"))
+                                if params.get("limit")
+                                else None
+                            ),
+                            template_content=event.template_content,
+                        )
+
+                        # Publish A2A status update with template metadata
+                        if a2a_context:
+                            await _publish_data_part_status_update(
+                                host_component, a2a_context, template_data
+                            )
+                            log.info(
+                                "%s Published TemplateBlockData with ID: %s",
+                                log_identifier,
+                                template_id,
+                            )
+
+                        # Reconstruct the original template block text for peer-to-peer responses
+                        # Peer agents don't receive TemplateBlockData signals, so they need
+                        # the original block text to pass templates through to the gateway
+                        params_str = " ".join([f'{k}="{v}"' for k, v in params.items()])
+                        original_template_text = (
+                            f"{TEMPLATE_LIQUID_START_SEQUENCE} {params_str}\n"
+                            f"{event.template_content}"
+                            f"{ARTIFACT_BLOCK_DELIMITER_CLOSE}"
+                        )
+
+                        # For RUN_BASED sessions (peer-to-peer agent requests), preserve the
+                        # template block in the response text at its original position.
+                        # This allows the calling agent to forward it to the gateway.
+                        # Gateway requests use streaming sessions and receive TemplateBlockData
+                        # signals instead.
+                        is_run_based = a2a_context and a2a_context.get(
+                            "is_run_based_session", False
+                        )
+                        if is_run_based and llm_response.partial:
+                            processed_parts.append(
+                                adk_types.Part(text=original_template_text)
+                            )
+                            log.debug(
+                                "%s Preserved template block in RUN_BASED peer response. Template ID: %s",
+                                log_identifier,
+                                template_id,
+                            )
+
+                        # Store template_id and original text in session for potential future use
+                        # (Gateway will handle the actual resolution via signals,
+                        # but peer agents need the original text in their responses)
+                        if (
+                            "completed_template_blocks_list" not in session.state
+                            or session.state["completed_template_blocks_list"] is None
+                        ):
+                            session.state["completed_template_blocks_list"] = []
+                        session.state["completed_template_blocks_list"].append(
+                            {
+                                "template_id": template_id,
+                                "data_artifact": data_artifact,
+                                "original_text": original_template_text,
+                            }
+                        )
+
                     elif isinstance(event, BlockInvalidatedEvent):
                         log.debug(
                             "%s Event: BlockInvalidated. Rolled back: '%s'",
@@ -347,6 +642,11 @@ async def process_artifact_blocks_callback(
                 )
                 params = event.params
                 filename = params.get("filename", "unknown_artifact")
+                if filename == "unknown_artifact":
+                    log.warning(
+                        "%s Unterminated fenced artifact block is missing a valid 'filename'. Failing operation.",
+                        log_identifier,
+                    )
                 if (
                     "completed_artifact_blocks_list" not in session.state
                     or session.state["completed_artifact_blocks_list"] is None
@@ -392,8 +692,12 @@ async def process_artifact_blocks_callback(
                 len(completed_blocks_list),
             )
 
+            # Get a2a_context for sending signals
+            a2a_context = callback_context.state.get("a2a_context")
+
             tool_call_parts = []
             for block_info in completed_blocks_list:
+                function_call_id = f"host-notify-{uuid.uuid4()}"
                 notify_tool_call = adk_types.FunctionCall(
                     name="_notify_artifact_save",
                     args={
@@ -401,10 +705,41 @@ async def process_artifact_blocks_callback(
                         "version": block_info["version"],
                         "status": block_info["status"],
                     },
-                    id=f"host-notify-{uuid.uuid4()}",
+                    id=function_call_id,
                 )
                 tool_call_parts.append(adk_types.Part(function_call=notify_tool_call))
 
+                # Send artifact saved notification now that we have the function_call_id
+                # This ensures the signal and tool call arrive together
+                if block_info["status"] == "success" and a2a_context:
+                    try:
+                        artifact_info = ArtifactInfo(
+                            filename=block_info["filename"],
+                            version=block_info["version"],
+                            mime_type=block_info.get("mime_type")
+                            or "application/octet-stream",
+                            size=block_info.get("bytes_transferred", 0),
+                            description=block_info.get("description"),
+                            version_count=None,  # Count not available in save context
+                        )
+                        await host_component.notify_artifact_saved(
+                            artifact_info=artifact_info,
+                            a2a_context=a2a_context,
+                            function_call_id=function_call_id,
+                        )
+                        log.debug(
+                            "%s Published artifact saved notification for fenced block: %s (function_call_id=%s)",
+                            log_identifier,
+                            block_info["filename"],
+                            function_call_id,
+                        )
+                    except Exception as signal_err:
+                        log.warning(
+                            "%s Failed to publish artifact saved notification: %s",
+                            log_identifier,
+                            signal_err,
+                        )
+
             existing_parts = llm_response.content.parts if llm_response.content else []
             final_existing_parts = existing_parts
 
@@ -419,6 +754,7 @@ async def process_artifact_blocks_callback(
         session.state[parser_state_key] = None
         session.state["completed_artifact_blocks_list"] = None
         session.state["artifact_block_original_text"] = None
+        session.state["completed_template_blocks_list"] = None
         log.debug("%s Cleaned up parser session state.", log_identifier)
 
     return None
@@ -671,26 +1007,24 @@ async def manage_large_mcp_tool_responses_callback(
     message_parts_for_llm: list[str] = []
 
     if needs_truncation_for_llm:
-        truncation_suffix = "... [Response truncated due to size limit.]"
-        adjusted_max_bytes = llm_max_bytes - len(truncation_suffix.encode("utf-8"))
-        if adjusted_max_bytes < 0:
-            adjusted_max_bytes = 0
-
-        truncated_bytes = serialized_original_response_str.encode("utf-8")[
-            :adjusted_max_bytes
-        ]
-        truncated_preview_str = (
-            truncated_bytes.decode("utf-8", "ignore") + truncation_suffix
-        )
-
+        # ALL-OR-NOTHING APPROACH: Do not include truncated data to prevent LLM hallucination.
+        # When LLMs receive partial data, they tend to confidently fill in gaps with
+        # hallucinated information. By withholding partial data entirely, we force the LLM
+        # to use reliable mechanisms (template_liquid, load_artifact) to access the full data.
         final_llm_response_dict["mcp_tool_output"] = {
-            "type": "truncated_json_string",
-            "content": truncated_preview_str,
+            "type": "data_in_artifact_only",
+            "message": "Data exceeds size limit. Full data saved as artifact - use template_liquid, load_artifact or other artifact analysis tools to process and access.",
         }
         message_parts_for_llm.append(
-            f"The response from tool '{tool.name}' was too large ({original_response_bytes} bytes) for direct display and has been truncated."
+            f"The response from tool '{tool.name}' was too large ({original_response_bytes} bytes) to display directly. "
+            "The data has NOT been included here to prevent incomplete information. "
+            "You MUST use template_liquid (for displaying to users) or load_artifact or other "
+            "artifact analysis tools (for processing) to access the full data."
+        )
+        log.debug(
+            "%s MCP tool output withheld from LLM (all-or-nothing approach).",
+            log_identifier,
         )
-        log.debug("%s MCP tool output truncated for LLM.", log_identifier)
 
     if needs_saving_as_artifact:
         if save_result and save_result.status in [
@@ -707,19 +1041,27 @@ async def manage_large_mcp_tool_responses_callback(
                 filename = first_artifact.data_filename
                 version = first_artifact.data_version
                 if total_artifacts > 1:
-                    message_parts_for_llm.append(
-                        f"The full response has been saved as {total_artifacts} artifacts, starting with '{filename}' (version {version})."
-                    )
+                    artifact_msg = f"The full response has been saved as {total_artifacts} artifacts, starting with '{filename}' (version {version})."
                 else:
-                    message_parts_for_llm.append(
-                        f"The full response has been saved as artifact '{filename}' (version {version})."
+                    artifact_msg = f"The full response has been saved as artifact '{filename}' (version {version})."
+
+                # When data was too large and truncated, provide explicit guidance
+                if needs_truncation_for_llm:
+                    artifact_msg += (
+                        f' To display this data to the user, use template_liquid with data="{filename}". '
+                        f'To process the data yourself, use load_artifact("{filename}").'
                     )
+                message_parts_for_llm.append(artifact_msg)
             elif save_result.fallback_artifact:
                 filename = save_result.fallback_artifact.data_filename
                 version = save_result.fallback_artifact.data_version
-                message_parts_for_llm.append(
-                    f"The full response has been saved as artifact '{filename}' (version {version})."
-                )
+                artifact_msg = f"The full response has been saved as artifact '{filename}' (version {version})."
+                if needs_truncation_for_llm:
+                    artifact_msg += (
+                        f' To display this data to the user, use template_liquid with data="{filename}". '
+                        f'To process the data yourself, use load_artifact("{filename}").'
+                    )
+                message_parts_for_llm.append(artifact_msg)
 
             log.debug(
                 "%s Added saved artifact details to LLM response.", log_identifier
@@ -744,16 +1086,18 @@ async def manage_large_mcp_tool_responses_callback(
         and save_result.status in [McpSaveStatus.SUCCESS, McpSaveStatus.PARTIAL_SUCCESS]
     ):
         if needs_truncation_for_llm:
-            final_llm_response_dict["status"] = "processed_saved_and_truncated"
+            # Data was too large - withheld from LLM, only available via artifact
+            final_llm_response_dict["status"] = "processed_saved_artifact_only"
         else:
             final_llm_response_dict["status"] = "processed_and_saved"
     elif needs_saving_as_artifact:
         if needs_truncation_for_llm:
-            final_llm_response_dict["status"] = "processed_truncated_save_failed"
+            final_llm_response_dict["status"] = "processed_artifact_only_save_failed"
         else:
             final_llm_response_dict["status"] = "processed_save_failed"
     elif needs_truncation_for_llm:
-        final_llm_response_dict["status"] = "processed_truncated"
+        # This case shouldn't happen (truncation implies saving), but handle it
+        final_llm_response_dict["status"] = "processed_data_withheld"
     else:
         final_llm_response_dict["status"] = "processed"
 
@@ -769,34 +1113,121 @@ async def manage_large_mcp_tool_responses_callback(
     return final_llm_response_dict
 
 
+def _generate_fenced_block_syntax_rules() -> str:
+    """Generates the shared syntax rules for all fenced blocks."""
+    open_delim = ARTIFACT_BLOCK_DELIMITER_OPEN
+    close_delim = ARTIFACT_BLOCK_DELIMITER_CLOSE
+    return f"""
+**Fenced Block Syntax Rules (Applies to `save_artifact` and `template_liquid`):**
+To create content blocks, you MUST use the EXACT syntax shown below.
+
+**EXACT SYNTAX (copy this pattern exactly):**
+{open_delim}keyword: parameter="value" ...
+The content for the block goes here.
+It can span multiple lines.
+{close_delim}
+
+**CRITICAL FORMATTING RULES:**
+  1. The opening delimiter MUST be EXACTLY `{open_delim}`.
+  2. Immediately after the delimiter, write the keyword (`save_artifact` or `template_liquid`) followed by a colon, with NO space before the colon (e.g., `{open_delim}save_artifact:`).
+  3. All parameters (like `filename`, `data`, `mime_type`) must be on the SAME line as the opening delimiter.
+  4. All parameter values **MUST** be enclosed in double quotes (e.g., `filename="example.txt"`).
+  5. You **MUST NOT** use double quotes `"` inside parameter values. Use single quotes or rephrase instead.
+  6. The block's content begins on the line immediately following the parameters.
+  7. Close the block with EXACTLY `{close_delim}` (three angle brackets) on its own line.
+  8. Do NOT surround the block with triple backticks (```). The `{open_delim}` and `{close_delim}` delimiters are sufficient.
+
+**COMMON ERRORS TO AVOID:**
+  ❌ WRONG: `{open_delim[0:1]}template_liquid:` (only 1 angle brackets)
+  ❌ WRONG: `{open_delim[0:2]}save_artifact:` (only 2 angle brackets)
+  ❌ WRONG: `{open_delim}save_artifact` (missing colon)
+  ✅ CORRECT: `{open_delim}save_artifact: filename="test.txt" mime_type="text/plain"`
+"""
+
+
 def _generate_fenced_artifact_instruction() -> str:
     """Generates the instruction text for using fenced artifact blocks."""
     open_delim = ARTIFACT_BLOCK_DELIMITER_OPEN
+    return f"""\
+**Creating Text-Based Artifacts (`{open_delim}save_artifact: ...`):**
+
+When to Create Artifacts:
+Create an artifact when the content provides value as a standalone file, such as:
+- Content with special formatting (HTML, Markdown, CSS).
+- Documents intended for use outside the conversation (reports, emails).
+- Structured reference content (schedules, guides, templates).
+- Substantial text documents or technical documentation.
+
+When NOT to Create Artifacts:
+- Simple answers, explanations, or conversational responses.
+- Brief advice, opinions, or short lists.
+
+Behavior of Created Artifacts:
+- They are sent to the user as an interactive file component.
+- The user can see the content, so there is no need to return or embed it again.
+
+Parameters for `{open_delim}save_artifact: ...`:
+- `filename="your_filename.ext"` (REQUIRED)
+- `mime_type="text/plain"` (optional, defaults to text/plain)
+- `description="A brief description."` (optional)
+
+The system will automatically save the content and confirm it in the next turn.
+"""
+
+
+def _generate_inline_template_instruction() -> str:
+    """Generates the instruction text for using inline Liquid templates."""
+    open_delim = ARTIFACT_BLOCK_DELIMITER_OPEN
     close_delim = ARTIFACT_BLOCK_DELIMITER_CLOSE
     return f"""\
-**Creating Text-Based Artifacts:**
-To create an artifact from content you generate (like code, a report, or a document), you MUST use a special `save_artifact` block. This is the only reliable way to ensure your content is saved correctly.
+**Inline Liquid Templates (`{open_delim}template_liquid: ...`):**
 
-**Syntax:**
-{open_delim}save_artifact: filename="your_filename.ext" mime_type="text/plain" description="A brief description."
-The full content you want to save goes here.
-It can span multiple lines.
+Use inline Liquid templates to dynamically render data from artifacts for user-friendly display. This is faster and more accurate than reading the artifact and reformatting it yourself.
+
+IMPORTANT: Template Format
+- Templates use Liquid template syntax (same as Shopify templates - NOTE that Jekyll extensions are NOT supported).
+
+When to Use Inline Templates:
+- Formatting CSV, JSON, or YAML data into tables or lists.
+- Applying simple transformations (filtering, limiting rows).
+
+Parameters for `{open_delim}template_liquid: ...`:
+- `data="filename.ext"` (REQUIRED): The data artifact to render. Can include version: `data="file.csv:2"`.
+- `jsonpath="$.expression"` (optional): JSONPath to extract a subset of JSON/YAML data.
+- `limit="N"` (optional): Limit to the first N rows (CSV) or items (JSON/YAML arrays).
+
+Data Context for Liquid Templates:
+- CSV data: Available as `headers` (array of column names) and `data_rows` (array of row arrays).
+- JSON/YAML arrays: Available as `items`.
+- JSON/YAML objects: Keys are directly available (e.g., `name`, `email`).
+
+Example - CSV Table:
+{open_delim}template_liquid: data="sales_data.csv" limit="5"
+| {{% for h in headers %}}{{{{ h }}}} | {{% endfor %}}
+|{{% for h in headers %}}---|{{% endfor %}}
+{{% for row in data_rows %}}| {{% for cell in row %}}{{{{ cell }}}} | {{% endfor %}}{{% endfor %}}
 {close_delim}
 
-- **Rules:**
-  - The parameters `filename` and `mime_type` are required. `description` is optional but recommended.
-  - All parameter values **MUST** be enclosed in double quotes.
-  - You **MUST NOT** use double quotes `"` inside the parameter values (e.g., within the description string). Use single quotes or rephrase instead.
-  - Do not surround a save_artifact block with '```' (triple backticks). This will create rendering issues.
+**IMPORTANT - Pipe Characters in Markdown Tables:**
+Text data may contain "|" characters which will break markdown table rendering by pushing data into wrong columns. For text fields that might contain pipes, use the `replace` filter to escape them:
+`{{{{ item.summary | replace: "|", "&#124;" }}}}`
+Only apply this to text fields that might contain pipes - numerical columns don't need it.
 
-The system will automatically save the content and give you a confirmation in the next turn."""
+Negative Examples
+Use {{ issues.size }} instead of {{ issues|length }}
+Use {{ forloop.index }} instead of {{ loop.index }} (Liquid uses forloop not loop)
+Use {{ issue.fields.description | truncate: 200 }} instead of slicing with [:200]
+Do not use Jekyll-specific tags or filters (e.g., `{{% assign %}}`, `{{% capture %}}`, `where`, `sort`, `where_exp`, etc.)
+
+The rendered output will appear inline in your response automatically.
+"""
 
 
 def _generate_artifact_creation_instruction() -> str:
     return """
     **Creating Text-Based Artifacts:**
 
-    **When to Create Text-based Artifacts:**
+    When to Create Text-based Artifacts:
     Create an artifact when the content provides value as a standalone file:
     - Content with special formatting (HTML, Markdown, CSS, structured markup) that requires proper rendering
     - Content explicitly intended for use outside this conversation (reports, emails, presentations, reference documents)
@@ -805,15 +1236,123 @@ def _generate_artifact_creation_instruction() -> str:
     - Substantial text documents
     - Technical documentation meant as reference material
 
-    **When NOT to Create Text-based Artifacts:**
+    When NOT to Create Text-based Artifacts:
     - Simple answers, explanations, or conversational responses
     - Brief advice, opinions, or quick information
-    - Short lists, summaries, or single paragraphs  
+    - Short lists, summaries, or single paragraphs
     - Temporary content only relevant to the immediate conversation
     - Basic explanations that don't require reference material
     """
 
 
+def _generate_examples_instruction() -> str:
+    open_delim = ARTIFACT_BLOCK_DELIMITER_OPEN
+    close_delim = ARTIFACT_BLOCK_DELIMITER_CLOSE
+    embed_open_delim = EMBED_DELIMITER_OPEN
+    embed_close_delim = EMBED_DELIMITER_CLOSE
+
+    return (
+        f"""\
+    Example 1:
+    - User: "Create a markdown file with your two csv files as tables."
+    <note>There are two csv files already uploaded: data1.csv and data2.csv</note>
+    - OrchestratorAgent:
+    {embed_open_delim}status_update:Creating Markdown tables from CSV files...{embed_close_delim}
+    {open_delim}save_artifact: filename="data_tables.md" mime_type="text/markdown" description="Markdown tables from CSV files"
+    # Data Tables
+    ## Data 1
+    {open_delim}template_liquid: data="data1.csv"
+    """
+        + """| {% for h in headers %}{{ h }} | {% endfor %}
+    |{% for h in headers %}---|{% endfor %}
+    {% for row in data_rows %}| {% for cell in row %}{{ cell }} | {% endfor %}{% endfor %}
+    """
+        + f"""{close_delim}
+    ## Data 2
+    {open_delim}template_liquid: data="data2.csv"
+    """
+        + """| {% for h in headers %}{{ h }} | {% endfor %}
+    |{% for h in headers %}---|{% endfor %}
+    {% for row in data_rows %}| {% for cell in row %}{{ cell }} | {% endfor %}{% endfor %}
+    """
+        + f"""{close_delim}
+    {close_delim}
+    Example 2:
+    - User: "Create a text file with the result of sqrt(12345) + sqrt(67890) + sqrt(13579) + sqrt(24680)."
+    - OrchestratorAgent:
+    {embed_open_delim}status_update:Calculating and creating text file...{embed_close_delim}
+    {open_delim}save_artifact: filename="math.txt" mime_type="text/plain" description="Result of sqrt(12345) + sqrt(67890) + sqrt(13579) + sqrt(24680)"
+    result = {embed_open_delim}math: sqrt(12345) + sqrt(67890) + sqrt(13579) + sqrt(24680) | .2f{embed_close_delim}
+    {close_delim}
+    
+    Example 3:
+    - User: "Show me the first 10 entries from data1.csv"
+    - OrchestratorAgent:
+    {embed_open_delim}status_update:Loading CSV data...{embed_close_delim}
+    {open_delim}template_liquid: data="data1.csv" limit="10"
+    """
+        + """| {% for h in headers %}{{ h }} | {% endfor %}
+    |{% for h in headers %}---|{% endfor %}
+    {% for row in data_rows %}| {% for cell in row %}{{ cell }} | {% endfor %}{% endfor %}
+    """
+        + f"""{close_delim}
+
+    Example 4:
+    - User: "Show me the Jira issues as a table"
+    <note>There is a JSON artifact jira_issues.json with items containing key, summary, status, type, assignee, updated fields</note>
+    - OrchestratorAgent:
+    {embed_open_delim}status_update:Rendering Jira issues table...{embed_close_delim}
+    {open_delim}template_liquid: data="jira_issues.json" limit="10"
+    """
+        + """| Key | Summary | Status | Type | Assignee | Updated |
+    |-----|---------|--------|------|----------|---------|
+    {% for item in items %}| [{{ item.key }}](https://jira.example.com/browse/{{ item.key }}) | {{ item.summary | replace: "|", "&#124;" }} | {{ item.status }} | {{ item.type }} | {{ item.assignee }} | {{ item.updated }} |
+    {% endfor %}"""
+        + f"""
+    {close_delim}
+    <note>The replace filter on item.summary escapes any pipe characters that would break the markdown table. Only apply to text fields that might contain pipes.</note>
+
+    Example 5:
+    - User: "Search the database for all orders from last month"
+    - OrchestratorAgent:
+    {embed_open_delim}status_update:Querying order database...{embed_close_delim}
+    [calls search_database tool with no visible text]
+    [After getting results:]
+    Found 247 orders from last month totaling $45,231.
+
+    Example 6:
+    - User: "Create an HTML with the chart image you just generated with the customer data."
+    - OrchestratorAgent:
+    {embed_open_delim}status_update:Generating HTML report with chart...{embed_close_delim}
+    {open_delim}save_artifact: filename="customer_analysis.html" mime_type="text/html" description="Interactive customer analysis dashboard"
+    <!DOCTYPE html>
+    <html>
+    <head>
+        <title>Customer Chart - {embed_open_delim}datetime:%Y-%m-%d{embed_close_delim}</title>
+    """
+        + """
+        <style>
+            body { font-family: Arial, sans-serif; margin: 20px; }
+            .metric { background: #f0f0f0; padding: 10px; margin: 10px 0; }
+            img { max-width: 100%; height: auto; }
+    """
+        + f"""    </style>
+        </head>
+    <body>
+    <h1>Customer Analysis Report</h1>
+    <p>Generated: {embed_open_delim}datetime:iso{embed_close_delim}</p>
+        
+    <h2>Customer Distribution Chart</h2>
+    <img src="{embed_open_delim}artifact_content:customer_chart.png >>> format:datauri{embed_close_delim}" alt="Customer Distribution">
+    
+    </body>
+    </html>
+    {close_delim}
+
+    """
+    )
+
+
 def _generate_embed_instruction(
     include_artifact_content: bool,
     log_identifier: str,
@@ -823,12 +1362,18 @@ def _generate_embed_instruction(
     close_delim = EMBED_DELIMITER_CLOSE
     chain_delim = EMBED_CHAIN_DELIMITER
     early_types = "`math`, `datetime`, `uuid`, `artifact_meta`"
-    modifier_list = ", ".join(
-        [f"`{prefix}`" for prefix in MODIFIER_IMPLEMENTATIONS.keys()]
-    )
+
+    modifier_list = MODIFIER_IMPLEMENTATIONS.keys()
+    # Remove apply_to_template from the modifier list as it's been deprecated
+    if "apply_to_template" in modifier_list:
+        modifier_list = list(modifier_list)
+        modifier_list.remove("apply_to_template")
+    modifier_list = ", ".join([f"`{prefix}`" for prefix in modifier_list])
 
     base_instruction = f"""\
-You can use dynamic embeds in your text responses and tool parameters using the syntax {open_delim}type:expression {chain_delim} format{close_delim}. This allows you to
+**Using Dynamic Embeds in Responses:**
+
+You can use dynamic embeds in your text responses and tool parameters using the syntax {open_delim}type:expression {chain_delim} format{close_delim}. NOTE that this differs from 'save_artifact', which has  different delimiters. This allows you to
 always have correct information in your output. Specifically, make sure you always use embeds for math, even if it is simple. You will make mistakes if you try to do math yourself.
 Use HTML entities to escape the delimiters.
 This host resolves the following embed types *early* (before sending to the LLM or tool): {early_types}. This means the embed is replaced with its resolved value.
@@ -836,31 +1381,59 @@ This host resolves the following embed types *early* (before sending to the LLM
 - `{open_delim}datetime:format_or_keyword{close_delim}`: Inserts current date/time. Use Python strftime format (e.g., `%Y-%m-%d`) or keywords (`iso`, `timestamp`, `date`, `time`, `now`).
 - `{open_delim}uuid:{close_delim}`: Inserts a random UUID.
 - `{open_delim}artifact_meta:filename[:version]{close_delim}`: Inserts a summary of the artifact's metadata (latest version if unspecified).
-- `{open_delim}status_update:Your message here{close_delim}`: Generates an immediate, distinct status message event that is displayed to the user (e.g., 'Thinking...', 'Searching database...'). This message appears in a status area, not as part of the main chat conversation. Use this to provide interim feedback during processing."""
+- `{open_delim}status_update:Your message here{close_delim}`: Generates an immediate, distinct status message event that is displayed to the user (e.g., 'Thinking...', 'Searching database...'). This message appears in a status area, not as part of the main chat conversation. Use this to provide interim feedback during processing.
+
+Examples:
+- `{open_delim}status_update:Analyzing data...{close_delim}` (Shows 'Analyzing data...' as a status update)
+- `The result of 23.5 * 4.2 is {open_delim}math:23.5 * 4.2 | .2f{close_delim}` (Embeds calculated result with 2 decimal places)
+
+The following embeds are resolved *late* (by the gateway before final display):
+- `{open_delim}artifact_return:filename[:version]{close_delim}`: Attaches an artifact to your message so the user receives the file. The embed itself is removed from the text.
+
+  **CRITICAL - Returning Artifacts to Users:**
+  Only artifacts created with the `{open_delim}save_artifact:...{close_delim}` fenced block syntax are automatically sent to the user.
+
+  **You MUST use artifact_return for:**
+  - Artifacts created by tools (e.g., image generation, chart creation, file conversion)
+  - Artifacts created by other agents you called
+  - Artifacts from MCP servers
+
+  **When deciding whether to return an artifact:**
+  - Return artifacts the user explicitly requested or that answer their question
+  - Return final outputs (charts, reports, images, documents)
+  - Do NOT return intermediate/temporary artifacts (e.g., temp files, internal data)
+
+  **Example - Tool creates an image:**
+  User: "Create a chart of sales data"
+  [You call a charting tool that creates sales_chart.png]
+  Your response: "Here's the sales chart. {open_delim}artifact_return:sales_chart.png{close_delim}"
+
+  **Example - Agent creates a report:**
+  User: "Generate a quarterly report"
+  [You call ReportAgent which creates quarterly_report.pdf]
+  Your response: "The quarterly report is ready. {open_delim}artifact_return:quarterly_report.pdf{close_delim}"
+"""
 
     artifact_content_instruction = f"""
 - `{open_delim}artifact_content:filename[:version] {chain_delim} modifier1:value1 {chain_delim} ... {chain_delim} format:output_format{close_delim}`: Embeds artifact content after applying a chain of modifiers. This is resolved *late* (typically by a gateway before final display).
+    - If this embed resolves to binary content (like an image), it will be automatically converted into an attached file, similar to `artifact_return`.
     - Use `{chain_delim}` to separate the artifact identifier from the modifier steps and the final format step.
     - Available modifiers: {modifier_list}.
     - The `format:output_format` step *must* be the last step in the chain. Supported formats include `text`, `datauri`, `json`, `json_pretty`, `csv`. Formatting as datauri, will include the data URI prefix, so do not add it yourself.
     - Use `artifact_meta` first to check size; embedding large files may fail.
-    - **Using `apply_to_template` Modifier:**
-        - This modifier renders a Mustache template artifact using the data from the previous step.
-        - **Data Context:**
-            - If the input data's original MIME type was `text/csv` or `application/csv`, it's automatically parsed into an object with two keys: `headers` (a list of column name strings) and `data_rows` (a list of lists, where each inner list contains the string values for a row). Example template usage: `<thead><tr>{{{{#headers}}}}<th>{{{{.}}}}</th>{{{{/headers}}}}</tr></thead><tbody>{{{{#data_rows}}}}<tr>{{{{#.}}}}<td>{{{{.}}}}</td>{{{{/.}}}}</tr>{{{{/data_rows}}}}</tbody>`. If CSV parsing fails, the raw string content is available under `text`.
-            - If the input data is a **list** (e.g., from `jsonpath` or a JSON array), it's available under `items`.
-            - If the input data is a **dictionary** (e.g., from a JSON object), its keys are directly available (e.g., `{{{{key1}}}}`).
-            - If the input data is a **plain string** (and not auto-parsed as CSV), it's available under `text`.
-        - The template filename can include a version (e.g., `template.mustache:2`). Defaults to latest.
-        - The template itself can contain `«artifact_content:...»` embeds, which will be resolved before rendering.
+    - Efficient workflows for large artifacts:
+        - To extract specific line ranges: `load_artifact(filename, version, include_line_numbers=True)` to identify lines, then use `slice_lines:start:end` modifier to extract that range.
+        - To fill templates with many placeholders: use `artifact_search_and_replace_regex` with `replacements` array (single atomic operation instead of multiple calls).
+        - Line numbers are display-only; `slice_lines` always operates on original content.
     - Examples:
         - `<img src="{open_delim}artifact_content:image.png {chain_delim} format:datauri{close_delim}`"> (Embed image as data URI - NOTE that this includes the datauri prefix. Do not add it yourself.)
         - `{open_delim}artifact_content:data.json {chain_delim} jsonpath:$.items[*] {chain_delim} select_fields:name,status {chain_delim} format:json_pretty{close_delim}` (Extract and format JSON fields)
         - `{open_delim}artifact_content:logs.txt {chain_delim} grep:ERROR {chain_delim} head:10 {chain_delim} format:text{close_delim}` (Get first 10 error lines)
-        - `{open_delim}artifact_content:products.csv {chain_delim} apply_to_template:product_table.html.mustache {chain_delim} format:text{close_delim}` (CSV is auto-parsed to `headers` and `data_rows` for the HTML template)
         - `{open_delim}artifact_content:config.json {chain_delim} jsonpath:$.userPreferences.theme {chain_delim} format:text{close_delim}` (Extract a single value from a JSON artifact)
-        - `{open_delim}artifact_content:sensor_readings.csv {chain_delim} filter_rows_eq:status:critical {chain_delim} select_cols:timestamp,sensor_id,value {chain_delim} format:csv{close_delim}` (Filter critical sensor readings and select specific columns, output as CSV)
-        - `{open_delim}artifact_content:server.log {chain_delim} tail:100 {chain_delim} grep:WARN {chain_delim} format:text{close_delim}` (Get warning lines from the last 100 lines of a log file)"""
+        - `{open_delim}artifact_content:server.log {chain_delim} tail:100 {chain_delim} grep:WARN {chain_delim} format:text{close_delim}` (Get warning lines from the last 100 lines of a log file)
+        - `{open_delim}artifact_content:template.html {chain_delim} slice_lines:10:50 {chain_delim} format:text{close_delim}` (Extract lines 10-50 from a large file)
+        - `<img src="{open_delim}artifact_content:diagram.png {chain_delim} format:datauri{close_delim}`"> (Embed an PNG diagram as a data URI)`
+"""
 
     final_instruction = base_instruction
     if include_artifact_content:
@@ -872,6 +1445,64 @@ Ensure the syntax is exactly `{open_delim}type:expression{close_delim}` or `{ope
     return final_instruction
 
 
+def _generate_conversation_flow_instruction() -> str:
+    """Generates instruction text for conversation flow and response formatting."""
+    open_delim = EMBED_DELIMITER_OPEN
+    close_delim = EMBED_DELIMITER_CLOSE
+    return f"""\
+**Conversation Flow and Response Formatting:**
+
+**CRITICAL: Minimize Narration - Maximize Results**
+
+You do NOT need to produce visible text on every turn. Many turns should contain ONLY status updates and tool calls, with NO visible text at all.
+Only produce visible text when you have actual results, answers, or insights to share with the user.
+
+Response Content Rules:
+1. Visible responses should contain ONLY:
+   - Direct answers to the user's question
+   - Analysis and insights derived from tool results
+   - Final results and data
+   - Follow-up questions when needed
+   - Plans for complex multi-step tasks
+
+2. DO NOT include visible text for:
+   - Process narration ("Let me...", "I'll...", "Now I will...")
+   - Acknowledgments of tool calls ("I'm calling...", "Searching...")
+   - Descriptions of what you're about to do
+   - Play-by-play commentary on your actions
+   - Transitional phrases between tool calls
+
+3. Use invisible status_update embeds for ALL process updates:
+   - "Searching for..."
+   - "Analyzing..."
+   - "Creating..."
+   - "Querying..."
+   - "Calling agent X..."
+
+4. NEVER mix process narration with status updates - if you use a status_update embed, do NOT repeat that information in visible text.
+
+Examples:
+
+**Excellent (no visible text, just status and tools):**
+"{open_delim}status_update:Retrieving sales data...{close_delim}" [then calls tool, no visible text]
+
+**Good (visible text only contains results):**
+"{open_delim}status_update:Analyzing Q4 sales...{close_delim}" [calls tool]
+"Sales increased 23% in Q4, driven primarily by enterprise accounts."
+
+**Bad (unnecessary narration):**
+"Let me retrieve the sales data for you." [then calls tool]
+
+**Bad (narration mixed with results):**
+"I've analyzed the data and found that sales increased 23% in Q4."
+
+**Bad (play-by-play commentary):**
+"Now I'll search for the information. After that I'll analyze it."
+
+Remember: The user can see status updates and tool calls. You don't need to announce them in visible text.
+"""
+
+
 def _generate_tool_instructions_from_registry(
     active_tools: List[BuiltinTool],
     log_identifier: str,
@@ -882,6 +1513,10 @@ def _generate_tool_instructions_from_registry(
 
     instructions_by_category = defaultdict(list)
     for tool in sorted(active_tools, key=lambda t: (t.category, t.name)):
+        # Skip internal tools (those starting with underscore)
+        if tool.name.startswith("_"):
+            continue
+
         param_parts = []
         if tool.parameters and tool.parameters.properties:
             for name, schema in tool.parameters.properties.items():
@@ -937,27 +1572,51 @@ def inject_dynamic_instructions_callback(
 Parallel Tool Calling:
 The system is capable of calling multiple tools in parallel to speed up processing. Please try to run tools in parallel when they don't depend on each other. This saves money and time, providing faster results to the user.
 
+**Response Formatting - CRITICAL**:
+In most cases when calling tools, you should produce NO visible text at all - only status_update embeds and the tool calls themselves.
+The user can see your tool calls and status updates, so narrating your actions is redundant and creates noise.
+
+If you do include visible text:
+- It must contain actual results, insights, or answers - NOT process narration
+- Do NOT end with a colon (":") before tool calls, as this leaves it hanging
+- Prefer ending with a period (".") if you must include visible text
+
+Examples:
+ - BEST: "{open_delim}status_update:Searching database...{close_delim}" [then calls tool, NO visible text]
+ - BAD: "Let me search for that information." [then calls tool]
+ - BAD: "Searching for information..." [then calls tool]
+
+**CRITICAL - No Links From Training Data**:
+- DO NOT include URLs, links, or markdown links from your training data in responses
+- NEVER include markdown links like [text](url) or raw URLs like https://example.com unless they came from a tool result
+- If a delegated agent's response contains [[cite:searchN]] citations, those are properly formatted - preserve them exactly
+- If a delegated agent's response has no links, do NOT add any links yourself
+- The ONLY acceptable links are those returned by tools (web search, deep research, etc.) with proper citation format
+- Your role is to coordinate and present results, not to augment them with links from your training data
+
 Embeds in responses from agents:
-To be efficient, agents may response with artifact_content embeds in their responses. These will not be resolved until they are sent back to a gateway. If it makes
+To be efficient, peer agents may respond with artifact_content in their responses. These will not be resolved until they are sent back to a gateway. If it makes
 sense, just carry that embed forward to your response to the user. For example, if you ask for an org chart from another agent and its response contains an embed like
 `{open_delim}artifact_content:org_chart.md{close_delim}`, you can just include that embed in your response to the user. The gateway will resolve it and display the org chart.
 
+Similarly, template_liquid blocks in peer agent responses can be carried forward to your response to the user for resolution by the gateway.
+
 When faced with a complex goal or request that involves multiple steps, data retrieval, or artifact summarization to produce a new report or document, you MUST first create a plan.
 Simple, direct requests like 'create an image of a dog' or 'write an email to thank my boss' do not require a plan.
 
 If a plan is created:
 1. It should be a terse, hierarchical list describing the steps needed, with each checkbox item on its own line.
-2. Use '☐' (empty checkbox emoji) for pending items and '☑' (checked checkbox emoji) for completed items.
+2. Use '⬜' for pending items, '✅' for completed items, and '❌' for cancelled items.
 3. If the plan changes significantly during execution, restate the updated plan.
 4. As items are completed, update the plan to check them off.
 
 """
     injected_instructions.append(planning_instruction)
-    log.debug("%s Added hardcoded planning instructions.", log_identifier)
-    artifact_creation_instruction = _generate_artifact_creation_instruction()
-    injected_instructions.append(artifact_creation_instruction)
-    fenced_artifact_instruction = _generate_fenced_artifact_instruction()
-    injected_instructions.append(fenced_artifact_instruction)
+
+    # Add the consolidated block instructions
+    injected_instructions.append(_generate_fenced_artifact_instruction())
+    injected_instructions.append(_generate_inline_template_instruction())
+    injected_instructions.append(_generate_fenced_block_syntax_rules())
 
     agent_instruction_str: Optional[str] = None
     if host_component._agent_system_instruction_callback:
@@ -1025,6 +1684,11 @@ If a plan is created:
                 include_artifact_content_instr,
             )
 
+        instruction = _generate_conversation_flow_instruction()
+        if instruction:
+            injected_instructions.append(instruction)
+            log.debug("%s Prepared conversation flow instructions.", log_identifier)
+
     if active_builtin_tools:
         instruction = _generate_tool_instructions_from_registry(
             active_builtin_tools, log_identifier
@@ -1094,6 +1758,8 @@ If a plan is created:
             e_last_call,
         )
 
+    injected_instructions.append(_generate_examples_instruction())
+
     if injected_instructions:
         combined_instructions = "\n\n---\n\n".join(injected_instructions)
         if llm_request.config is None:
@@ -1612,9 +2278,11 @@ def notify_tool_invocation_start_callback(
             tool_args=serializable_args,
             function_call_id=tool_context.function_call_id,
         )
-        asyncio.run_coroutine_threadsafe(
-            _publish_data_part_status_update(host_component, a2a_context, tool_data),
-            host_component.get_async_loop(),
+        host_component.publish_data_signal_from_thread(
+            a2a_context=a2a_context,
+            signal_data=tool_data,
+            skip_buffer_flush=False,
+            log_identifier=log_identifier,
         )
         log.debug(
             "%s Scheduled tool_invocation_start notification.",
@@ -1685,9 +2353,11 @@ def notify_tool_execution_result_callback(
             result_data=serializable_response,
             function_call_id=tool_context.function_call_id,
         )
-        asyncio.run_coroutine_threadsafe(
-            _publish_data_part_status_update(host_component, a2a_context, tool_data),
-            host_component.get_async_loop(),
+        host_component.publish_data_signal_from_thread(
+            a2a_context=a2a_context,
+            signal_data=tool_data,
+            skip_buffer_flush=False,
+            log_identifier=log_identifier,
         )
         log.debug(
             "%s Scheduled tool_result notification for function call ID %s.",
@@ -1785,3 +2455,76 @@ def auto_continue_on_max_tokens_callback(
     )
 
     return hijacked_response
+
+
+def preregister_long_running_tools_callback(
+    callback_context: CallbackContext,
+    llm_response: LlmResponse,
+    host_component: "SamAgentComponent",
+) -> Optional[LlmResponse]:
+    """
+    ADK after_model_callback to pre-register all long-running tool calls
+    before any tool execution begins. This prevents race conditions where
+    one tool completes before another has registered.
+
+    The race condition occurs because tools are executed via asyncio.gather
+    (non-deterministic order) and each tool calls register_parallel_call_sent()
+    inside its run_async(). If Tool A completes before Tool B even registers,
+    the system thinks all calls are done (completed=1, total=1).
+
+    By pre-registering all long-running tools in this callback (which runs
+    BEFORE tool execution), we ensure the total count is set correctly upfront.
+    """
+    log_identifier = "[Callback:PreregisterLongRunning]"
+
+    # Only process non-partial responses with function calls
+    if llm_response.partial:
+        return None
+
+    if not llm_response.content or not llm_response.content.parts:
+        return None
+
+    # Find all long-running tool calls (identified by peer_ prefix)
+    long_running_calls = []
+    for part in llm_response.content.parts:
+        if part.function_call:
+            tool_name = part.function_call.name
+            if tool_name.startswith(PEER_TOOL_PREFIX):
+                long_running_calls.append(part.function_call)
+
+    if not long_running_calls:
+        return None
+
+    # Get task context
+    a2a_context = callback_context.state.get("a2a_context")
+    if not a2a_context:
+        log.warning("%s No a2a_context, cannot pre-register tools", log_identifier)
+        return None
+
+    logical_task_id = a2a_context.get("logical_task_id")
+    invocation_id = callback_context._invocation_context.invocation_id
+
+    with host_component.active_tasks_lock:
+        task_context = host_component.active_tasks.get(logical_task_id)
+
+    if not task_context:
+        log.warning(
+            "%s TaskContext not found for %s, cannot pre-register",
+            log_identifier,
+            logical_task_id,
+        )
+        return None
+
+    # Pre-register ALL long-running calls atomically
+    for fc in long_running_calls:
+        task_context.register_parallel_call_sent(invocation_id)
+
+    log.info(
+        "%s Pre-registered %d long-running tool call(s) for invocation %s (task %s)",
+        log_identifier,
+        len(long_running_calls),
+        invocation_id,
+        logical_task_id,
+    )
+
+    return None  # Don't alter the response