nvidia-nat 1.4.0a20251112__py3-none-any.whl → 1.4.0a20260113__py3-none-any.whl

nat/front_ends/fastapi/fastapi_front_end_plugin_worker.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -39,6 +39,9 @@ from pydantic import BaseModel
 from pydantic import Field
 from starlette.websockets import WebSocket
 
+from nat.builder.context import Context
+from nat.builder.eval_builder import WorkflowEvalBuilder
+from nat.builder.evaluator import EvaluatorInfo
 from nat.builder.function import Function
 from nat.builder.workflow_builder import WorkflowBuilder
 from nat.data_models.api_server import ChatRequest
@@ -51,11 +54,15 @@ from nat.data_models.object_store import NoSuchKeyError
 from nat.eval.config import EvaluationRunOutput
 from nat.eval.evaluate import EvaluationRun
 from nat.eval.evaluate import EvaluationRunConfig
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.front_ends.fastapi.async_job import run_generation
 from nat.front_ends.fastapi.auth_flow_handlers.http_flow_handler import HTTPAuthenticationFlowHandler
 from nat.front_ends.fastapi.auth_flow_handlers.websocket_flow_handler import FlowState
 from nat.front_ends.fastapi.auth_flow_handlers.websocket_flow_handler import WebSocketAuthenticationFlowHandler
 from nat.front_ends.fastapi.fastapi_front_end_config import AsyncGenerateResponse
 from nat.front_ends.fastapi.fastapi_front_end_config import AsyncGenerationStatusResponse
+from nat.front_ends.fastapi.fastapi_front_end_config import EvaluateItemRequest
+from nat.front_ends.fastapi.fastapi_front_end_config import EvaluateItemResponse
 from nat.front_ends.fastapi.fastapi_front_end_config import EvaluateRequest
 from nat.front_ends.fastapi.fastapi_front_end_config import EvaluateResponse
 from nat.front_ends.fastapi.fastapi_front_end_config import EvaluateStatusResponse
@@ -69,6 +76,7 @@ from nat.front_ends.fastapi.utils import get_config_file_path
 from nat.object_store.models import ObjectStoreItem
 from nat.runtime.loader import load_workflow
 from nat.runtime.session import SessionManager
+from nat.utils.log_utils import setup_logging
 
 logger = logging.getLogger(__name__)
 
@@ -100,6 +108,9 @@ class FastApiFrontEndPluginWorkerBase(ABC):
         self._scheduler_address = os.environ.get("NAT_DASK_SCHEDULER_ADDRESS")
         self._db_url = os.environ.get("NAT_JOB_STORE_DB_URL")
         self._config_file_path = get_config_file_path()
+        self._use_dask_threads = os.environ.get("NAT_USE_DASK_THREADS", "0") == "1"
+        self._log_level = int(os.environ.get("NAT_FASTAPI_LOG_LEVEL", logging.INFO))
+        setup_logging(self._log_level)
 
         if self._scheduler_address is not None:
             if not _DASK_AVAILABLE:
@@ -227,6 +238,78 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
         self._outstanding_flows: dict[str, FlowState] = {}
         self._outstanding_flows_lock = asyncio.Lock()
 
+        # Track session managers for each route
+        self._session_managers: list[SessionManager] = []
+
+        # Evaluator storage for single-item evaluation
+        self._evaluators: dict[str, EvaluatorInfo] = {}
+        self._eval_builder: WorkflowEvalBuilder | None = None
+
+    async def initialize_evaluators(self, config: Config):
+        """Initialize and store evaluators from config for single-item evaluation."""
+        if not config.eval or not config.eval.evaluators:
+            logger.info("No evaluators configured, skipping evaluator initialization")
+            return
+
+        try:
+            # Build evaluators using WorkflowEvalBuilder (same pattern as nat eval)
+            # Start with registry=None and let populate_builder set everything up
+            self._eval_builder = WorkflowEvalBuilder(general_config=config.general,
+                                                     eval_general_config=config.eval.general,
+                                                     registry=None)
+
+            # Enter the async context and keep it alive
+            await self._eval_builder.__aenter__()
+
+            # Populate builder with config (this sets up LLMs, functions, etc.)
+            # Skip workflow build since we already have it from the main builder
+            await self._eval_builder.populate_builder(config, skip_workflow=True)
+
+            # Now evaluators should be populated by populate_builder
+            for name in config.eval.evaluators.keys():
+                self._evaluators[name] = self._eval_builder.get_evaluator(name)
+                logger.info(f"Initialized evaluator: {name}")
+
+            logger.info(f"Successfully initialized {len(self._evaluators)} evaluators")
+
+        except Exception as e:
+            logger.error(f"Failed to initialize evaluators: {e}")
+            # Don't fail startup, just log the error
+            self._evaluators = {}
+
+    async def _create_session_manager(self,
+                                      builder: WorkflowBuilder,
+                                      entry_function: str | None = None) -> SessionManager:
+        """Create and register a SessionManager."""
+
+        sm = await SessionManager.create(config=self._config, shared_builder=builder, entry_function=entry_function)
+        self._session_managers.append(sm)
+
+        return sm
+
+    async def cleanup_session_managers(self):
+        """Clean up all SessionManager resources on shutdown."""
+        for sm in self._session_managers:
+            try:
+                await sm.shutdown()
+            except Exception as e:
+                logger.error(f"Error cleaning up SessionManager: {e}")
+
+        self._session_managers.clear()
+        logger.info("All SessionManagers cleaned up")
+
+    async def cleanup_evaluators(self):
+        """Clean up evaluator resources on shutdown."""
+        if self._eval_builder:
+            try:
+                await self._eval_builder.__aexit__(None, None, None)
+                logger.info("Evaluator builder context cleaned up")
+            except Exception as e:
+                logger.error(f"Error cleaning up evaluator builder: {e}")
+            finally:
+                self._eval_builder = None
+                self._evaluators.clear()
+
     def get_step_adaptor(self) -> StepAdaptor:
 
         return StepAdaptor(self.front_end_config.step_adaptor)
@@ -236,21 +319,34 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
         # Do things like setting the base URL and global configuration options
         app.root_path = self.front_end_config.root_path
 
+        # Initialize evaluators for single-item evaluation
+        # TODO: we need config control over this as it's not always needed
+        await self.initialize_evaluators(self._config)
+
+        # Ensure session manager resources are cleaned up when the app shuts down
+        app.add_event_handler("shutdown", self.cleanup_session_managers)
+
+        # Ensure evaluator resources are cleaned up when the app shuts down
+        app.add_event_handler("shutdown", self.cleanup_evaluators)
+
         await self.add_routes(app, builder)
 
     async def add_routes(self, app: FastAPI, builder: WorkflowBuilder):
 
-        await self.add_default_route(app, SessionManager(await builder.build()))
-        await self.add_evaluate_route(app, SessionManager(await builder.build()))
+        await self.add_default_route(app, await self._create_session_manager(builder))
+        await self.add_evaluate_route(app, await self._create_session_manager(builder))
+        await self.add_evaluate_item_route(app, await self._create_session_manager(builder))
+
         await self.add_static_files_route(app, builder)
         await self.add_authorization_route(app)
         await self.add_mcp_client_tool_list_route(app, builder)
+        await self.add_monitor_route(app)
 
         for ep in self.front_end_config.endpoints:
 
-            entry_workflow = await builder.build(entry_function=ep.function_name)
-
-            await self.add_route(app, endpoint=ep, session_manager=SessionManager(entry_workflow))
+            await self.add_route(app,
+                                 endpoint=ep,
+                                 session_manager=await self._create_session_manager(builder, ep.function_name))
 
     async def add_default_route(self, app: FastAPI, session_manager: SessionManager):
 
@@ -439,6 +535,69 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
             else:
                 logger.warning("Dask is not available, evaluation endpoints will not be added.")
 
+    async def add_evaluate_item_route(self, app: FastAPI, session_manager: SessionManager):
+        """Add the single-item evaluation endpoint to the FastAPI app."""
+
+        async def evaluate_single_item(request: EvaluateItemRequest, http_request: Request) -> EvaluateItemResponse:
+            """Handle single-item evaluation requests."""
+
+            async with session_manager.session(http_connection=http_request):
+
+                # Check if evaluator exists
+                if request.evaluator_name not in self._evaluators:
+                    raise HTTPException(status_code=404,
+                                        detail=f"Evaluator '{request.evaluator_name}' not found. "
+                                        f"Available evaluators: {list(self._evaluators.keys())}")
+
+                try:
+                    # Get the evaluator
+                    evaluator = self._evaluators[request.evaluator_name]
+
+                    # Run evaluation on single item
+                    result = await evaluator.evaluate_fn(EvalInput(eval_input_items=[request.item]))
+
+                    # Extract the single output item
+                    if result.eval_output_items:
+                        output_item = result.eval_output_items[0]
+                        return EvaluateItemResponse(success=True, result=output_item, error=None)
+                    else:
+                        return EvaluateItemResponse(success=False, result=None, error="Evaluator returned no results")
+
+                except Exception as e:
+                    logger.exception(f"Error evaluating item with {request.evaluator_name}")
+                    return EvaluateItemResponse(success=False, result=None, error=f"Evaluation failed: {str(e)}")
+
+        # Register the route
+        if self.front_end_config.evaluate_item.path:
+            app.add_api_route(path=self.front_end_config.evaluate_item.path,
+                              endpoint=evaluate_single_item,
+                              methods=[self.front_end_config.evaluate_item.method],
+                              response_model=EvaluateItemResponse,
+                              description=self.front_end_config.evaluate_item.description,
+                              responses={
+                                  404: {
+                                      "description": "Evaluator not found",
+                                      "content": {
+                                          "application/json": {
+                                              "example": {
+                                                  "detail": "Evaluator 'unknown' not found"
+                                              }
+                                          }
+                                      }
+                                  },
+                                  500: {
+                                      "description": "Internal Server Error",
+                                      "content": {
+                                          "application/json": {
+                                              "example": {
+                                                  "detail": "Internal server error occurred"
+                                              }
+                                          }
+                                      }
+                                  }
+                              })
+            logger.info(f"Added evaluate_item route at {self.front_end_config.evaluate_item.path}")
+
     async def add_static_files_route(self, app: FastAPI, builder: WorkflowBuilder):
 
         if not self.front_end_config.object_store:
@@ -538,11 +697,15 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
                         endpoint: FastApiFrontEndConfig.EndpointBase,
                         session_manager: SessionManager):
 
-        workflow = session_manager.workflow
+        GenerateBodyType = session_manager.get_workflow_input_schema()
+        GenerateStreamResponseType = session_manager.get_workflow_streaming_output_schema()
+        GenerateSingleResponseType = session_manager.get_workflow_single_output_schema()
 
-        GenerateBodyType = workflow.input_schema
-        GenerateStreamResponseType = workflow.streaming_output_schema
-        GenerateSingleResponseType = workflow.single_output_schema
+        def add_context_headers_to_response(response: Response) -> None:
+            """Add context-based headers to response if available."""
+            observability_trace_id = Context.get().observability_trace_id
+            if observability_trace_id:
+                response.headers["Observability-Trace-Id"] = observability_trace_id
 
         # Skip async generation for custom routes (those with function_name)
         if self._dask_available and not hasattr(endpoint, 'function_name'):
@@ -591,10 +754,13 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
 
                 response.headers["Content-Type"] = "application/json"
 
-                async with session_manager.session(http_connection=request,
-                                                   user_authentication_callback=self._http_flow_handler.authenticate):
+                async with session_manager.session(
+                        http_connection=request,
+                        user_authentication_callback=self._http_flow_handler.authenticate) as session:
 
-                    return await generate_single_response(None, session_manager, result_type=result_type)
+                    result = await generate_single_response(None, session, result_type=result_type)
+                    add_context_headers_to_response(response)
+                    return result
 
             return get_single
 
@@ -602,13 +768,14 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
 
             async def get_stream(request: Request):
 
-                async with session_manager.session(http_connection=request,
-                                                   user_authentication_callback=self._http_flow_handler.authenticate):
+                async with session_manager.session(
+                        http_connection=request,
+                        user_authentication_callback=self._http_flow_handler.authenticate) as session:
 
                     return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
                                              content=generate_streaming_response_as_str(
                                                  None,
-                                                 session_manager=session_manager,
+                                                 session=session,
                                                  streaming=streaming,
                                                  step_adaptor=self.get_step_adaptor(),
                                                  result_type=result_type,
@@ -620,14 +787,14 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
 
             async def get_stream(filter_steps: str | None = None):
 
-                return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
-                                         content=generate_streaming_response_full_as_str(
-                                             None,
-                                             session_manager=session_manager,
-                                             streaming=streaming,
-                                             result_type=result_type,
-                                             output_type=output_type,
-                                             filter_steps=filter_steps))
+                async with session_manager.session(http_connection=None) as session:
+                    return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
+                                             content=generate_streaming_response_full_as_str(None,
+                                                                                             session=session,
+                                                                                             streaming=streaming,
+                                                                                             result_type=result_type,
+                                                                                             output_type=output_type,
+                                                                                             filter_steps=filter_steps))
 
             return get_stream
 
@@ -637,10 +804,13 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
 
                 response.headers["Content-Type"] = "application/json"
 
-                async with session_manager.session(http_connection=request,
-                                                   user_authentication_callback=self._http_flow_handler.authenticate):
+                async with session_manager.session(
+                        http_connection=request,
+                        user_authentication_callback=self._http_flow_handler.authenticate) as session:
 
-                    return await generate_single_response(payload, session_manager, result_type=result_type)
+                    result = await generate_single_response(payload, session, result_type=result_type)
+                    add_context_headers_to_response(response)
+                    return result
 
             return post_single
 
@@ -651,13 +821,14 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
 
             async def post_stream(request: Request, payload: request_type):
 
-                async with session_manager.session(http_connection=request,
-                                                   user_authentication_callback=self._http_flow_handler.authenticate):
+                async with session_manager.session(
+                        http_connection=request,
+                        user_authentication_callback=self._http_flow_handler.authenticate) as session:
 
                     return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
                                              content=generate_streaming_response_as_str(
                                                  payload,
-                                                 session_manager=session_manager,
+                                                 session=session,
                                                  streaming=streaming,
                                                  step_adaptor=self.get_step_adaptor(),
                                                  result_type=result_type,
@@ -675,14 +846,14 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
 
             async def post_stream(payload: request_type, filter_steps: str | None = None):
 
-                return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
-                                         content=generate_streaming_response_full_as_str(
-                                             payload,
-                                             session_manager=session_manager,
-                                             streaming=streaming,
-                                             result_type=result_type,
-                                             output_type=output_type,
-                                             filter_steps=filter_steps))
+                async with session_manager.session(http_connection=None) as session:
+                    return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
+                                             content=generate_streaming_response_full_as_str(payload,
+                                                                                             session=session,
+                                                                                             streaming=streaming,
+                                                                                             result_type=result_type,
+                                                                                             output_type=output_type,
+                                                                                             filter_steps=filter_steps))
 
             return post_stream
 
@@ -698,20 +869,22 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
                 response.headers["Content-Type"] = "application/json"
                 stream_requested = getattr(payload, 'stream', False)
 
-                async with session_manager.session(http_connection=request):
+                async with session_manager.session(http_connection=request) as session:
                     if stream_requested:
 
                         # Return streaming response
                         return StreamingResponse(headers={"Content-Type": "text/event-stream; charset=utf-8"},
                                                  content=generate_streaming_response_as_str(
                                                      payload,
-                                                     session_manager=session_manager,
+                                                     session=session,
                                                      streaming=True,
                                                      step_adaptor=self.get_step_adaptor(),
                                                      result_type=ChatResponseChunk,
                                                      output_type=ChatResponseChunk))
 
-                    return await generate_single_response(payload, session_manager, result_type=ChatResponse)
+                    result = await generate_single_response(payload, session, result_type=ChatResponse)
+                    add_context_headers_to_response(response)
+                    return result
 
             return post_openai_api_compatible
 
@@ -732,23 +905,6 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
                                                  updated_at=job.updated_at,
                                                  expires_at=self._job_store.get_expires_at(job))
 
-        async def run_generation(scheduler_address: str,
-                                 db_url: str,
-                                 config_file_path: str,
-                                 job_id: str,
-                                 payload: typing.Any):
-            """Background task to run the workflow."""
-            job_store = JobStore(scheduler_address=scheduler_address, db_url=db_url)
-            try:
-                async with load_workflow(config_file_path) as local_session_manager:
-                    result = await generate_single_response(
-                        payload, local_session_manager, result_type=local_session_manager.workflow.single_output_schema)
-
-                await job_store.update_status(job_id, JobStatus.SUCCESS, output=result)
-            except Exception as e:
-                logger.exception("Error in async job %s", job_id)
-                await job_store.update_status(job_id, JobStatus.FAILURE, error=str(e))
-
         def post_async_generation(request_type: type):
 
             async def start_async_generation(
@@ -771,6 +927,8 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
                         job_fn=run_generation,
                         sync_timeout=request.sync_timeout,
                         job_args=[
+                            not self._use_dask_threads,
+                            self._log_level,
                             self._scheduler_address,
                             self._db_url,
                             self._config_file_path,
@@ -1104,7 +1262,7 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
                     if configured_group.config.type != "mcp_client":
                         continue
 
-                    from nat.plugins.mcp.client_config import MCPClientConfig
+                    from nat.plugins.mcp.client.client_config import MCPClientConfig
 
                     config = configured_group.config
                     assert isinstance(config, MCPClientConfig)
@@ -1253,6 +1411,107 @@ class FastApiFrontEndPluginWorker(FastApiFrontEndPluginWorkerBase):
                 }
             })
 
+    async def add_monitor_route(self, app: FastAPI):
+        """Add the per-user monitoring endpoint to the FastAPI app.
+
+        Security Warning:
+            This endpoint exposes per-user identifiers and usage metrics. It should be
+            protected by deploying behind an internal network, a reverse proxy with
+            authentication, or similar access controls to prevent exposure to untrusted callers.
+        """
+        # Check if monitoring is enabled in config
+        if not self._config.general.enable_per_user_monitoring:
+            logger.debug("Per-user monitoring disabled, skipping /monitor/users endpoint")
+            return
+
+        from nat.runtime.metrics import PerUserMetricsCollector
+        from nat.runtime.metrics import PerUserMonitorResponse
+        from nat.runtime.metrics import PerUserResourceUsage
+
+        async def get_per_user_metrics(user_id: str | None = None) -> PerUserMonitorResponse:
+            """
+            Get resource usage metrics for per-user workflows.
+
+            Args:
+                user_id: Optional user ID to filter metrics for a specific user
+
+            Returns:
+                PerUserMonitorResponse with metrics for all or specified users
+            """
+            # Collect metrics from all session managers that have per-user workflows
+            all_users: list[PerUserResourceUsage] = []
+
+            for session_manager in self._session_managers:
+                if not session_manager.is_workflow_per_user:
+                    continue
+
+                collector = PerUserMetricsCollector(session_manager)
+
+                if user_id is not None:
+                    # Filter for specific user
+                    user_metrics = await collector.collect_user_metrics(user_id)
+                    if user_metrics:
+                        all_users.append(user_metrics)
+                else:
+                    # Get all users
+                    response = await collector.collect_all_metrics()
+                    all_users.extend(response.users)
+
+            from datetime import datetime
+            return PerUserMonitorResponse(
+                timestamp=datetime.now(),
+                total_active_users=len(all_users),
+                users=all_users,
+            )
+
+        # Register the monitoring endpoint
+        app.add_api_route(path="/monitor/users",
+                          endpoint=get_per_user_metrics,
+                          methods=["GET"],
+                          response_model=PerUserMonitorResponse,
+                          description="Get resource usage metrics for per-user workflows",
+                          tags=["Monitoring"],
+                          responses={
+                              200: {
+                                  "description": "Successfully retrieved per-user metrics",
+                                  "content": {
+                                      "application/json": {
+                                          "example": {
+                                              "timestamp":
+                                                  "2025-12-16T10:30:00Z",
+                                              "total_active_users":
+                                                  2,
+                                              "users": [{
+                                                  "user_id": "alice",
+                                                  "session": {
+                                                      "created_at": "2025-12-16T09:00:00Z",
+                                                      "last_activity": "2025-12-16T10:29:55Z",
+                                                      "ref_count": 1,
+                                                      "is_active": True
+                                                  },
+                                                  "requests": {
+                                                      "total_requests": 42,
+                                                      "active_requests": 1,
+                                                      "avg_latency_ms": 1250.5,
+                                                      "error_count": 2
+                                                  },
+                                                  "memory": {
+                                                      "per_user_functions_count": 2,
+                                                      "per_user_function_groups_count": 1,
+                                                      "exit_stack_size": 3
+                                                  }
+                                              }]
+                                          }
+                                      }
+                                  }
+                              },
+                              500: {
+                                  "description": "Internal Server Error"
+                              }
+                          })
+
+        logger.info("Added per-user monitoring endpoint at /monitor/users")
+
     async def _add_flow(self, state: str, flow_state: FlowState):
         async with self._outstanding_flows_lock:
             self._outstanding_flows[state] = flow_state

nat/front_ends/fastapi/html_snippets/__init__.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

nat/front_ends/fastapi/html_snippets/auth_code_grant_success.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

nat/front_ends/fastapi/intermediate_steps_subscriber.py

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -32,6 +32,7 @@ async def pull_intermediate(_q, adapter):
     intermediate_done = asyncio.Event()
     context = Context.get()
     loop = asyncio.get_running_loop()
+    trace_id_emitted = False
 
     async def set_intermediate_done():
         intermediate_done.set()
@@ -43,6 +44,16 @@ async def pull_intermediate(_q, adapter):
         If adapter is None, convert the raw IntermediateStep into the complete
         ResponseIntermediateStep and place it into the queue.
         """
+        nonlocal trace_id_emitted
+
+        # Check if trace ID is now available and emit it once
+        if not trace_id_emitted:
+            observability_trace_id = context.observability_trace_id
+            if observability_trace_id:
+                from nat.data_models.api_server import ResponseObservabilityTrace
+                loop.create_task(_q.put(ResponseObservabilityTrace(observability_trace_id=observability_trace_id)))
+                trace_id_emitted = True
+
         if adapter is None:
             adapted = ResponseIntermediateStep(id=item.UUID,
                                                type=item.event_type,