nvidia-nat-test 1.3.0a20251108__py3-none-any.whl

nat/test/tool_test_runner.py

@@ -0,0 +1,516 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import inspect
+import logging
+import typing
+from collections.abc import Sequence
+from contextlib import asynccontextmanager
+from unittest.mock import AsyncMock
+from unittest.mock import MagicMock
+
+from nat.authentication.interfaces import AuthProviderBase
+from nat.builder.builder import Builder
+from nat.builder.function import Function
+from nat.builder.function import FunctionGroup
+from nat.builder.function_info import FunctionInfo
+from nat.cli.type_registry import GlobalTypeRegistry
+from nat.data_models.authentication import AuthProviderBaseConfig
+from nat.data_models.embedder import EmbedderBaseConfig
+from nat.data_models.function import FunctionBaseConfig
+from nat.data_models.function import FunctionGroupBaseConfig
+from nat.data_models.function_dependencies import FunctionDependencies
+from nat.data_models.llm import LLMBaseConfig
+from nat.data_models.memory import MemoryBaseConfig
+from nat.data_models.object_store import ObjectStoreBaseConfig
+from nat.data_models.retriever import RetrieverBaseConfig
+from nat.data_models.ttc_strategy import TTCStrategyBaseConfig
+from nat.experimental.test_time_compute.models.stage_enums import PipelineTypeEnum
+from nat.experimental.test_time_compute.models.stage_enums import StageTypeEnum
+from nat.memory.interfaces import MemoryEditor
+from nat.object_store.interfaces import ObjectStore
+from nat.runtime.loader import PluginTypes
+from nat.runtime.loader import discover_and_register_plugins
+
+logger = logging.getLogger(__name__)
+
+
+class MockBuilder(Builder):
+    """
+    A lightweight mock builder for tool testing that provides minimal dependencies.
+    """
+
+    def __init__(self):
+        self._functions = {}
+        self._mocks = {}
+
+    def mock_function(self, name: str, mock_response: typing.Any):
+        """Add a mock function that returns a fixed response."""
+        self._mocks[name] = mock_response
+
+    def mock_function_group(self, name: str, mock_response: typing.Any):
+        """Add a mock function group that returns a fixed response."""
+        self._mocks[name] = mock_response
+
+    def mock_llm(self, name: str, mock_response: typing.Any):
+        """Add a mock LLM that returns a fixed response."""
+        self._mocks[f"llm_{name}"] = mock_response
+
+    def mock_embedder(self, name: str, mock_response: typing.Any):
+        """Add a mock embedder that returns a fixed response."""
+        self._mocks[f"embedder_{name}"] = mock_response
+
+    def mock_memory_client(self, name: str, mock_response: typing.Any):
+        """Add a mock memory client that returns a fixed response."""
+        self._mocks[f"memory_{name}"] = mock_response
+
+    def mock_retriever(self, name: str, mock_response: typing.Any):
+        """Add a mock retriever that returns a fixed response."""
+        self._mocks[f"retriever_{name}"] = mock_response
+
+    def mock_object_store(self, name: str, mock_response: typing.Any):
+        """Add a mock object store that returns a fixed response."""
+        self._mocks[f"object_store_{name}"] = mock_response
+
+    def mock_ttc_strategy(self, name: str, mock_response: typing.Any):
+        """Add a mock TTC strategy that returns a fixed response."""
+        self._mocks[f"ttc_strategy_{name}"] = mock_response
+
+    def mock_auth_provider(self, name: str, mock_response: typing.Any):
+        """Add a mock auth provider that returns a fixed response."""
+        self._mocks[f"auth_provider_{name}"] = mock_response
+
+    async def add_ttc_strategy(self, name: str, config: TTCStrategyBaseConfig) -> None:
+        """Mock implementation (no‑op)."""
+        pass
+
+    async def get_ttc_strategy(self, strategy_name: str, pipeline_type: PipelineTypeEnum,
+                               stage_type: StageTypeEnum) -> typing.Any:
+        """Return a mock TTC strategy if one is configured."""
+        key = f"ttc_strategy_{strategy_name}"
+        if key in self._mocks:
+            mock_strategy = MagicMock()
+            # Provide common callable patterns used in tests
+            mock_strategy.invoke = MagicMock(return_value=self._mocks[key])
+            mock_strategy.ainvoke = AsyncMock(return_value=self._mocks[key])
+            return mock_strategy
+        raise ValueError(f"TTC strategy '{strategy_name}' not mocked. Use mock_ttc_strategy() to add it.")
+
+    async def get_ttc_strategy_config(self,
+                                      strategy_name: str,
+                                      pipeline_type: PipelineTypeEnum,
+                                      stage_type: StageTypeEnum) -> TTCStrategyBaseConfig:
+        """Mock implementation."""
+        return TTCStrategyBaseConfig()
+
+    async def add_auth_provider(self, name: str, config: AuthProviderBaseConfig) -> None:
+        """Mock implementation (no‑op)."""
+        pass
+
+    async def get_auth_provider(self, auth_provider_name: str) -> AuthProviderBase:
+        """Return a mock auth provider if one is configured."""
+        key = f"auth_provider_{auth_provider_name}"
+        if key in self._mocks:
+            mock_auth = MagicMock()
+            mock_auth.authenticate = AsyncMock(return_value=self._mocks[key])
+            return mock_auth
+        raise ValueError(f"Auth provider '{auth_provider_name}' not mocked. Use mock_auth_provider() to add it.")
+
+    async def add_function(self, name: str, config: FunctionBaseConfig) -> Function:
+        """Mock implementation - not used in tool testing."""
+        raise NotImplementedError("Mock implementation does not support add_function")
+
+    async def get_function(self, name: str) -> Function:
+        """Return a mock function if one is configured."""
+        if name in self._mocks:
+            mock_fn = AsyncMock()
+            mock_fn.ainvoke = AsyncMock(return_value=self._mocks[name])
+            return mock_fn
+        raise ValueError(f"Function '{name}' not mocked. Use mock_function() to add it.")
+
+    def get_function_config(self, name: str) -> FunctionBaseConfig:
+        """Mock implementation."""
+        return FunctionBaseConfig()
+
+    async def add_function_group(self, name: str, config: FunctionGroupBaseConfig) -> FunctionGroup:
+        """Mock implementation - not used in tool testing."""
+        raise NotImplementedError("Mock implementation does not support add_function_group")
+
+    async def get_function_group(self, name: str) -> FunctionGroup:
+        """Return a mock function group if one is configured."""
+        if name in self._mocks:
+            mock_fn_group = MagicMock(spec=FunctionGroup)
+            mock_fn_group.ainvoke = AsyncMock(return_value=self._mocks[name])
+            return mock_fn_group
+        raise ValueError(f"Function group '{name}' not mocked. Use mock_function_group() to add it.")
+
+    def get_function_group_config(self, name: str) -> FunctionGroupBaseConfig:
+        """Mock implementation."""
+        return FunctionGroupBaseConfig()
+
+    async def set_workflow(self, config: FunctionBaseConfig) -> Function:
+        """Mock implementation."""
+        mock_fn = AsyncMock()
+        mock_fn.ainvoke = AsyncMock(return_value="mock_workflow_result")
+        return mock_fn
+
+    def get_workflow(self) -> Function:
+        """Mock implementation."""
+        mock_fn = AsyncMock()
+        mock_fn.ainvoke = AsyncMock(return_value="mock_workflow_result")
+        return mock_fn
+
+    def get_workflow_config(self) -> FunctionBaseConfig:
+        """Mock implementation."""
+        return FunctionBaseConfig()
+
+    async def get_tools(self, tool_names: Sequence[str], wrapper_type) -> list[typing.Any]:
+        """Mock implementation."""
+        return []
+
+    async def get_tool(self, fn_name: str, wrapper_type) -> typing.Any:
+        """Mock implementation."""
+        pass
+
+    async def add_llm(self, name: str, config) -> None:
+        """Mock implementation."""
+        pass
+
+    async def get_llm(self, llm_name: str, wrapper_type):
+        """Return a mock LLM if one is configured."""
+        key = f"llm_{llm_name}"
+        if key in self._mocks:
+            mock_llm = MagicMock()
+            mock_llm.invoke = MagicMock(return_value=self._mocks[key])
+            mock_llm.ainvoke = AsyncMock(return_value=self._mocks[key])
+            return mock_llm
+        raise ValueError(f"LLM '{llm_name}' not mocked. Use mock_llm() to add it.")
+
+    def get_llm_config(self, llm_name: str) -> LLMBaseConfig:
+        """Mock implementation."""
+        return LLMBaseConfig()
+
+    async def add_embedder(self, name: str, config) -> None:
+        """Mock implementation."""
+        pass
+
+    async def get_embedder(self, embedder_name: str, wrapper_type):
+        """Return a mock embedder if one is configured."""
+        key = f"embedder_{embedder_name}"
+        if key in self._mocks:
+            mock_embedder = MagicMock()
+            mock_embedder.embed_query = MagicMock(return_value=self._mocks[key])
+            mock_embedder.embed_documents = MagicMock(return_value=self._mocks[key])
+            return mock_embedder
+        raise ValueError(f"Embedder '{embedder_name}' not mocked. Use mock_embedder() to add it.")
+
+    def get_embedder_config(self, embedder_name: str) -> EmbedderBaseConfig:
+        """Mock implementation."""
+        return EmbedderBaseConfig()
+
+    async def add_memory_client(self, name: str, config) -> MemoryEditor:
+        return MagicMock(spec=MemoryEditor)
+
+    async def get_memory_client(self, memory_name: str) -> MemoryEditor:
+        """Return a mock memory client if one is configured."""
+        key = f"memory_{memory_name}"
+        if key in self._mocks:
+            mock_memory = MagicMock()
+            mock_memory.add = AsyncMock(return_value=self._mocks[key])
+            mock_memory.search = AsyncMock(return_value=self._mocks[key])
+            return mock_memory
+        raise ValueError(f"Memory client '{memory_name}' not mocked. Use mock_memory_client() to add it.")
+
+    def get_memory_client_config(self, memory_name: str) -> MemoryBaseConfig:
+        """Mock implementation."""
+        return MemoryBaseConfig()
+
+    async def add_retriever(self, name: str, config) -> None:
+        """Mock implementation."""
+        pass
+
+    async def get_retriever(self, retriever_name: str, wrapper_type=None):
+        """Return a mock retriever if one is configured."""
+        key = f"retriever_{retriever_name}"
+        if key in self._mocks:
+            mock_retriever = MagicMock()
+            mock_retriever.retrieve = AsyncMock(return_value=self._mocks[key])
+            return mock_retriever
+        raise ValueError(f"Retriever '{retriever_name}' not mocked. Use mock_retriever() to add it.")
+
+    async def get_retriever_config(self, retriever_name: str) -> RetrieverBaseConfig:
+        """Mock implementation."""
+        return RetrieverBaseConfig()
+
+    async def add_object_store(self, name: str, config: ObjectStoreBaseConfig) -> ObjectStore:
+        """Mock implementation for object store."""
+        return MagicMock(spec=ObjectStore)
+
+    async def get_object_store_client(self, object_store_name: str) -> ObjectStore:
+        """Return a mock object store client if one is configured."""
+        key = f"object_store_{object_store_name}"
+        if key in self._mocks:
+            mock_object_store = MagicMock()
+            mock_object_store.put_object = AsyncMock(return_value=self._mocks[key])
+            mock_object_store.get_object = AsyncMock(return_value=self._mocks[key])
+            mock_object_store.delete_object = AsyncMock(return_value=self._mocks[key])
+            mock_object_store.list_objects = AsyncMock(return_value=self._mocks[key])
+            return mock_object_store
+        raise ValueError(f"Object store '{object_store_name}' not mocked. Use mock_object_store() to add it.")
+
+    def get_object_store_config(self, object_store_name: str) -> ObjectStoreBaseConfig:
+        """Mock implementation for object store config."""
+        return ObjectStoreBaseConfig()
+
+    def get_user_manager(self):
+        """Mock implementation."""
+        mock_user = MagicMock()
+        mock_user.get_id = MagicMock(return_value="test_user")
+        return mock_user
+
+    def get_function_dependencies(self, fn_name: str) -> FunctionDependencies:
+        """Mock implementation."""
+        return FunctionDependencies()
+
+    def get_function_group_dependencies(self, fn_name: str) -> FunctionDependencies:
+        """Mock implementation."""
+        return FunctionDependencies()
+
+
+class ToolTestRunner:
+    """
+    A test runner that enables isolated testing of NAT tools without requiring
+    full workflow setup, LLMs, or complex dependencies.
+
+    Usage:
+        runner = ToolTestRunner()
+
+        # Test a tool with minimal setup
+        result = await runner.test_tool(
+            config_type=MyToolConfig,
+            config_params={"param1": "value1"},
+            input_data="test input"
+        )
+
+        # Test a tool with mocked dependencies
+        async with runner.with_mocks() as mock_builder:
+            mock_builder.mock_llm("my_llm", "mocked response")
+            result = await runner.test_tool(
+                config_type=MyToolConfig,
+                config_params={"llm_name": "my_llm"},
+                input_data="test input"
+            )
+    """
+
+    def __init__(self):
+        self._ensure_plugins_loaded()
+
+    def _ensure_plugins_loaded(self):
+        """Ensure all plugins are loaded for tool registration."""
+        discover_and_register_plugins(PluginTypes.CONFIG_OBJECT)
+
+    async def test_tool(self,
+                        config_type: type[FunctionBaseConfig],
+                        config_params: dict[str, typing.Any] | None = None,
+                        input_data: typing.Any = None,
+                        expected_output: typing.Any = None,
+                        **kwargs) -> typing.Any:
+        """
+        Test a tool in isolation with minimal setup.
+
+        Args:
+            config_type: The tool configuration class
+            config_params: Parameters to pass to the config constructor
+            input_data: Input data to pass to the tool
+            expected_output: Expected output for assertion (optional)
+            **kwargs: Additional parameters
+
+        Returns:
+            The tool's output
+
+        Raises:
+            AssertionError: If expected_output is provided and doesn't match
+            ValueError: If tool registration or execution fails
+        """
+        config_params = config_params or {}
+
+        # Create tool configuration
+        config = config_type(**config_params)
+
+        # Get the registered tool function
+        registry = GlobalTypeRegistry.get()
+        try:
+            tool_registration = registry.get_function(config_type)
+        except KeyError:
+            raise ValueError(
+                f"Tool {config_type} is not registered. Make sure it's imported and registered with @register_function."
+            )
+
+        # Create a mock builder for dependencies
+        mock_builder = MockBuilder()
+
+        # Build the tool function
+        async with tool_registration.build_fn(config, mock_builder) as tool_result:
+
+            # Handle different tool result types
+            if isinstance(tool_result, Function):
+                tool_function = tool_result
+            elif isinstance(tool_result, FunctionInfo):
+                # Extract the actual function from FunctionInfo
+                if tool_result.single_fn:
+                    tool_function = tool_result.single_fn
+                elif tool_result.stream_fn:
+                    tool_function = tool_result.stream_fn
+                else:
+                    raise ValueError("Tool function not found in FunctionInfo")
+            elif callable(tool_result):
+                tool_function = tool_result
+            else:
+                raise ValueError(f"Unexpected tool result type: {type(tool_result)}")
+
+            # Execute the tool
+            if input_data is not None:
+                if isinstance(tool_function, Function):
+                    result = await tool_function.ainvoke(input_data)
+                elif asyncio.iscoroutinefunction(tool_function):
+                    result = await tool_function(input_data)
+                else:
+                    result = tool_function(input_data)
+            elif isinstance(tool_function, Function):
+                # Function objects require input, so pass None if no input_data
+                result = await tool_function.ainvoke(None)
+            elif asyncio.iscoroutinefunction(tool_function):
+                result = await tool_function()
+            else:
+                result = tool_function()
+
+            # Assert expected output if provided
+            if expected_output is not None:
+                assert result == expected_output, f"Expected {expected_output}, got {result}"
+
+            return result
+
+    @asynccontextmanager
+    async def with_mocks(self):
+        """
+        Context manager that provides a mock builder for setting up dependencies.
+
+        Usage:
+            async with runner.with_mocks() as mock_builder:
+                mock_builder.mock_llm("my_llm", "mocked response")
+                result = await runner.test_tool_with_builder(
+                    config_type=MyToolConfig,
+                    builder=mock_builder,
+                    input_data="test input"
+                )
+        """
+        mock_builder = MockBuilder()
+        try:
+            yield mock_builder
+        finally:
+            pass
+
+    async def test_tool_with_builder(
+        self,
+        config_type: type[FunctionBaseConfig],
+        builder: MockBuilder,
+        config_params: dict[str, typing.Any] | None = None,
+        input_data: typing.Any = None,
+        expected_output: typing.Any = None,
+    ) -> typing.Any:
+        """
+        Test a tool with a pre-configured mock builder.
+
+        Args:
+            config_type: The tool configuration class
+            builder: Pre-configured MockBuilder with mocked dependencies
+            config_params: Parameters to pass to the config constructor
+            input_data: Input data to pass to the tool
+            expected_output: Expected output for assertion (optional)
+
+        Returns:
+            The tool's output
+        """
+        config_params = config_params or {}
+
+        # Create tool configuration
+        config = config_type(**config_params)
+
+        # Get the registered tool function
+        registry = GlobalTypeRegistry.get()
+        try:
+            tool_registration = registry.get_function(config_type)
+        except KeyError:
+            raise ValueError(
+                f"Tool {config_type} is not registered. Make sure it's imported and registered with @register_function."
+            )
+
+        # Build the tool function with the provided builder
+        async with tool_registration.build_fn(config, builder) as tool_result:
+
+            # Handle different tool result types (same as above)
+            if isinstance(tool_result, Function):
+                tool_function = tool_result
+            elif isinstance(tool_result, FunctionInfo):
+                if tool_result.single_fn:
+                    tool_function = tool_result.single_fn
+                elif tool_result.stream_fn:
+                    tool_function = tool_result.stream_fn
+                else:
+                    raise ValueError("Tool function not found in FunctionInfo")
+            elif callable(tool_result):
+                tool_function = tool_result
+            else:
+                raise ValueError(f"Unexpected tool result type: {type(tool_result)}")
+
+            # Execute the tool
+            if input_data is not None:
+                if isinstance(tool_function, Function):
+                    result = await tool_function.ainvoke(input_data)
+                else:
+                    maybe_result = tool_function(input_data)
+                    result = await maybe_result if inspect.isawaitable(maybe_result) else maybe_result
+            elif isinstance(tool_function, Function):
+                # Function objects require input, so pass None if no input_data
+                result = await tool_function.ainvoke(None)
+            else:
+                maybe_result = tool_function()
+                result = await maybe_result if inspect.isawaitable(maybe_result) else maybe_result
+
+            # Assert expected output if provided
+            if expected_output is not None:
+                assert result == expected_output, f"Expected {expected_output}, got {result}"
+
+            return result
+
+
+@asynccontextmanager
+async def with_mocked_dependencies():
+    """
+    Convenience context manager for testing tools with mocked dependencies.
+
+    Usage:
+        async with with_mocked_dependencies() as (runner, mock_builder):
+            mock_builder.mock_llm("my_llm", "mocked response")
+            result = await runner.test_tool_with_builder(
+                config_type=MyToolConfig,
+                builder=mock_builder,
+                input_data="test input"
+            )
+    """
+    runner = ToolTestRunner()
+    async with runner.with_mocks() as mock_builder:
+        yield runner, mock_builder

nat/test/utils.py

@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.resources
+import inspect
+import json
+import subprocess
+import typing
+from contextlib import asynccontextmanager
+from pathlib import Path
+
+if typing.TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from httpx import AsyncClient
+
+    from nat.data_models.config import Config
+    from nat.front_ends.fastapi.fastapi_front_end_plugin_worker import FastApiFrontEndPluginWorker
+    from nat.utils.type_utils import StrPath
+
+
+def locate_repo_root() -> Path:
+    result = subprocess.run(["git", "rev-parse", "--show-toplevel"], check=False, capture_output=True, text=True)
+    assert result.returncode == 0, f"Failed to get git root: {result.stderr}"
+    return Path(result.stdout.strip())
+
+
+def locate_example_src_dir(example_config_class: type) -> Path:
+    """
+    Locate the example src directory for an example's config class.
+    """
+    package_name = inspect.getmodule(example_config_class).__package__
+    return importlib.resources.files(package_name)
+
+
+def locate_example_dir(example_config_class: type) -> Path:
+    """
+    Locate the example directory for an example's config class.
+    """
+    src_dir = locate_example_src_dir(example_config_class)
+    example_dir = src_dir.parent.parent
+    return example_dir
+
+
+def locate_example_config(example_config_class: type,
+                          config_file: str = "config.yml",
+                          assert_exists: bool = True) -> Path:
+    """
+    Locate the example config file for an example's config class, assumes the example contains a 'configs' directory
+    """
+    example_dir = locate_example_src_dir(example_config_class)
+    config_path = example_dir.joinpath("configs", config_file).absolute()
+    if assert_exists:
+        assert config_path.exists(), f"Config file {config_path} does not exist"
+
+    return config_path
+
+
+async def run_workflow(
+    *,
+    config: "Config | None" = None,
+    config_file: "StrPath | None" = None,
+    question: str,
+    expected_answer: str,
+    assert_expected_answer: bool = True,
+) -> str:
+    from nat.builder.workflow_builder import WorkflowBuilder
+    from nat.runtime.loader import load_config
+    from nat.runtime.session import SessionManager
+
+    if config is None:
+        assert config_file is not None, "Either config_file or config must be provided"
+        assert Path(config_file).exists(), f"Config file {config_file} does not exist"
+        config = load_config(config_file)
+
+    async with WorkflowBuilder.from_config(config=config) as workflow_builder:
+        workflow = SessionManager(await workflow_builder.build())
+        async with workflow.run(question) as runner:
+            result = await runner.result(to_type=str)
+
+    if assert_expected_answer:
+        assert expected_answer.lower() in result.lower(), f"Expected '{expected_answer}' in '{result}'"
+
+    return result
+
+
+@asynccontextmanager
+async def build_nat_client(
+        config: "Config",
+        worker_class: "type[FastApiFrontEndPluginWorker] | None" = None) -> "AsyncIterator[AsyncClient]":
+    """
+    Build a NAT client for testing purposes.
+
+    Creates a test client with an ASGI transport for the specified configuration.
+    The client is backed by a FastAPI application built from the provided worker class.
+
+    Args:
+        config: The NAT configuration to use for building the client.
+        worker_class: Optional worker class to use. Defaults to FastApiFrontEndPluginWorker.
+
+    Yields:
+        An AsyncClient instance configured for testing.
+    """
+    from asgi_lifespan import LifespanManager
+    from httpx import ASGITransport
+    from httpx import AsyncClient
+
+    from nat.front_ends.fastapi.fastapi_front_end_plugin_worker import FastApiFrontEndPluginWorker
+
+    if worker_class is None:
+        worker_class = FastApiFrontEndPluginWorker
+
+    worker = worker_class(config)
+    app = worker.build_app()
+
+    async with LifespanManager(app):
+        async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as client:
+            yield client
+
+
+def validate_workflow_output(workflow_output_file: Path) -> None:
+    """
+    Validate the contents of the workflow output file.
+    WIP: output format should be published as a schema and this validation should be done against that schema.
+    """
+    # Ensure the workflow_output.json file was created
+    assert workflow_output_file.exists(), "The workflow_output.json file was not created"
+
+    # Read and validate the workflow_output.json file
+    try:
+        with open(workflow_output_file, encoding="utf-8") as f:
+            result_json = json.load(f)
+    except json.JSONDecodeError as err:
+        raise RuntimeError("Failed to parse workflow_output.json as valid JSON") from err
+
+    assert isinstance(result_json, list), "The workflow_output.json file is not a list"
+    assert len(result_json) > 0, "The workflow_output.json file is empty"
+    assert isinstance(result_json[0], dict), "The workflow_output.json file is not a list of dictionaries"
+
+    # Ensure required keys exist
+    required_keys = ["id", "question", "answer", "generated_answer", "intermediate_steps"]
+    for key in required_keys:
+        assert all(item.get(key) for item in result_json), f"The '{key}' key is missing in workflow_output.json"