nvidia-nat 1.2.0__py3-none-any.whl

nat/eval/intermediate_step_adapter.py

@@ -0,0 +1,99 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from langchain_core.agents import AgentAction
+
+from nat.data_models.intermediate_step import IntermediateStep
+from nat.data_models.intermediate_step import IntermediateStepType
+
+logger = logging.getLogger(__name__)
+
+
+class IntermediateStepAdapter:
+    DEFAULT_EVENT_FILTER = [IntermediateStepType.LLM_END, IntermediateStepType.TOOL_END]
+
+    def filter_intermediate_steps(self,
+                                  intermediate_steps: list[IntermediateStep],
+                                  event_filter: list[IntermediateStepType]) -> list[IntermediateStep]:
+        """ Filters intermediate steps"""
+        if not event_filter:
+            return intermediate_steps
+        return [step for step in intermediate_steps if step.event_type in event_filter]
+
+    def validate_intermediate_steps(self, intermediate_steps: list[dict]) -> list[IntermediateStep]:
+        validated_steps = []
+        for step_data in intermediate_steps:
+            try:
+                validated_steps.append(IntermediateStep.model_validate(step_data))
+            except Exception as e:
+                logger.exception("Validation failed for step: %r, Error: %s", step_data, e, exc_info=True)
+        return validated_steps
+
+    def serialize_intermediate_steps(self, intermediate_steps: list[IntermediateStep]) -> list[dict]:
+        """Converts a list of IntermediateStep objects to a list of dictionaries."""
+        return [step.model_dump() for step in intermediate_steps]
+
+    @staticmethod
+    def agent_action_to_dict(action) -> dict:
+        """Convert AgentAction to a JSON-serializable dictionary."""
+        return {
+            "tool": action.tool,
+            "tool_input": action.tool_input,
+            "log": action.log,
+            "type": action.type,
+        }
+
+    def get_agent_action_single(self, step: IntermediateStep,
+                                last_llm_end_step: IntermediateStep | None) -> tuple[AgentAction, str]:
+        """Converts a single intermediate step to Tuple[AgentAction, str]."""
+        # use the previous llm output as log
+        log = getattr(last_llm_end_step.data, "output", "") if last_llm_end_step else ""
+        tool_name = step.name or ""
+        tool_input = getattr(step.data, "input", "") if step.data else ""
+        tool_output = getattr(step.data, "output", "") if step.data else ""
+
+        action = AgentAction(tool=tool_name, tool_input=tool_input, log=log)
+
+        return action, tool_output
+
+    def get_agent_actions(self, intermediate_steps: list[IntermediateStep],
+                          event_filter: list[IntermediateStepType]) -> list[tuple[AgentAction, str]]:
+        """Converts a list of intermediate steps to a list of (AgentAction, output)."""
+        steps = self.filter_intermediate_steps(intermediate_steps, event_filter)
+        last_llm_end_step = None
+        agent_actions = []
+        for step in steps:
+            if step.event_type == IntermediateStepType.LLM_END:
+                last_llm_end_step = step
+                action = self.get_agent_action_single(step, "")
+                agent_actions.append(action)
+            else:
+                action = self.get_agent_action_single(step, last_llm_end_step)
+                agent_actions.append(action)
+
+        return agent_actions
+
+    def get_context(self, intermediate_steps: list[IntermediateStep],
+                    event_filter: list[IntermediateStepType]) -> list[str]:
+        """Grab the output of all the tools and return them as retrieved context."""
+        count = 0
+        agent_actions = []
+        for step in intermediate_steps:
+            if step.event_type in event_filter and step.data and step.data.output:
+                agent_actions.append(f"**Step {count}**\n{str(step.data.output)}")
+                count += 1
+        return agent_actions

nat/eval/rag_evaluator/evaluate.py

@@ -0,0 +1,178 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import math
+from collections.abc import Sequence
+
+from pydantic import BaseModel
+from ragas import EvaluationDataset
+from ragas import SingleTurnSample
+from ragas.dataset_schema import EvaluationResult
+from ragas.llms import LangchainLLMWrapper
+from ragas.metrics import Metric
+from tqdm import tqdm
+
+from nat.data_models.intermediate_step import IntermediateStepType
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
+from nat.eval.evaluator.evaluator_model import EvalOutput
+from nat.eval.evaluator.evaluator_model import EvalOutputItem
+from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+
+logger = logging.getLogger(__name__)
+
+
+class RAGEvaluator:
+
+    def __init__(self,
+                 evaluator_llm: LangchainLLMWrapper,
+                 metrics: Sequence[Metric],
+                 max_concurrency=8,
+                 input_obj_field: str | None = None):
+        self.evaluator_llm = evaluator_llm
+        self.metrics = metrics
+        self.max_concurrency = max_concurrency
+        self.input_obj_field = input_obj_field
+
+    def extract_input_obj(self, item: EvalInputItem) -> str:
+        """Extracts the input object from EvalInputItem based on the configured input_obj_field."""
+        input_obj = item.input_obj
+        if isinstance(input_obj, BaseModel):
+            if self.input_obj_field and hasattr(input_obj, self.input_obj_field):
+                # If input_obj_field is specified, return the value of that field
+                return str(getattr(input_obj, self.input_obj_field, ""))
+
+            # If no input_obj_field is specified, return the string representation of the model
+            return input_obj.model_dump_json()
+
+        if isinstance(input_obj, dict):
+            # If input_obj is a dict, return the JSON string representation
+            if self.input_obj_field and self.input_obj_field in input_obj:
+                # If input_obj_field is specified, return the value of that field
+                return str(input_obj[self.input_obj_field])
+
+        return str(input_obj)  # Fallback to string representation of the dict
+
+    def eval_input_to_ragas(self, eval_input: EvalInput) -> EvaluationDataset:
+        """Converts EvalInput into a Ragas-compatible EvaluationDataset."""
+        from nat.eval.intermediate_step_adapter import IntermediateStepAdapter
+        event_filter = [IntermediateStepType.TOOL_END, IntermediateStepType.LLM_END, IntermediateStepType.CUSTOM_END]
+        samples = []
+
+        intermediate_step_adapter = IntermediateStepAdapter()
+        for item in eval_input.eval_input_items:
+            # Extract required fields from EvalInputItem
+            user_input = self.extract_input_obj(item)  # Extract input object as string
+            reference = item.expected_output_obj  # Reference correct answer
+            response = item.output_obj  # Model's generated response
+
+            # Handle context extraction from trajectory if available
+            reference_contexts = [""]  # Default to empty context
+            # implement context extraction from expected_trajectory
+
+            retrieved_contexts = intermediate_step_adapter.get_context(item.trajectory, event_filter)
+            # implement context extraction from expected_trajectory
+
+            # Create a SingleTurnSample
+            sample = SingleTurnSample(
+                user_input=user_input,
+                reference=reference,
+                response=response,
+                reference_contexts=reference_contexts,
+                retrieved_contexts=retrieved_contexts,
+            )
+            samples.append(sample)
+
+        return EvaluationDataset(samples=samples)
+
+    def ragas_to_eval_output(self, eval_input: EvalInput, results_dataset: EvaluationResult | None) -> EvalOutput:
+        """Converts the ragas EvaluationResult to nat EvalOutput"""
+
+        if not results_dataset:
+            logger.error("Ragas evaluation failed with no results")
+            return EvalOutput(average_score=0.0, eval_output_items=[])
+
+        scores: list[dict[str, float]] = results_dataset.scores
+
+        # If Ragas returned no scores, return empty output to avoid downstream errors
+        if not scores:
+            logger.warning("Ragas returned empty score list")
+            return EvalOutput(average_score=0.0, eval_output_items=[])
+
+        def _nan_to_zero(v: float | None) -> float:
+            """Convert NaN or None to 0.0 for safe arithmetic/serialization."""
+            return 0.0 if v is None or (isinstance(v, float) and math.isnan(v)) else v
+
+        # Convert from list of dicts to dict of lists, coercing NaN/None to 0.0
+        scores_dict = {metric: [_nan_to_zero(score.get(metric)) for score in scores] for metric in scores[0]}
+        first_metric_name = list(scores_dict.keys())[0] if scores_dict else None
+
+        # Compute the average of each metric, guarding against empty lists
+        average_scores = {
+            metric: (sum(values) / len(values) if values else 0.0)
+            for metric, values in scores_dict.items()
+        }
+
+        first_avg_score = average_scores.get(list(scores_dict.keys())[0], 0.0)
+        if isinstance(first_avg_score, float) and math.isnan(first_avg_score):
+            first_avg_score = 0.0
+
+        df = results_dataset.to_pandas()
+        # Get id from eval_input if df size matches number of eval_input_items
+        if len(eval_input.eval_input_items) >= len(df):
+            ids = [item.id for item in eval_input.eval_input_items]  # Extract IDs
+        else:
+            ids = df["user_input"].tolist()  # Use "user_input" as ID fallback
+
+        # Construct EvalOutputItem list
+        eval_output_items = [
+            EvalOutputItem(
+                id=ids[i],
+                score=_nan_to_zero(getattr(row, first_metric_name, 0.0) if first_metric_name else 0.0),
+                reasoning={
+                    key:
+                        getattr(row, key, None)  # Use getattr to safely access attributes
+                    for key in ["user_input", "reference", "response", "retrieved_contexts"]
+                }) for i, row in enumerate(df.itertuples(index=False))
+        ]
+        # Return EvalOutput
+        return EvalOutput(average_score=first_avg_score, eval_output_items=eval_output_items)
+
+    async def evaluate(self, eval_input: EvalInput) -> EvalOutput:
+        """Run Ragas metrics evaluation on the provided EvalInput"""
+        from ragas import evaluate as ragas_evaluate
+        from ragas.run_config import RunConfig
+
+        ragas_dataset = self.eval_input_to_ragas(eval_input)
+        tqdm_position = TqdmPositionRegistry.claim()
+        first_metric_name = self.metrics[0].name
+        pbar = tqdm(total=len(ragas_dataset), desc=f"Evaluating Ragas {first_metric_name}", position=tqdm_position)
+        try:
+            results_dataset = ragas_evaluate(dataset=ragas_dataset,
+                                             metrics=self.metrics,
+                                             show_progress=True,
+                                             llm=self.evaluator_llm,
+                                             run_config=RunConfig(max_workers=self.max_concurrency),
+                                             _pbar=pbar)
+        except Exception as e:
+            # On exception we still continue with other evaluators. Log and return an avg_score of 0.0
+            logger.exception("Error evaluating ragas metric, Error: %s", e, exc_info=True)
+            results_dataset = None
+        finally:
+            pbar.close()
+            TqdmPositionRegistry.release(tqdm_position)
+
+        return self.ragas_to_eval_output(eval_input, results_dataset)

nat/eval/rag_evaluator/register.py

@@ -0,0 +1,143 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import model_validator
+
+from nat.builder.builder import EvalBuilder
+from nat.builder.evaluator import EvaluatorInfo
+from nat.builder.framework_enum import LLMFrameworkEnum
+from nat.cli.register_workflow import register_evaluator
+from nat.data_models.evaluator import EvaluatorBaseConfig
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalOutput
+
+logger = logging.getLogger(__name__)
+
+
+class RagasMetricConfig(BaseModel):
+    ''' RAGAS metrics configuration
+    skip: Allows the metric config to be present but not used
+    kwargs: Additional arguments to pass to the metric's callable
+    '''
+    skip: bool = False
+    # kwargs specific to the metric's callable
+    kwargs: dict | None = None
+
+
+class RagasEvaluatorConfig(EvaluatorBaseConfig, name="ragas"):
+    """Evaluation using RAGAS metrics."""
+
+    llm_name: str = Field(description="LLM as a judge.")
+    # Ragas metric
+    metric: str | dict[str, RagasMetricConfig] = Field(default="AnswerAccuracy",
+                                                       description="RAGAS metric callable with optional 'kwargs:'")
+    input_obj_field: str | None = Field(
+        default=None, description="The field in the input object that contains the content to evaluate.")
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_metric(cls, values):
+        """Ensures metric is either a string or a single-item dictionary."""
+        metric = values.get("metric")
+
+        if isinstance(metric, dict):
+            if len(metric) != 1:
+                raise ValueError("Only one metric is allowed in the configuration.")
+            _, value = next(iter(metric.items()))
+            if not isinstance(value, dict):
+                raise ValueError("Metric value must be a RagasMetricConfig object.")
+        elif not isinstance(metric, str):
+            raise ValueError("Metric must be either a string or a single-item dictionary.")
+
+        return values
+
+    @property
+    def metric_name(self) -> str:
+        """Returns the single metric name."""
+        if isinstance(self.metric, str):
+            return self.metric
+        if isinstance(self.metric, dict) and self.metric:
+            return next(iter(self.metric.keys()))  # pylint: disable=no-member
+        return ""
+
+    @property
+    def metric_config(self) -> RagasMetricConfig:
+        """Returns the metric configuration (or a default if only a string is provided)."""
+        if isinstance(self.metric, str):
+            return RagasMetricConfig()  # Default config when only a metric name is given
+        if isinstance(self.metric, dict) and self.metric:
+            return next(iter(self.metric.values()))  # pylint: disable=no-member
+        return RagasMetricConfig()  # Default config when an invalid type is provided
+
+
+@register_evaluator(config_type=RagasEvaluatorConfig)
+async def register_ragas_evaluator(config: RagasEvaluatorConfig, builder: EvalBuilder):
+    from ragas.metrics import Metric
+
+    def get_ragas_metric(metric_name: str) -> Metric | None:
+        """
+        Fetch callable for RAGAS metrics
+        """
+        try:
+            import ragas.metrics as ragas_metrics
+
+            return getattr(ragas_metrics, metric_name)
+        except ImportError as e:
+            message = f"Ragas metrics not found {e}."
+            logger.error(message)
+            raise ValueError(message) from e
+        except AttributeError as e:
+            message = f"Ragas metric {metric_name} not found {e}."
+            logger.error(message)
+            return None
+
+    async def evaluate_fn(eval_input: EvalInput) -> EvalOutput:
+        '''Run the RAGAS evaluation and return the average scores and evaluation results dataframe'''
+        if not _evaluator:
+            logger.warning("No evaluator found for RAGAS metrics.")
+            # return empty results if no evaluator is found
+            return EvalOutput(average_score=0.0, eval_output_items=[])
+
+        return await _evaluator.evaluate(eval_input)
+
+    from .evaluate import RAGEvaluator
+
+    # Get LLM
+    llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)
+
+    # Get RAGAS metric callable from the metric config and create a list of metric-callables
+    metrics = []
+    # currently only one metric is supported
+    metric_name = config.metric_name  # Extracts the metric name
+    metric_config = config.metric_config  # Extracts the config (handles str/dict cases)
+
+    # Skip if `skip` is True
+    if not metric_config.skip:
+        metric_callable = get_ragas_metric(metric_name)
+        if metric_callable:
+            kwargs = metric_config.kwargs or {}
+            metrics.append(metric_callable(**kwargs))
+
+    # Create the RAG evaluator
+    _evaluator = RAGEvaluator(evaluator_llm=llm,
+                              metrics=metrics,
+                              max_concurrency=builder.get_max_concurrency(),
+                              input_obj_field=config.input_obj_field) if metrics else None
+
+    yield EvaluatorInfo(config=config, evaluate_fn=evaluate_fn, description="Evaluator for RAGAS metrics")

nat/eval/register.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# flake8: noqa
+# pylint: disable=unused-import
+
+# Import evaluators which need to be automatically registered here
+from .rag_evaluator.register import register_ragas_evaluator
+from .swe_bench_evaluator.register import register_swe_bench_evaluator
+from .trajectory_evaluator.register import register_trajectory_evaluator
+from .tunable_rag_evaluator.register import register_tunable_rag_evaluator

nat/eval/remote_workflow.py

@@ -0,0 +1,133 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import asyncio
+import json
+import logging
+
+import aiohttp
+from pydantic import ValidationError
+from tqdm import tqdm
+
+from nat.data_models.api_server import ResponseIntermediateStep
+from nat.data_models.intermediate_step import IntermediateStep
+from nat.data_models.intermediate_step import IntermediateStepPayload
+from nat.data_models.invocation_node import InvocationNode
+from nat.eval.config import EvaluationRunConfig
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
+
+logger = logging.getLogger(__name__)
+
+# Constants for streaming response prefixes
+DATA_PREFIX = "data: "
+INTERMEDIATE_DATA_PREFIX = "intermediate_data: "
+
+
+class EvaluationRemoteWorkflowHandler:
+
+    def __init__(self, config: EvaluationRunConfig, max_concurrency: int):
+        self.config = config
+        # Run metadata
+        self.semaphore = asyncio.Semaphore(max_concurrency)
+
+    async def run_workflow_remote_single(self, session: aiohttp.ClientSession, item: EvalInputItem):
+        """
+        Sends a single input to the endpoint hosting the workflow and retrieves the response.
+        """
+        question = item.input_obj
+        # generate request format
+        payload = {"input_message": question}
+
+        try:
+            # Use the streaming endpoint
+            endpoint = f"{self.config.endpoint}/generate/full"
+            async with session.post(endpoint, json=payload) as response:
+                response.raise_for_status()  # Raise an exception for HTTP errors
+
+                # Initialize variables to store the response
+                final_response = None
+                intermediate_steps = []
+
+                # Process the streaming response
+                async for line in response.content:
+                    line = line.decode('utf-8').strip()
+                    if not line:
+                        continue
+
+                    if line.startswith(DATA_PREFIX):
+                        # This is a generate response chunk
+                        try:
+                            chunk_data = json.loads(line[len(DATA_PREFIX):])
+                            if chunk_data.get("value"):
+                                final_response = chunk_data.get("value")
+                        except json.JSONDecodeError as e:
+                            logger.error("Failed to parse generate response chunk: %s", e)
+                            continue
+                    elif line.startswith(INTERMEDIATE_DATA_PREFIX):
+                        # This is an intermediate step
+                        try:
+                            step_data = json.loads(line[len(INTERMEDIATE_DATA_PREFIX):])
+                            response_intermediate = ResponseIntermediateStep.model_validate(step_data)
+                            # The payload is expected to be IntermediateStepPayload
+                            payload = IntermediateStepPayload.model_validate_json(response_intermediate.payload)
+                            intermediate_step = IntermediateStep(parent_id="remote",
+                                                                 function_ancestry=InvocationNode(
+                                                                     function_name=payload.name or "remote_function",
+                                                                     function_id=payload.UUID or "remote_function_id"),
+                                                                 payload=payload)
+                            intermediate_steps.append(intermediate_step)
+                        except (json.JSONDecodeError, ValidationError) as e:
+                            logger.error("Failed to parse intermediate step: %s", e)
+                            continue
+
+        except aiohttp.ClientError as e:
+            # Handle connection or HTTP-related errors
+            logger.error("Request failed for question %s: %s", question, e)
+            item.output_obj = None
+            item.trajectory = []
+            return
+
+        # Extract and fill the item with the response and intermediate steps
+        item.output_obj = final_response
+        item.trajectory = intermediate_steps
+        return
+
+    async def run_workflow_remote_with_limits(self, session: aiohttp.ClientSession, item: EvalInputItem, pbar: tqdm):
+        """
+        Sends limited number of concurrent requests to a remote workflow and retrieves responses.
+        """
+        async with self.semaphore:
+            await self.run_workflow_remote_single(session=session, item=item)
+            pbar.update(1)
+
+    async def run_workflow_remote(self, eval_input: EvalInput) -> EvalInput:
+        """
+        Sends inputs to a workflow hosted on a remote endpoint.
+        """
+        timeout = aiohttp.ClientTimeout(total=self.config.endpoint_timeout)
+        try:
+            pbar = tqdm(total=len(eval_input.eval_input_items), desc="Running workflow", unit="item")
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                # get the questions from the eval_input
+                tasks = [
+                    self.run_workflow_remote_with_limits(session, item, pbar) for item in eval_input.eval_input_items
+                ]
+                await asyncio.gather(*tasks)
+
+        finally:
+            pbar.close()
+
+        return eval_input

nat/eval/runners/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nat/eval/runners/config.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import BaseModel
+
+from nat.eval.config import EvaluationRunConfig
+from nat.eval.config import EvaluationRunOutput
+
+
+class MultiEvaluationRunConfig(BaseModel):
+    """
+    Parameters used for a multi-evaluation run.
+    This includes a dict of configs. The key is an id of any type.
+    Each pass loads the config, applies the overrides and runs to completion
+    before the next pass starts.
+    """
+    configs: dict[typing.Any, EvaluationRunConfig]
+
+
+class MultiEvaluationRunOutput(BaseModel):
+    """
+    Output of a multi-evaluation run.
+    The results per-pass are accumulated in the evaluation_run_outputs dict.
+    """
+    evaluation_run_outputs: dict[typing.Any, EvaluationRunOutput]