nvidia-nat 1.3.0a20250823__py3-none-any.whl → 1.3.0a20250826__py3-none-any.whl

nat/data_models/evaluate.py

@@ -57,6 +57,9 @@ class EvalOutputConfig(BaseModel):
     dir: Path = Path("./.tmp/nat/examples/default/")
     # S3 prefix for the workflow and evaluation results
     remote_dir: str | None = None
+    # Custom function to pre-evaluation process the eval input
+    # Format: "module.path.function_name"
+    custom_pre_eval_process_function: str | None = None
     # Custom scripts to run after the workflow and evaluation results are saved
     custom_scripts: dict[str, EvalCustomScriptConfig] = {}
     # S3 config for uploading the contents of the output directory

nat/data_models/gated_field_mixin.py

@@ -0,0 +1,244 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+from re import Pattern
+from typing import Generic
+from typing import TypeVar
+
+from pydantic import model_validator
+
+
+@dataclass
+class GatedFieldMixinConfig:
+    """Configuration for a gated field mixin."""
+
+    field_name: str
+    default_if_supported: object | None
+    unsupported: Sequence[Pattern[str]] | None
+    supported: Sequence[Pattern[str]] | None
+    keys: Sequence[str]
+
+
+T = TypeVar("T")
+
+
+class GatedFieldMixin(Generic[T]):
+    """
+    A mixin that gates a field based on specified keys.
+
+    This should be used to automatically validate a field based on a given key.
+
+    Parameters
+    ----------
+    field_name: `str`
+                The name of the field.
+    default_if_supported: `T | None`
+                          The default value of the field if it is supported for the key.
+    keys: `Sequence[str]`
+          A sequence of keys that are used to validate the field.
+    unsupported: `Sequence[Pattern[str]] | None`
+                 A sequence of regex patterns that match the key names NOT supported for the field.
+                 Defaults to None.
+    supported: `Sequence[Pattern[str]] | None`
+               A sequence of regex patterns that match the key names supported for the field.
+               Defaults to None.
+    """
+
+    def __init_subclass__(
+        cls,
+        field_name: str | None = None,
+        default_if_supported: T | None = None,
+        keys: Sequence[str] | None = None,
+        unsupported: Sequence[Pattern[str]] | None = None,
+        supported: Sequence[Pattern[str]] | None = None,
+    ) -> None:
+        """Store the class variables for the field and define the gated field validator."""
+        super().__init_subclass__()
+
+        # Check if this class directly inherits from GatedFieldMixin
+        has_gated_field_mixin = GatedFieldMixin in cls.__bases__
+
+        if has_gated_field_mixin:
+            if keys is None:
+                raise ValueError("keys must be provided when subclassing GatedFieldMixin")
+            if field_name is None:
+                raise ValueError("field_name must be provided when subclassing GatedFieldMixin")
+
+            cls._setup_direct_mixin(field_name, default_if_supported, unsupported, supported, keys)
+
+        # Always try to collect mixins and create validators for multiple inheritance
+        # This handles both direct inheritance and deep inheritance chains
+        all_mixins = cls._collect_all_mixin_configs()
+        if all_mixins:
+            cls._create_combined_validator(all_mixins)
+
+    @classmethod
+    def _setup_direct_mixin(
+        cls,
+        field_name: str,
+        default_if_supported: T | None,
+        unsupported: Sequence[Pattern[str]] | None,
+        supported: Sequence[Pattern[str]] | None,
+        keys: Sequence[str],
+    ) -> None:
+        """Set up a class that directly inherits from GatedFieldMixin."""
+        cls._validate_mixin_parameters(unsupported, supported, keys)
+
+        # Create and store validator
+        validator = cls._create_gated_field_validator(field_name, default_if_supported, unsupported, supported, keys)
+        validator_name = f"_gated_field_validator_{field_name}"
+        setattr(cls, validator_name, validator)
+
+        # Store mixin info for multiple inheritance
+        if not hasattr(cls, "_gated_field_mixins"):
+            cls._gated_field_mixins = []
+
+        cls._gated_field_mixins.append(
+            GatedFieldMixinConfig(
+                field_name,
+                default_if_supported,
+                unsupported,
+                supported,
+                keys,
+            ))
+
+    @classmethod
+    def _validate_mixin_parameters(
+        cls,
+        unsupported: Sequence[Pattern[str]] | None,
+        supported: Sequence[Pattern[str]] | None,
+        keys: Sequence[str],
+    ) -> None:
+        """Validate that all required parameters are provided."""
+        if unsupported is None and supported is None:
+            raise ValueError("Either unsupported or supported must be provided")
+        if unsupported is not None and supported is not None:
+            raise ValueError("Only one of unsupported or supported must be provided")
+        if len(keys) == 0:
+            raise ValueError("keys must be provided and non-empty when subclassing GatedFieldMixin")
+
+    @classmethod
+    def _create_gated_field_validator(
+        cls,
+        field_name: str,
+        default_if_supported: T | None,
+        unsupported: Sequence[Pattern[str]] | None,
+        supported: Sequence[Pattern[str]] | None,
+        keys: Sequence[str],
+    ):
+        """Create the model validator function."""
+
+        @model_validator(mode="after")
+        def gated_field_validator(self):
+            """Validate the gated field."""
+            current_value = getattr(self, field_name, None)
+            is_supported = cls._check_field_support(self, unsupported, supported, keys)
+            if not is_supported:
+                if current_value is not None:
+                    blocking_key = cls._find_blocking_key(self, unsupported, supported, keys)
+                    value = getattr(self, blocking_key, "<unknown>")
+                    raise ValueError(f"{field_name} is not supported for {blocking_key}: {value}")
+            elif current_value is None:
+                setattr(self, field_name, default_if_supported)
+            return self
+
+        return gated_field_validator
+
+    @classmethod
+    def _check_field_support(
+        cls,
+        instance: object,
+        unsupported: Sequence[Pattern[str]] | None,
+        supported: Sequence[Pattern[str]] | None,
+        keys: Sequence[str],
+    ) -> bool:
+        """Check if a specific field is supported based on its configuration and keys."""
+        for key in keys:
+            if not hasattr(instance, key):
+                continue
+            value = str(getattr(instance, key))
+            if supported is not None:
+                return any(p.search(value) for p in supported)
+            elif unsupported is not None:
+                return not any(p.search(value) for p in unsupported)
+        # Default to supported if no model keys found
+        return True
+
+    @classmethod
+    def _find_blocking_key(
+        cls,
+        instance: object,
+        unsupported: Sequence[Pattern[str]] | None,
+        supported: Sequence[Pattern[str]] | None,
+        keys: Sequence[str],
+    ) -> str:
+        """Find which key is blocking the field."""
+        for key in keys:
+            if not hasattr(instance, key):
+                continue
+            value = str(getattr(instance, key))
+            if supported is not None:
+                if not any(p.search(value) for p in supported):
+                    return key
+            elif unsupported is not None:
+                if any(p.search(value) for p in unsupported):
+                    return key
+
+        return "<unknown>"
+
+    @classmethod
+    def _collect_all_mixin_configs(cls) -> list[GatedFieldMixinConfig]:
+        """Collect all mixin configurations from base classes."""
+        all_mixins = []
+        for base in cls.__bases__:
+            if hasattr(base, "_gated_field_mixins"):
+                all_mixins.extend(base._gated_field_mixins)
+        return all_mixins
+
+    @classmethod
+    def _create_combined_validator(cls, all_mixins: list[GatedFieldMixinConfig]) -> None:
+        """Create a combined validator that handles all fields."""
+
+        @model_validator(mode="after")
+        def combined_gated_field_validator(self):
+            """Validate all gated fields."""
+            for mixin_config in all_mixins:
+                field_name_local = mixin_config.field_name
+                current_value = getattr(self, field_name_local, None)
+                if not self._check_field_support_instance(mixin_config):
+                    if current_value is not None:
+                        blocking_key = self._find_blocking_key_instance(mixin_config)
+                        value = getattr(self, blocking_key, "<unknown>")
+                        raise ValueError(f"{field_name_local} is not supported for {blocking_key}: {value}")
+                elif current_value is None:
+                    setattr(self, field_name_local, mixin_config.default_if_supported)
+
+            return self
+
+        cls._combined_gated_field_validator = combined_gated_field_validator
+
+        # Add helper methods
+        def _check_field_support_instance(self, mixin_config: GatedFieldMixinConfig) -> bool:
+            """Check if a specific field is supported based on its configuration and keys."""
+            return cls._check_field_support(self, mixin_config.unsupported, mixin_config.supported, mixin_config.keys)
+
+        def _find_blocking_key_instance(self, mixin_config: GatedFieldMixinConfig) -> str:
+            """Find which key is blocking the field."""
+            return cls._find_blocking_key(self, mixin_config.unsupported, mixin_config.supported, mixin_config.keys)
+
+        cls._check_field_support_instance = _check_field_support_instance
+        cls._find_blocking_key_instance = _find_blocking_key_instance

nat/data_models/temperature_mixin.py

@@ -18,19 +18,26 @@ import re
 from pydantic import BaseModel
 from pydantic import Field
 
-from nat.data_models.model_gated_field_mixin import ModelGatedFieldMixin
-
-_UNSUPPORTED_TEMPERATURE_MODELS = (re.compile(r"gpt-?5", re.IGNORECASE), )
+from nat.data_models.gated_field_mixin import GatedFieldMixin
 
 
 class TemperatureMixin(
         BaseModel,
-        ModelGatedFieldMixin[float],
+        GatedFieldMixin[float],
         field_name="temperature",
         default_if_supported=0.0,
-        unsupported_models=_UNSUPPORTED_TEMPERATURE_MODELS,
+        keys=("model_name", "model", "azure_deployment"),
+        unsupported=(re.compile(r"gpt-?5", re.IGNORECASE), ),
 ):
     """
-    Mixin class for temperature configuration.
+    Mixin class for temperature configuration. Unsupported on models like gpt-5.
+
+    Attributes:
+        temperature: Sampling temperature in [0, 1]. Defaults to 0.0 when supported on the model.
     """
-    temperature: float | None = Field(default=None, ge=0.0, le=1.0, description="Sampling temperature in [0, 1].")
+    temperature: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description="Sampling temperature in [0, 1]. Defaults to 0.0 when supported on the model.",
+    )

nat/data_models/top_p_mixin.py

@@ -18,19 +18,26 @@ import re
 from pydantic import BaseModel
 from pydantic import Field
 
-from nat.data_models.model_gated_field_mixin import ModelGatedFieldMixin
-
-_UNSUPPORTED_TOP_P_MODELS = (re.compile(r"gpt-?5", re.IGNORECASE), )
+from nat.data_models.gated_field_mixin import GatedFieldMixin
 
 
 class TopPMixin(
         BaseModel,
-        ModelGatedFieldMixin[float],
+        GatedFieldMixin[float],
         field_name="top_p",
         default_if_supported=1.0,
-        unsupported_models=_UNSUPPORTED_TOP_P_MODELS,
+        keys=("model_name", "model", "azure_deployment"),
+        unsupported=(re.compile(r"gpt-?5", re.IGNORECASE), ),
 ):
     """
-    Mixin class for top-p configuration.
+    Mixin class for top-p configuration. Unsupported on models like gpt-5.
+
+    Attributes:
+        top_p: Top-p for distribution sampling. Defaults to 1.0 when supported on the model.
     """
-    top_p: float | None = Field(default=None, ge=0.0, le=1.0, description="Top-p for distribution sampling.")
+    top_p: float | None = Field(
+        default=None,
+        ge=0.0,
+        le=1.0,
+        description="Top-p for distribution sampling. Defaults to 1.0 when supported on the model.",
+    )

nat/eval/dataset_handler/dataset_handler.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import importlib
 import json
 import math
 from pathlib import Path
@@ -41,7 +42,8 @@ class DatasetHandler:
                  reps: int,
                  concurrency: int,
                  num_passes: int = 1,
-                 adjust_dataset_size: bool = False):
+                 adjust_dataset_size: bool = False,
+                 custom_pre_eval_process_function: str | None = None):
         from nat.eval.intermediate_step_adapter import IntermediateStepAdapter
 
         self.dataset_config = dataset_config
@@ -53,6 +55,9 @@ class DatasetHandler:
         self.num_passes = num_passes
         self.adjust_dataset_size = adjust_dataset_size
 
+        # Custom pre-evaluation process function
+        self.custom_pre_eval_process_function = custom_pre_eval_process_function
+
         # Helpers
         self.intermediate_step_adapter = IntermediateStepAdapter()
 
@@ -330,6 +335,66 @@ class DatasetHandler:
         filtered_steps = self.intermediate_step_adapter.filter_intermediate_steps(intermediate_steps, event_filter)
         return self.intermediate_step_adapter.serialize_intermediate_steps(filtered_steps)
 
+    def pre_eval_process_eval_input(self, eval_input: EvalInput) -> EvalInput:
+        """
+        Pre-evaluation process the eval input using custom function if provided.
+
+        The custom pre-evaluation process function should have the signature:
+        def custom_pre_eval_process(item: EvalInputItem) -> EvalInputItem
+
+        The framework will iterate through all items and call this function on each one.
+
+        Args:
+            eval_input: The EvalInput object to pre-evaluation process
+
+        Returns:
+            The pre-evaluation processed EvalInput object
+        """
+        if self.custom_pre_eval_process_function:
+            try:
+                custom_function = self._load_custom_pre_eval_process_function()
+                processed_items = []
+
+                for item in eval_input.eval_input_items:
+                    processed_item = custom_function(item)
+                    if not isinstance(processed_item, EvalInputItem):
+                        raise TypeError(f"Custom pre-evaluation '{self.custom_pre_eval_process_function}' must return "
+                                        f"EvalInputItem, got {type(processed_item)}")
+                    processed_items.append(processed_item)
+
+                return EvalInput(eval_input_items=processed_items)
+            except Exception as e:
+                raise RuntimeError(f"Error calling custom pre-evaluation process function "
+                                   f"'{self.custom_pre_eval_process_function}': {e}") from e
+
+        return eval_input
+
+    def _load_custom_pre_eval_process_function(self):
+        """
+        Import and return the custom pre-evaluation process function using standard Python import path.
+
+        The function should process individual EvalInputItem objects.
+        """
+        # Split the function path to get module and function name
+        if "." not in self.custom_pre_eval_process_function:
+            raise ValueError(f"Invalid custom_pre_eval_process_function '{self.custom_pre_eval_process_function}'. "
+                             "Expected format: '<module_path>.<function_name>'")
+        module_path, function_name = self.custom_pre_eval_process_function.rsplit(".", 1)
+
+        # Import the module
+        module = importlib.import_module(module_path)
+
+        # Get the function from the module
+        if not hasattr(module, function_name):
+            raise AttributeError(f"Function '{function_name}' not found in module '{module_path}'")
+
+        custom_function = getattr(module, function_name)
+
+        if not callable(custom_function):
+            raise ValueError(f"'{self.custom_pre_eval_process_function}' is not callable")
+
+        return custom_function
+
     def publish_eval_input(self,
                            eval_input,
                            workflow_output_step_filter: list[IntermediateStepType] | None = None) -> str:

nat/eval/evaluate.py

@@ -486,11 +486,14 @@ class EvaluationRun:
                                        usage_stats=UsageStats(),
                                        profiler_results=ProfilerResults())
 
+        custom_pre_eval_process_function = self.eval_config.general.output.custom_pre_eval_process_function \
+            if self.eval_config.general.output else None
         dataset_handler = DatasetHandler(dataset_config=dataset_config,
                                          reps=self.config.reps,
                                          concurrency=self.eval_config.general.max_concurrency,
                                          num_passes=self.config.num_passes,
-                                         adjust_dataset_size=self.config.adjust_dataset_size)
+                                         adjust_dataset_size=self.config.adjust_dataset_size,
+                                         custom_pre_eval_process_function=custom_pre_eval_process_function)
         self.eval_input = dataset_handler.get_eval_input_from_dataset(self.config.dataset)
         if not self.eval_input.eval_input_items:
             logger.info("Dataset is empty. Nothing to evaluate.")
@@ -507,8 +510,8 @@ class EvaluationRun:
             # Initialize Weave integration
             self.weave_eval.initialize_logger(workflow_alias, self.eval_input, config)
 
-            # Run workflow
             with self.eval_trace_context.evaluation_context():
+                # Run workflow
                 if self.config.endpoint:
                     await self.run_workflow_remote()
                 elif not self.config.skip_workflow:
@@ -517,6 +520,9 @@ class EvaluationRun:
                                                          max_concurrency=self.eval_config.general.max_concurrency)
                     await self.run_workflow_local(session_manager)
 
+                # Pre-evaluation process the workflow output
+                self.eval_input = dataset_handler.pre_eval_process_eval_input(self.eval_input)
+
                 # Evaluate
                 evaluators = {name: eval_workflow.get_evaluator(name) for name in self.eval_config.evaluators}
                 await self.run_evaluators(evaluators)

nat/eval/evaluator/evaluator_model.py

@@ -29,6 +29,19 @@ class EvalInputItem(BaseModel):
     trajectory: list[IntermediateStep] = []  # populated by the workflow
     full_dataset_entry: typing.Any
 
+    def copy_with_updates(self, **updates) -> "EvalInputItem":
+        """
+        Copy EvalInputItem with optional field updates.
+        """
+        # Get all current fields
+        item_data = self.model_dump()
+
+        # Apply any updates
+        item_data.update(updates)
+
+        # Create new item with all fields
+        return EvalInputItem(**item_data)
+
 
 class EvalInput(BaseModel):
     eval_input_items: list[EvalInputItem]

{nvidia_nat-1.3.0a20250823.dist-info → nvidia_nat-1.3.0a20250826.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nvidia-nat
-Version: 1.3.0a20250823
+Version: 1.3.0a20250826
 Summary: NVIDIA NeMo Agent toolkit
 Author: NVIDIA Corporation
 Maintainer: NVIDIA Corporation

{nvidia_nat-1.3.0a20250823.dist-info → nvidia_nat-1.3.0a20250826.dist-info}/RECORD

@@ -103,18 +103,18 @@ nat/data_models/config.py,sha256=VBin3qeWxX8DUkw5lJ6RD7dStR7qPp52XIsy0cJkUP4,171
 nat/data_models/dataset_handler.py,sha256=UDQLpStUwLEMm0e6BvuOFEsHYSopWu5DhinxEv2U6og,5523
 nat/data_models/discovery_metadata.py,sha256=cIAheyVuWksB5Tnv8yB65k4kPaCSUXrbcO75O7puUTc,13492
 nat/data_models/embedder.py,sha256=nPhthEQDtzAMGd8gFRB1ZfJpN5M9DJvv0h28ohHnTmI,1002
-nat/data_models/evaluate.py,sha256=4d9tbLXdSqQ8YUIXB3xbZ0nYnGUC1n2ioFbR_RVGQoI,4675
+nat/data_models/evaluate.py,sha256=L0GdNh_c8jii-MiK8oHW9sUUsGO3l1FMsprr-UazT5c,4836
 nat/data_models/evaluator.py,sha256=bd2njsyQB2t6ClJ66gJiCjYHsQpWZwPD7rsU0J109TI,939
 nat/data_models/front_end.py,sha256=z8k6lSWjt1vMOYFbjWQxodpwAqPeuGS0hRBjsriDW2s,932
 nat/data_models/function.py,sha256=M_duXVXL5MvYe0WVLvqEgEzXs0UAYNSMfy9ZTpxuKPA,1013
 nat/data_models/function_dependencies.py,sha256=pMTSZH2vFfW8DEmz60SAUZUeJtWsjRKtN5elx0ACtY0,2558
+nat/data_models/gated_field_mixin.py,sha256=94jmbB6dYciBuqb2BODKh6F73LHsUN-aInOY_xNlcQ8,9793
 nat/data_models/interactive.py,sha256=qOkxyYPQYEBIBMDAA1rjfYcdvf6-iCM4qPV8Awc4VGw,8794
 nat/data_models/intermediate_step.py,sha256=OhdGAtSBiMiCfT1R-BjihUvcVoQsRsxHDQcJ1gMDVXA,11585
 nat/data_models/invocation_node.py,sha256=nDRylgzBfJduGA-lme9xN4P6BdOYj0L6ytLHnTuetMI,1618
 nat/data_models/llm.py,sha256=PCTeCPg2YnYk5tvSu7XOEVr2Cg_wSkjAdVsoiTg8Joo,968
 nat/data_models/logging.py,sha256=1QtVjIQ99PgMYUuzw4h1FAoPRteZY7uf3oFTqV3ONgA,940
 nat/data_models/memory.py,sha256=IKwe7CflCto30j4yI5yQtq8DXfMilAJ17S5NcsSDrOQ,1052
-nat/data_models/model_gated_field_mixin.py,sha256=AvhYO7GNciRmGLfWu9QOhYLipwRcJheCxHBxcDmsp_k,5653
 nat/data_models/object_store.py,sha256=S8YY6i8ALgRPuggUI1FCG-xbvwPWuaCg1lJnZOx5scM,1515
 nat/data_models/profiler.py,sha256=z3IlEhj-veB4Yz85271bTkScSUkVwK50tR3dwlDRgcE,1781
 nat/data_models/registry_handler.py,sha256=g1rFaz4uSydMJn7qpdX-DNHJd_rNf8tXYN49dLDYHPo,968
@@ -125,8 +125,8 @@ nat/data_models/step_adaptor.py,sha256=1qn56wB_nIYBM5IjN4ftsltCAkqaJd3Sqe5v0TRR4
 nat/data_models/streaming.py,sha256=sSqJqLqb70qyw69_4R9QC2RMbRw7UjTLPdo3FYBUGxE,1159
 nat/data_models/swe_bench_model.py,sha256=uZs-hLFuT1B5CiPFwFg1PHinDW8PHne8TBzu7tHFv_k,1718
 nat/data_models/telemetry_exporter.py,sha256=P7kqxIQnFVuvo_UFpH9QSB8fACy_0U2Uzkw_IfWXagE,998
-nat/data_models/temperature_mixin.py,sha256=qra4YH6xmrwqMPo27zylbEFJz6z8XAL9b1U1-NyJ3_s,1291
-nat/data_models/top_p_mixin.py,sha256=4A3xa9gp2qv41jh6OqRL75q_iCpGa-GaKEExEW91X-s,1255
+nat/data_models/temperature_mixin.py,sha256=HI6xJscmAv10JUmK8gk909mMifN1GW-qkswk9stC5ks,1492
+nat/data_models/top_p_mixin.py,sha256=ld25YD4Xf70Odbl9qSHuaf391bmB5qgvON2LsuOfhlM,1463
 nat/data_models/ttc_strategy.py,sha256=tAkKWcyEBmBOOYtHMtQTgeCbHxFTk5SEkmFunNVnfyE,1114
 nat/embedder/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nat/embedder/azure_openai_embedder.py,sha256=8OM54DCVCmWuQ8eZ5lXxsKYQktRmHMoIelRHZmKurUk,2381
@@ -135,7 +135,7 @@ nat/embedder/openai_embedder.py,sha256=uxyEe_2R02RPvEDGdzO9HUNaekm0MVRxJPCPB5-CS
 nat/embedder/register.py,sha256=TM_LKuSlJr3tEceNVuHfAx_yrCzf1sryD5Ycep5rNGo,883
 nat/eval/__init__.py,sha256=Xs1JQ16L9btwreh4pdGKwskffAw1YFO48jKrU4ib_7c,685
 nat/eval/config.py,sha256=de_DrLalzobILLJDTevGhbY_zjaoMyzMqVhDjbtpjk8,2365
-nat/eval/evaluate.py,sha256=vF1LmZM7ixQXqb4fj2kF6qI_8dQaknZpccfu-5OdXtY,26115
+nat/eval/evaluate.py,sha256=a8rH1YEzS3Z7yzPWqWPDZpvR9C4ZHxoTXbc2lxPDmso,26551
 nat/eval/intermediate_step_adapter.py,sha256=E1AqH0hO_NNUT7nRGkIllitYJAszjjubul0IUwq-cyc,4498
 nat/eval/register.py,sha256=p8nM_P9yLWrglwZtTKAVjonAXfqzgBEwvYk1zlg4uw0,1044
 nat/eval/remote_workflow.py,sha256=IUsWHbAShBY5GgTsG9qTwK-SqRwrSNZM9CUfzBhlWo8,6012
@@ -144,10 +144,10 @@ nat/eval/usage_stats.py,sha256=_d_ErIvSKDN8wuvOyPn01kqM7zlFpwVxqOonaCV5XtY,1319
 nat/eval/dataset_handler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nat/eval/dataset_handler/dataset_downloader.py,sha256=w5Kjj7Dn2gQ14ekgt3rtGT8EPGJtXXXlckbtby9rJ2k,4553
 nat/eval/dataset_handler/dataset_filter.py,sha256=HrK0m3SQnPX6zOcKwJwYMrD9O-f6aOw8Vo3j9RKszqE,2115
-nat/eval/dataset_handler/dataset_handler.py,sha256=ii-y3RtHO7EZVajerSkW68UTH38nJbdou1Cuv1C9o4I,15233
+nat/eval/dataset_handler/dataset_handler.py,sha256=my28rKvQiiRN_h2TJz6fdKeMOjP3LC3_e2aJNnPPYhE,18159
 nat/eval/evaluator/__init__.py,sha256=GUJrgGtpvyMUCjUBvR3faAdv-tZzbU9W-izgx9aMEQg,680
 nat/eval/evaluator/base_evaluator.py,sha256=pubxUm2DXkruSiYPkzezPj3-gzFEDAFGTYYTw7iXglU,3233
-nat/eval/evaluator/evaluator_model.py,sha256=Mjf6kVpL__zCoL2B7vRkXxdNXRdFC3bJ6giRPZPLEzI,1515
+nat/eval/evaluator/evaluator_model.py,sha256=riGCcDW8YwC3Kd1yoVmbMdJE1Yf2kVmO8uhsGsKKJA4,1878
 nat/eval/rag_evaluator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nat/eval/rag_evaluator/evaluate.py,sha256=qvpa6FdQNQmChU6op1aELw9_Dabd81cSQ-HHQVOBjwc,8399
 nat/eval/rag_evaluator/register.py,sha256=3OZcQ0Q7K_2zxNTD7p6EY29Zv--qO4ExQkGdJszDKf0,5831
@@ -432,10 +432,10 @@ nat/utils/reactive/base/observer_base.py,sha256=6BiQfx26EMumotJ3KoVcdmFBYR_fnAss
 nat/utils/reactive/base/subject_base.py,sha256=UQOxlkZTIeeyYmG5qLtDpNf_63Y7p-doEeUA08_R8ME,2521
 nat/utils/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nat/utils/settings/global_settings.py,sha256=4tVWrZMDAuH_D6pdyiWrE1yAs61PBY99ssCHkDcbs7g,7324
-nvidia_nat-1.3.0a20250823.dist-info/licenses/LICENSE-3rd-party.txt,sha256=fOk5jMmCX9YoKWyYzTtfgl-SUy477audFC5hNY4oP7Q,284609
-nvidia_nat-1.3.0a20250823.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-nvidia_nat-1.3.0a20250823.dist-info/METADATA,sha256=jdTUL524mA1IjR55AW48aydzoct5x5oMepviLo32VhQ,21762
-nvidia_nat-1.3.0a20250823.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nvidia_nat-1.3.0a20250823.dist-info/entry_points.txt,sha256=FNh4pZVSe_61s29zdks66lmXBPtsnko8KSZ4ffv7WVE,653
-nvidia_nat-1.3.0a20250823.dist-info/top_level.txt,sha256=lgJWLkigiVZuZ_O1nxVnD_ziYBwgpE2OStdaCduMEGc,8
-nvidia_nat-1.3.0a20250823.dist-info/RECORD,,
+nvidia_nat-1.3.0a20250826.dist-info/licenses/LICENSE-3rd-party.txt,sha256=fOk5jMmCX9YoKWyYzTtfgl-SUy477audFC5hNY4oP7Q,284609
+nvidia_nat-1.3.0a20250826.dist-info/licenses/LICENSE.md,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+nvidia_nat-1.3.0a20250826.dist-info/METADATA,sha256=bGIcgBdPnY_qUO-k11S-YYBNG_k5wIQ1b3HX-FeZVQo,21762
+nvidia_nat-1.3.0a20250826.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nvidia_nat-1.3.0a20250826.dist-info/entry_points.txt,sha256=FNh4pZVSe_61s29zdks66lmXBPtsnko8KSZ4ffv7WVE,653
+nvidia_nat-1.3.0a20250826.dist-info/top_level.txt,sha256=lgJWLkigiVZuZ_O1nxVnD_ziYBwgpE2OStdaCduMEGc,8
+nvidia_nat-1.3.0a20250826.dist-info/RECORD,,

nat/data_models/model_gated_field_mixin.py

@@ -1,125 +0,0 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-# SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from collections.abc import Sequence
-from re import Pattern
-from typing import Generic
-from typing import TypeVar
-
-from pydantic import BaseModel
-from pydantic import model_validator
-
-T = TypeVar("T")
-
-
-class ModelGatedFieldMixin(Generic[T]):
-    """
-    A mixin that gates a field based on model support.
-
-    This should be used to automatically validate a field based on a given model.
-
-    Parameters
-    ----------
-    field_name: `str`
-                The name of the field.
-    default_if_supported: `T`
-                          The default value of the field if it is supported for the model.
-    unsupported_models: `Sequence[Pattern[str]] | None`
-                        A sequence of regex patterns that match the model names NOT supported for the field.
-                        Defaults to None.
-    supported_models: `Sequence[Pattern[str]] | None`
-                      A sequence of regex patterns that match the model names supported for the field.
-                      Defaults to None.
-    model_keys: `Sequence[str]`
-                A sequence of keys that are used to validate the field.
-                Defaults to ("model_name", "model", "azure_deployment",)
-    """
-
-    def __init_subclass__(
-            cls,
-            field_name: str | None = None,
-            default_if_supported: T | None = None,
-            unsupported_models: Sequence[Pattern[str]] | None = None,
-            supported_models: Sequence[Pattern[str]] | None = None,
-            model_keys: Sequence[str] = ("model_name", "model", "azure_deployment"),
-    ) -> None:
-        """
-        Store the class variables for the field and define the model validator.
-        """
-        super().__init_subclass__()
-        if ModelGatedFieldMixin in cls.__bases__:
-            if field_name is None:
-                raise ValueError("field_name must be provided when subclassing ModelGatedFieldMixin")
-            if default_if_supported is None:
-                raise ValueError("default_if_supported must be provided when subclassing ModelGatedFieldMixin")
-            if unsupported_models is None and supported_models is None:
-                raise ValueError("Either unsupported_models or supported_models must be provided")
-            if unsupported_models is not None and supported_models is not None:
-                raise ValueError("Only one of unsupported_models or supported_models must be provided")
-            if model_keys is not None and len(model_keys) == 0:
-                raise ValueError("model_keys must be provided and non-empty when subclassing ModelGatedFieldMixin")
-            cls.field_name = field_name
-            cls.default_if_supported = default_if_supported
-            cls.unsupported_models = unsupported_models
-            cls.supported_models = supported_models
-            if model_keys is not None:
-                cls.model_keys = model_keys
-
-            @classmethod
-            def check_model(cls, model_name: str) -> bool:
-                """
-                Check if a model is supported for a given field.
-
-                Args:
-                    model_name: The name of the model to check.
-                """
-                unsupported = getattr(cls, "unsupported_models", None)
-                supported = getattr(cls, "supported_models", None)
-                if unsupported is not None:
-                    return not any(p.search(model_name) for p in unsupported)
-                if supported is not None:
-                    return any(p.search(model_name) for p in supported)
-                return False
-
-            cls._model_gated_field_check_model = check_model
-
-            @classmethod
-            def detect_support(cls, instance: BaseModel) -> str | None:
-                for key in getattr(cls, "model_keys"):
-                    if hasattr(instance, key):
-                        model_name_value = getattr(instance, key)
-                        is_supported = getattr(cls, "_model_gated_field_check_model")(str(model_name_value))
-                        return key if not is_supported else None
-                return None
-
-            cls._model_gated_field_detect_support = detect_support
-
-            @model_validator(mode="after")
-            def model_validate(self):
-                klass = self.__class__
-
-                field_name_local = getattr(klass, "field_name")
-                current_value = getattr(self, field_name_local, None)
-
-                found_key = klass._model_gated_field_detect_support(self)
-                if found_key is not None:
-                    if current_value is not None:
-                        raise ValueError(
-                            f"{field_name_local} is not supported for {found_key}: {getattr(self, found_key)}")
-                elif current_value is None:
-                    setattr(self, field_name_local, getattr(klass, "default_if_supported", None))
-                return self
-
-            cls._model_gated_field_model_validator = model_validate