nvidia-nat 1.2.0rc5__py3-none-any.whl → 1.2.0rc7__py3-none-any.whl

{aiq → nat}/eval/dataset_handler/dataset_handler.py

@@ -15,17 +15,19 @@
 
 import json
 import math
+from pathlib import Path
 
 import pandas as pd
 
-from aiq.data_models.dataset_handler import EvalDatasetConfig
-from aiq.data_models.dataset_handler import EvalDatasetJsonConfig
-from aiq.data_models.intermediate_step import IntermediateStep
-from aiq.data_models.intermediate_step import IntermediateStepType
-from aiq.eval.dataset_handler.dataset_downloader import DatasetDownloader
-from aiq.eval.dataset_handler.dataset_filter import DatasetFilter
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
+from nat.data_models.dataset_handler import EvalDatasetConfig
+from nat.data_models.dataset_handler import EvalDatasetCustomConfig
+from nat.data_models.dataset_handler import EvalDatasetJsonConfig
+from nat.data_models.intermediate_step import IntermediateStep
+from nat.data_models.intermediate_step import IntermediateStepType
+from nat.eval.dataset_handler.dataset_downloader import DatasetDownloader
+from nat.eval.dataset_handler.dataset_filter import DatasetFilter
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
 
 
 class DatasetHandler:
@@ -38,9 +40,9 @@ class DatasetHandler:
                  dataset_config: EvalDatasetConfig,
                  reps: int,
                  concurrency: int,
-                 num_passes: int | None = None,
+                 num_passes: int = 1,
                  adjust_dataset_size: bool = False):
-        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
+        from nat.eval.intermediate_step_adapter import IntermediateStepAdapter
 
         self.dataset_config = dataset_config
         self.dataset_filter = DatasetFilter(dataset_config.filter)
@@ -184,6 +186,10 @@ class DatasetHandler:
         # if a dataset file has been provided in the command line, use that
         dataset_config = EvalDatasetJsonConfig(file_path=dataset) if dataset else self.dataset_config
 
+        # Handle custom dataset type with special processing
+        if isinstance(self.dataset_config, EvalDatasetCustomConfig):
+            return self._handle_custom_dataset(dataset)
+
         # Download the dataset if it is remote
         downloader = DatasetDownloader(dataset_config=dataset_config)
         downloader.download_dataset()
@@ -192,6 +198,19 @@ class DatasetHandler:
         # Parse the dataset into a DataFrame
         input_df = parser(dataset_config.file_path, **kwargs)
 
+        # Apply standard preprocessing and convert to EvalInput
+        return self._preprocess_eval_dataframe(input_df)
+
+    def _preprocess_dataframe(self, input_df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Apply standard preprocessing to a DataFrame: filters, deduplication, repetitions, and size adjustment.
+
+        Args:
+            input_df: DataFrame to preprocess
+
+        Returns:
+            Preprocessed DataFrame
+        """
         # Apply filters and deduplicate
         input_df = self.dataset_filter.apply_filters(input_df)
         input_df.drop_duplicates(subset=[self.dataset_config.id_key], inplace=True)
@@ -205,12 +224,104 @@ class DatasetHandler:
         elif self.adjust_dataset_size:
             input_df = self.adjust_dataset(input_df)
 
-        # Convert the DataFrame to a list of EvalInput objects
-        return self.get_eval_input_from_df(input_df)
+        return input_df
+
+    def _preprocess_eval_dataframe(self, input_df: pd.DataFrame) -> EvalInput:
+        """
+        Apply standard preprocessing to a DataFrame and convert to EvalInput.
+
+        Args:
+            input_df: DataFrame to preprocess
+
+        Returns:
+            Preprocessed EvalInput object
+        """
+        processed_df = self._preprocess_dataframe(input_df)
+        return self.get_eval_input_from_df(processed_df)
+
+    def _preprocess_eval_input(self, eval_input: EvalInput) -> EvalInput:
+        """
+        Apply standard preprocessing to an EvalInput object.
+
+        Thin wrapper that converts EvalInput to DataFrame, processes it, and converts back.
+
+        Args:
+            eval_input: EvalInput object to preprocess
+
+        Returns:
+            Preprocessed EvalInput object
+        """
+        if not eval_input.eval_input_items:
+            return eval_input
+
+        input_df = self._eval_input_to_dataframe(eval_input)
+        return self._preprocess_eval_dataframe(input_df)
+
+    def _handle_custom_dataset(self, dataset: str | None) -> EvalInput:
+        """
+        Handle custom dataset type by calling the user-defined function
+        and applying standard preprocessing to the result.
+
+        Args:
+            dataset: Optional dataset file path from command line
+
+        Returns:
+            Preprocessed EvalInput object
+        """
+        # Determine input path - use command line dataset or config file_path
+        input_path = Path(dataset) if dataset else Path(self.dataset_config.file_path)
+
+        # Download the dataset if it is remote (for custom datasets too)
+        downloader = DatasetDownloader(dataset_config=self.dataset_config)
+        downloader.download_dataset()
+
+        # Load and call custom function
+        custom_function, kwargs = self.dataset_config.parser()
+
+        try:
+            # Call the custom function with file_path and kwargs
+            eval_input = custom_function(file_path=input_path, **kwargs)
+
+            if not isinstance(eval_input, EvalInput):
+                raise ValueError(f"Custom function must return an EvalInput object, "
+                                 f"but returned {type(eval_input)}")
+
+        except Exception as e:
+            raise RuntimeError(f"Error calling custom dataset function: {e}") from e
+
+        # Apply standard preprocessing (filters, deduplication, repetitions)
+        return self._preprocess_eval_input(eval_input)
+
+    def _eval_input_to_dataframe(self, eval_input: EvalInput) -> pd.DataFrame:
+        """
+        Convert an EvalInput object to a pandas DataFrame for processing.
+
+        Args:
+            eval_input: EvalInput object to convert
+
+        Returns:
+            DataFrame representation of the EvalInput
+        """
+        data = []
+        for item in eval_input.eval_input_items:
+            row = item.full_dataset_entry.copy() if item.full_dataset_entry else {}
+
+            # Ensure key fields are present
+            row[self.id_key] = item.id
+            if self.is_structured_input():
+                row[self.question_key] = item.input_obj
+                row[self.answer_key] = item.expected_output_obj
+                row[self.generated_answer_key] = item.output_obj
+                row[self.trajectory_key] = item.trajectory
+                row[self.expected_trajectory_key] = item.expected_trajectory
+
+            data.append(row)
+
+        return pd.DataFrame(data)
 
     def filter_intermediate_steps(self,
                                   intermediate_steps: list[IntermediateStep],
-                                  event_filter: list[IntermediateStepType] = None) -> list[dict]:
+                                  event_filter: list[IntermediateStepType] | None = None) -> list[dict]:
         """
         Filter out the intermediate steps that are not relevant for evaluation.
         The output is written with with the intention of re-running the evaluation using the original config file.
@@ -220,7 +331,9 @@ class DatasetHandler:
         filtered_steps = self.intermediate_step_adapter.filter_intermediate_steps(intermediate_steps, event_filter)
         return self.intermediate_step_adapter.serialize_intermediate_steps(filtered_steps)
 
-    def publish_eval_input(self, eval_input, workflow_output_step_filter: list[IntermediateStepType] = None) -> str:
+    def publish_eval_input(self,
+                           eval_input,
+                           workflow_output_step_filter: list[IntermediateStepType] | None = None) -> str:
         """
         Convert the EvalInput object to a JSON output for storing in a file. Use the orginal keys to
         allow re-running evaluation using the orignal config file and '--skip_workflow' option.

{aiq → nat}/eval/evaluate.py

@@ -23,21 +23,21 @@ from uuid import uuid4
 from pydantic import BaseModel
 from tqdm import tqdm
 
-from aiq.data_models.evaluate import EvalConfig
-from aiq.data_models.evaluate import JobEvictionPolicy
-from aiq.eval.config import EvaluationRunConfig
-from aiq.eval.config import EvaluationRunOutput
-from aiq.eval.dataset_handler.dataset_handler import DatasetHandler
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-from aiq.eval.usage_stats import UsageStats
-from aiq.eval.usage_stats import UsageStatsItem
-from aiq.eval.usage_stats import UsageStatsLLM
-from aiq.eval.utils.output_uploader import OutputUploader
-from aiq.eval.utils.weave_eval import WeaveEvaluationIntegration
-from aiq.profiler.data_models import ProfilerResults
-from aiq.runtime.session import AIQSessionManager
+from nat.data_models.evaluate import EvalConfig
+from nat.data_models.evaluate import JobEvictionPolicy
+from nat.eval.config import EvaluationRunConfig
+from nat.eval.config import EvaluationRunOutput
+from nat.eval.dataset_handler.dataset_handler import DatasetHandler
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
+from nat.eval.evaluator.evaluator_model import EvalOutput
+from nat.eval.usage_stats import UsageStats
+from nat.eval.usage_stats import UsageStatsItem
+from nat.eval.usage_stats import UsageStatsLLM
+from nat.eval.utils.output_uploader import OutputUploader
+from nat.eval.utils.weave_eval import WeaveEvaluationIntegration
+from nat.profiler.data_models import ProfilerResults
+from nat.runtime.session import SessionManager
 
 logger = logging.getLogger(__name__)
 
@@ -45,13 +45,17 @@ logger = logging.getLogger(__name__)
 class EvaluationRun:  # pylint: disable=too-many-public-methods
     """
     Instantiated for each evaluation run and used to store data for that single run.
+
+    .. warning::
+        **Experimental Feature**: The Evaluation API is experimental and may change in future releases.
+        Future versions may introduce breaking changes without notice.
     """
 
     def __init__(self, config: EvaluationRunConfig):
         """
         Initialize an EvaluationRun with configuration.
         """
-        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
+        from nat.eval.intermediate_step_adapter import IntermediateStepAdapter
 
         # Run-specific configuration
         self.config: EvaluationRunConfig = config
@@ -79,7 +83,7 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
     def _compute_usage_stats(self, item: EvalInputItem):
         """Compute usage stats for a single item using the intermediate steps"""
         # get the prompt and completion tokens from the intermediate steps
-        from aiq.profiler.intermediate_property_adapter import IntermediatePropertyAdaptor
+        from nat.profiler.intermediate_property_adapter import IntermediatePropertyAdaptor
         steps = [IntermediatePropertyAdaptor.from_intermediate_step(step) for step in item.trajectory]
         usage_stats_per_llm = {}
         total_tokens = 0
@@ -129,14 +133,14 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
                                                                      llm_latency=llm_latency)
         return self.usage_stats.usage_stats_items[item.id]
 
-    async def run_workflow_local(self, session_manager: AIQSessionManager):
+    async def run_workflow_local(self, session_manager: SessionManager):
         '''
         Launch the workflow with the specified questions and extract the output using the jsonpath
         '''
         # import function level dependencies
         from jsonpath_ng import parse
 
-        from aiq.eval.runtime_event_subscriber import pull_intermediate
+        from nat.eval.runtime_event_subscriber import pull_intermediate
 
         # Run the workflow
         jsonpath_expr = parse(self.config.result_json_path)
@@ -220,7 +224,7 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
         pbar.close()
 
     async def run_workflow_remote(self):
-        from aiq.eval.remote_workflow import EvaluationRemoteWorkflowHandler
+        from nat.eval.remote_workflow import EvaluationRemoteWorkflowHandler
         handler = EvaluationRemoteWorkflowHandler(self.config, self.eval_config.general.max_concurrency)
         await handler.run_workflow_remote(self.eval_input)
         for item in self.eval_input.eval_input_items:
@@ -237,7 +241,7 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
             logger.info("Profiler is not enabled. Skipping profiling.")
             return ProfilerResults()
 
-        from aiq.profiler.profile_runner import ProfilerRunner
+        from nat.profiler.profile_runner import ProfilerRunner
 
         all_stats = []
         for input_item in self.eval_input.eval_input_items:
@@ -306,7 +310,7 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
             except Exception as e:
                 logger.exception("Failed to delete old job directory: %s: %s", dir_to_delete, e, exc_info=True)
 
-    def write_output(self, dataset_handler: DatasetHandler, profiler_results: ProfilerResults):
+    def write_output(self, dataset_handler: DatasetHandler, profiler_results: ProfilerResults):  # pylint: disable=unused-argument  # noqa: E501
         workflow_output_file = self.eval_config.general.output_dir / "workflow_output.json"
         workflow_output_file.parent.mkdir(parents=True, exist_ok=True)
 
@@ -374,17 +378,17 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
             await self.weave_eval.afinish_loggers()
 
     def apply_overrides(self):
-        from aiq.cli.cli_utils.config_override import load_and_override_config
-        from aiq.data_models.config import AIQConfig
-        from aiq.runtime.loader import PluginTypes
-        from aiq.runtime.loader import discover_and_register_plugins
-        from aiq.utils.data_models.schema_validator import validate_schema
+        from nat.cli.cli_utils.config_override import load_and_override_config
+        from nat.data_models.config import Config
+        from nat.runtime.loader import PluginTypes
+        from nat.runtime.loader import discover_and_register_plugins
+        from nat.utils.data_models.schema_validator import validate_schema
 
         # Register plugins before validation
         discover_and_register_plugins(PluginTypes.CONFIG_OBJECT)
 
         config_dict = load_and_override_config(self.config.config_file, self.config.override)
-        config = validate_schema(config_dict, AIQConfig)
+        config = validate_schema(config_dict, Config)
         return config
 
     def _get_workflow_alias(self, workflow_type: str | None = None):
@@ -393,20 +397,20 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
             return self.eval_config.general.workflow_alias
 
         if not workflow_type or workflow_type == "EmptyFunctionConfig":
-            return "aiqtoolkit-eval"
+            return "nat-eval"
 
         return workflow_type
 
     async def run_and_evaluate(self,
-                               session_manager: AIQSessionManager | None = None,
+                               session_manager: SessionManager | None = None,
                                job_id: str | None = None) -> EvaluationRunOutput:
         """
         Run the workflow with the specified config file and evaluate the dataset
         """
         logger.info("Starting evaluation run with config file: %s", self.config.config_file)
 
-        from aiq.builder.eval_builder import WorkflowEvalBuilder
-        from aiq.runtime.loader import load_config
+        from nat.builder.eval_builder import WorkflowEvalBuilder
+        from nat.runtime.loader import load_config
 
         # Load and override the config
         if self.config.override:
@@ -469,8 +473,8 @@ class EvaluationRun:  # pylint: disable=too-many-public-methods
             else:
                 if not self.config.skip_workflow:
                     if session_manager is None:
-                        session_manager = AIQSessionManager(eval_workflow.build(),
-                                                            max_concurrency=self.eval_config.general.max_concurrency)
+                        session_manager = SessionManager(eval_workflow.build(),
+                                                         max_concurrency=self.eval_config.general.max_concurrency)
                     await self.run_workflow_local(session_manager)
 
             # Evaluate

{aiq → nat}/eval/evaluator/base_evaluator.py

@@ -19,17 +19,21 @@ from abc import abstractmethod
 
 from tqdm import tqdm
 
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
+from nat.eval.evaluator.evaluator_model import EvalOutput
+from nat.eval.evaluator.evaluator_model import EvalOutputItem
+from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry
 
 
 class BaseEvaluator(ABC):
     """
     Base class for custom evaluators.
 
+    .. warning::
+        **Experimental Feature**: The Evaluation API is experimental and may change in future releases.
+        Future versions may introduce breaking changes without notice.
+
     Each custom evaluator must implement the `evaluate_item` method which is used to evaluate a
     single EvalInputItem.
     """

{aiq → nat}/eval/evaluator/evaluator_model.py

@@ -17,16 +17,16 @@ import typing
 
 from pydantic import BaseModel
 
-from aiq.data_models.intermediate_step import IntermediateStep
+from nat.data_models.intermediate_step import IntermediateStep
 
 
 class EvalInputItem(BaseModel):
     id: typing.Any
     input_obj: typing.Any
     expected_output_obj: typing.Any
-    output_obj: typing.Any
-    expected_trajectory: list[IntermediateStep]
-    trajectory: list[IntermediateStep]
+    output_obj: typing.Any = None  # populated by the workflow
+    expected_trajectory: list[IntermediateStep] = []
+    trajectory: list[IntermediateStep] = []  # populated by the workflow
     full_dataset_entry: typing.Any
 
 

{aiq → nat}/eval/intermediate_step_adapter.py

@@ -17,8 +17,8 @@ import logging
 
 from langchain_core.agents import AgentAction
 
-from aiq.data_models.intermediate_step import IntermediateStep
-from aiq.data_models.intermediate_step import IntermediateStepType
+from nat.data_models.intermediate_step import IntermediateStep
+from nat.data_models.intermediate_step import IntermediateStepType
 
 logger = logging.getLogger(__name__)
 

{aiq → nat}/eval/rag_evaluator/evaluate.py

@@ -25,12 +25,12 @@ from ragas.llms import LangchainLLMWrapper
 from ragas.metrics import Metric
 from tqdm import tqdm
 
-from aiq.data_models.intermediate_step import IntermediateStepType
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutput
-from aiq.eval.evaluator.evaluator_model import EvalOutputItem
-from aiq.eval.utils.tqdm_position_registry import TqdmPositionRegistry
+from nat.data_models.intermediate_step import IntermediateStepType
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
+from nat.eval.evaluator.evaluator_model import EvalOutput
+from nat.eval.evaluator.evaluator_model import EvalOutputItem
+from nat.eval.utils.tqdm_position_registry import TqdmPositionRegistry
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +68,7 @@ class RAGEvaluator:
 
     def eval_input_to_ragas(self, eval_input: EvalInput) -> EvaluationDataset:
         """Converts EvalInput into a Ragas-compatible EvaluationDataset."""
-        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
+        from nat.eval.intermediate_step_adapter import IntermediateStepAdapter
         event_filter = [IntermediateStepType.TOOL_END, IntermediateStepType.LLM_END, IntermediateStepType.CUSTOM_END]
         samples = []
 
@@ -99,7 +99,7 @@ class RAGEvaluator:
         return EvaluationDataset(samples=samples)
 
     def ragas_to_eval_output(self, eval_input: EvalInput, results_dataset: EvaluationResult | None) -> EvalOutput:
-        """Converts the ragas EvaluationResult to aiq EvalOutput"""
+        """Converts the ragas EvaluationResult to nat EvalOutput"""
 
         if not results_dataset:
             logger.error("Ragas evaluation failed with no results")

{aiq → nat}/eval/rag_evaluator/register.py

@@ -19,13 +19,13 @@ from pydantic import BaseModel
 from pydantic import Field
 from pydantic import model_validator
 
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.builder.framework_enum import LLMFrameworkEnum
-from aiq.cli.register_workflow import register_evaluator
-from aiq.data_models.evaluator import EvaluatorBaseConfig
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalOutput
+from nat.builder.builder import EvalBuilder
+from nat.builder.evaluator import EvaluatorInfo
+from nat.builder.framework_enum import LLMFrameworkEnum
+from nat.cli.register_workflow import register_evaluator
+from nat.data_models.evaluator import EvaluatorBaseConfig
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalOutput
 
 logger = logging.getLogger(__name__)
 

{aiq → nat}/eval/remote_workflow.py

@@ -21,13 +21,13 @@ import aiohttp
 from pydantic import ValidationError
 from tqdm import tqdm
 
-from aiq.data_models.api_server import AIQResponseIntermediateStep
-from aiq.data_models.intermediate_step import IntermediateStep
-from aiq.data_models.intermediate_step import IntermediateStepPayload
-from aiq.data_models.invocation_node import InvocationNode
-from aiq.eval.config import EvaluationRunConfig
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
+from nat.data_models.api_server import ResponseIntermediateStep
+from nat.data_models.intermediate_step import IntermediateStep
+from nat.data_models.intermediate_step import IntermediateStepPayload
+from nat.data_models.invocation_node import InvocationNode
+from nat.eval.config import EvaluationRunConfig
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalInputItem
 
 logger = logging.getLogger(__name__)
 
@@ -80,7 +80,7 @@ class EvaluationRemoteWorkflowHandler:
                         # This is an intermediate step
                         try:
                             step_data = json.loads(line[len(INTERMEDIATE_DATA_PREFIX):])
-                            response_intermediate = AIQResponseIntermediateStep.model_validate(step_data)
+                            response_intermediate = ResponseIntermediateStep.model_validate(step_data)
                             # The payload is expected to be IntermediateStepPayload
                             payload = IntermediateStepPayload.model_validate_json(response_intermediate.payload)
                             intermediate_step = IntermediateStep(parent_id="remote",

{aiq → nat}/eval/runners/config.py

@@ -17,8 +17,8 @@ import typing
 
 from pydantic import BaseModel
 
-from aiq.eval.config import EvaluationRunConfig
-from aiq.eval.config import EvaluationRunOutput
+from nat.eval.config import EvaluationRunConfig
+from nat.eval.config import EvaluationRunOutput
 
 
 class MultiEvaluationRunConfig(BaseModel):

{aiq → nat}/eval/runners/multi_eval_runner.py

@@ -16,10 +16,10 @@
 import copy
 import typing
 
-from aiq.eval.config import EvaluationRunConfig
-from aiq.eval.config import EvaluationRunOutput
-from aiq.eval.evaluate import EvaluationRun
-from aiq.eval.runners.config import MultiEvaluationRunConfig
+from nat.eval.config import EvaluationRunConfig
+from nat.eval.config import EvaluationRunOutput
+from nat.eval.evaluate import EvaluationRun
+from nat.eval.runners.config import MultiEvaluationRunConfig
 
 
 class MultiEvaluationRunner:

{aiq → nat}/eval/runtime_event_subscriber.py

@@ -16,8 +16,8 @@
 import asyncio
 import logging
 
-from aiq.builder.context import AIQContext
-from aiq.data_models.intermediate_step import IntermediateStep
+from nat.builder.context import Context
+from nat.data_models.intermediate_step import IntermediateStep
 
 logger = logging.getLogger(__name__)
 
@@ -30,7 +30,7 @@ def pull_intermediate() -> asyncio.Future[list[dict]]:
     """
     future = asyncio.Future()
     intermediate_steps = []  # We'll store the dumped steps here.
-    context = AIQContext.get()
+    context = Context.get()
 
     def on_next_cb(item: IntermediateStep):
         # Append each new intermediate step (dumped to dict) to the list.

{aiq → nat}/eval/swe_bench_evaluator/evaluate.py

@@ -19,10 +19,10 @@ import os
 import shutil
 from pathlib import Path
 
-from aiq.data_models.swe_bench_model import SWEBenchInput
-from aiq.data_models.swe_bench_model import SWEBenchOutput
-from aiq.eval.evaluator.evaluator_model import EvalInput
-from aiq.eval.evaluator.evaluator_model import EvalOutput
+from nat.data_models.swe_bench_model import SWEBenchInput
+from nat.data_models.swe_bench_model import SWEBenchOutput
+from nat.eval.evaluator.evaluator_model import EvalInput
+from nat.eval.evaluator.evaluator_model import EvalOutput
 
 try:
     import swebench.harness.run_evaluation as swebench_eval
@@ -123,7 +123,7 @@ class SweBenchEvaluator:
                             for s in swebench_inputs if s not in supported_inputs})
 
         # Write SWEBenchInput to file
-        workflow_input_file = self.output_dir / "aiq_workflow_input.json"
+        workflow_input_file = self.output_dir / "nat_workflow_input.json"
         workflow_input_file.parent.mkdir(parents=True, exist_ok=True)
         Path(workflow_input_file).write_text(json.dumps([swebench.model_dump() for swebench in supported_inputs],
                                                         indent=2),
@@ -139,7 +139,7 @@ class SweBenchEvaluator:
             return None, None
 
         # Write SWEBenchOutput to file
-        workflow_output_file = self.output_dir / "aiq_workflow_output.json"
+        workflow_output_file = self.output_dir / "nat_workflow_output.json"
         Path(workflow_output_file).write_text(json.dumps([output.model_dump() for output in filtered_outputs],
                                                          indent=2),
                                               encoding="utf-8")

{aiq → nat}/eval/swe_bench_evaluator/register.py

@@ -15,10 +15,10 @@
 
 from pydantic import Field
 
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.cli.register_workflow import register_evaluator
-from aiq.data_models.evaluator import EvaluatorBaseConfig
+from nat.builder.builder import EvalBuilder
+from nat.builder.evaluator import EvaluatorInfo
+from nat.cli.register_workflow import register_evaluator
+from nat.data_models.evaluator import EvaluatorBaseConfig
 
 
 class SweBenchEvaluatorConfig(EvaluatorBaseConfig, name="swe_bench"):

{aiq → nat}/eval/trajectory_evaluator/evaluate.py

@@ -19,9 +19,9 @@ from langchain.evaluation import TrajectoryEvalChain
 from langchain_core.language_models import BaseChatModel
 from langchain_core.tools import BaseTool
 
-from aiq.eval.evaluator.base_evaluator import BaseEvaluator
-from aiq.eval.evaluator.evaluator_model import EvalInputItem
-from aiq.eval.evaluator.evaluator_model import EvalOutputItem
+from nat.eval.evaluator.base_evaluator import BaseEvaluator
+from nat.eval.evaluator.evaluator_model import EvalInputItem
+from nat.eval.evaluator.evaluator_model import EvalOutputItem
 
 logger = logging.getLogger(__name__)
 
@@ -48,8 +48,8 @@ class TrajectoryEvaluator(BaseEvaluator):
         """
         Evaluate a single EvalInputItem and return an EvalOutputItem.
         """
-        from aiq.data_models.intermediate_step import IntermediateStepType
-        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
+        from nat.data_models.intermediate_step import IntermediateStepType
+        from nat.eval.intermediate_step_adapter import IntermediateStepAdapter
 
         intermediate_step_adapter = IntermediateStepAdapter()
         event_filter = [IntermediateStepType.LLM_END, IntermediateStepType.TOOL_END]

{aiq → nat}/eval/trajectory_evaluator/register.py

@@ -15,10 +15,10 @@
 
 from pydantic import Field
 
-from aiq.builder.builder import EvalBuilder
-from aiq.builder.evaluator import EvaluatorInfo
-from aiq.cli.register_workflow import register_evaluator
-from aiq.data_models.evaluator import EvaluatorBaseConfig
+from nat.builder.builder import EvalBuilder
+from nat.builder.evaluator import EvaluatorInfo
+from nat.cli.register_workflow import register_evaluator
+from nat.data_models.evaluator import EvaluatorBaseConfig
 
 
 class TrajectoryEvaluatorConfig(EvaluatorBaseConfig, name="trajectory"):
@@ -29,7 +29,7 @@ class TrajectoryEvaluatorConfig(EvaluatorBaseConfig, name="trajectory"):
 
 @register_evaluator(config_type=TrajectoryEvaluatorConfig)
 async def register_trajectory_evaluator(config: TrajectoryEvaluatorConfig, builder: EvalBuilder):
-    from aiq.builder.framework_enum import LLMFrameworkEnum
+    from nat.builder.framework_enum import LLMFrameworkEnum
 
     from .evaluate import TrajectoryEvaluator
     llm = await builder.get_llm(config.llm_name, wrapper_type=LLMFrameworkEnum.LANGCHAIN)