nvidia-nat 1.2.0rc5__py3-none-any.whl

aiq/embedder/nim_embedder.py

@@ -0,0 +1,59 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import typing
+
+from pydantic import AfterValidator
+from pydantic import AliasChoices
+from pydantic import ConfigDict
+from pydantic import Field
+
+from aiq.builder.builder import Builder
+from aiq.builder.embedder import EmbedderProviderInfo
+from aiq.cli.register_workflow import register_embedder_provider
+from aiq.data_models.embedder import EmbedderBaseConfig
+from aiq.data_models.retry_mixin import RetryMixin
+
+allowed_truncate_values = ["NONE", "START", "END"]
+
+
+def option_in_allowed_values(v):
+    """Ensures option is allowed"""
+    assert v in allowed_truncate_values
+    return v
+
+
+TruncationOption = typing.Annotated[str, AfterValidator(option_in_allowed_values)]
+
+
+class NIMEmbedderModelConfig(EmbedderBaseConfig, RetryMixin, name="nim"):
+    """A NVIDIA Inference Microservice (NIM) embedder provider to be used with an embedder client."""
+
+    api_key: str | None = Field(default=None, description="NVIDIA API key to interact with hosted NIM.")
+    base_url: str | None = Field(default=None, description="Base url to the hosted NIM.")
+    model_name: str = Field(validation_alias=AliasChoices("model_name", "model"),
+                            serialization_alias="model",
+                            description="The model name for the hosted NIM.")
+    truncate: TruncationOption = Field(default="NONE",
+                                       description=("The truncation strategy if the input on the "
+                                                    "server side if it's too large."))
+
+    model_config = ConfigDict(protected_namespaces=())
+
+
+@register_embedder_provider(config_type=NIMEmbedderModelConfig)
+async def nim_embedder_model(embedder_config: NIMEmbedderModelConfig, builder: Builder):
+
+    yield EmbedderProviderInfo(config=embedder_config, description="A NIM model for use with an Embedder client.")

aiq/embedder/openai_embedder.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pydantic import AliasChoices
+from pydantic import ConfigDict
+from pydantic import Field
+
+from aiq.builder.builder import Builder
+from aiq.builder.embedder import EmbedderProviderInfo
+from aiq.cli.register_workflow import register_embedder_provider
+from aiq.data_models.embedder import EmbedderBaseConfig
+from aiq.data_models.retry_mixin import RetryMixin
+
+
+class OpenAIEmbedderModelConfig(EmbedderBaseConfig, RetryMixin, name="openai"):
+    """An OpenAI LLM provider to be used with an LLM client."""
+
+    model_config = ConfigDict(protected_namespaces=())
+
+    api_key: str | None = Field(default=None, description="OpenAI API key to interact with hosted model.")
+    base_url: str | None = Field(default=None, description="Base url to the hosted model.")
+    model_name: str = Field(validation_alias=AliasChoices("model_name", "model"),
+                            serialization_alias="model",
+                            description="The OpenAI hosted model name.")
+    max_retries: int = Field(default=2, description="The max number of retries for the request.")
+
+
+@register_embedder_provider(config_type=OpenAIEmbedderModelConfig)
+async def openai_llm(config: OpenAIEmbedderModelConfig, builder: Builder):
+
+    yield EmbedderProviderInfo(config=config, description="An OpenAI model for use with an Embedder client.")

aiq/embedder/register.py

@@ -0,0 +1,24 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# pylint: disable=unused-import
+# flake8: noqa
+# isort:skip_file
+
+# Import any providers which need to be automatically registered here
+from . import nim_embedder
+from . import openai_embedder
+# Import any clients which need to be automatically registered here
+from . import langchain_client

aiq/eval/__init__.py

@@ -0,0 +1,14 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

aiq/eval/config.py

@@ -0,0 +1,60 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+from pydantic import BaseModel
+
+from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.evaluator_model import EvalOutput
+from aiq.eval.usage_stats import UsageStats
+from aiq.profiler.data_models import ProfilerResults
+
+
+class EvaluationRunConfig(BaseModel):
+    """
+    Parameters used for a single evaluation run.
+    """
+    config_file: Path
+    dataset: str | None = None  # dataset file path can be specified in the config file
+    result_json_path: str = "$"
+    skip_workflow: bool = False
+    skip_completed_entries: bool = False
+    endpoint: str | None = None  # only used when running the workflow remotely
+    endpoint_timeout: int = 300
+    reps: int = 1
+    override: tuple[tuple[str, str], ...] = ()
+    # If false, the output will not be written to the output directory. This is
+    # useful when running evaluation via another tool.
+    write_output: bool = True
+    # if true, the dataset is adjusted to a multiple of the concurrency
+    adjust_dataset_size: bool = False
+    # number of passes at each concurrency, if 0 the dataset is adjusted to a multiple of the
+    # concurrency. The is only used if adjust_dataset_size is true
+    num_passes: int = 0
+
+
+class EvaluationRunOutput(BaseModel):
+    """
+    Output of a single evaluation run.
+    """
+    workflow_output_file: Path | None
+    evaluator_output_files: list[Path]
+    workflow_interrupted: bool
+
+    eval_input: EvalInput
+    evaluation_results: list[tuple[str, EvalOutput]]
+    usage_stats: UsageStats | None = None
+    profiler_results: ProfilerResults

aiq/eval/dataset_handler/dataset_downloader.py

@@ -0,0 +1,106 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+from pathlib import Path
+
+import boto3
+import requests
+from botocore.exceptions import NoCredentialsError
+
+from aiq.data_models.dataset_handler import EvalDatasetConfig
+
+logger = logging.getLogger(__name__)
+
+
+class DatasetDownloader:
+    """
+    Download remote datasets using signed URLs or S3 credentials.
+
+    One DatasetDownloader object is needed for each dataset to be downloaded.
+    """
+
+    def __init__(self, dataset_config: EvalDatasetConfig):
+        self.dataset_config = dataset_config
+        self._s3_client = None
+
+    @property
+    def s3_config(self):
+        return self.dataset_config.s3
+
+    @property
+    def s3_client(self):
+        """Lazy init the S3 client."""
+        if not self._s3_client:
+            try:
+                self._s3_client = boto3.client("s3",
+                                               endpoint_url=self.s3_config.endpoint_url,
+                                               aws_access_key_id=self.s3_config.access_key,
+                                               aws_secret_access_key=self.s3_config.secret_key)
+            except NoCredentialsError as e:
+                logger.error("AWS credentials not available: %s", e)
+                raise
+            except Exception as e:
+                logger.error("Failed to initialize S3 client: %s", e)
+                raise
+        return self._s3_client
+
+    @staticmethod
+    def ensure_directory_exists(file_path: str):
+        """Ensure the directory for the file exists."""
+        Path(file_path).parent.mkdir(parents=True, exist_ok=True)
+
+    def download_with_signed_url(self, remote_file_path: str, local_file_path: str, timeout: int = 300):
+        """Download a file using a signed URL."""
+        try:
+            response = requests.get(remote_file_path, stream=True, timeout=timeout)
+            response.raise_for_status()
+            with open(local_file_path, "wb") as file:
+                for chunk in response.iter_content(chunk_size=8192):
+                    file.write(chunk)
+            logger.info("File downloaded successfully to %s using signed URL.", local_file_path)
+        except requests.exceptions.RequestException as e:
+            logger.error("Error downloading file using signed URL: %s", e)
+            raise
+
+    def download_with_boto3(self, remote_file_path: str, local_file_path: str):
+        """Download a file using boto3 and credentials."""
+        try:
+            self.s3_client.download_file(self.dataset_config.s3.bucket, remote_file_path, local_file_path)
+            logger.info("File downloaded successfully to %s using S3 client.", local_file_path)
+        except Exception as e:
+            logger.error("Error downloading file from S3: %s", e)
+            raise
+
+    @staticmethod
+    def is_file_path_url(file_path: str) -> bool:
+        """Check if the file path is a URL."""
+        return file_path.startswith("http")
+
+    def download_file(self, remote_file_path: str, local_file_path: str):
+        """Download a file using the appropriate method."""
+        self.ensure_directory_exists(local_file_path)
+        if self.is_file_path_url(remote_file_path):
+            logger.info("Using signed URL to download the file %s...", remote_file_path)
+            self.download_with_signed_url(remote_file_path, local_file_path, timeout=120)
+        else:
+            logger.info("Using S3 credentials to download the file %s...", remote_file_path)
+            self.download_with_boto3(remote_file_path, local_file_path)
+
+    def download_dataset(self):
+        """Download datasets defined in the evaluation configuration."""
+        if self.dataset_config.remote_file_path:
+            logger.info("Downloading remote dataset %s")
+            self.download_file(remote_file_path=self.dataset_config.remote_file_path,
+                               local_file_path=self.dataset_config.file_path)

aiq/eval/dataset_handler/dataset_filter.py

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pandas as pd
+
+from aiq.data_models.dataset_handler import EvalFilterConfig
+
+
+class DatasetFilter:
+    """
+    Apply allowlist and denylist filters to the DataFrame based on specified column filters.
+        - If a allowlist is provided, only keep rows matching the filter values.
+        - If a denylist is provided, remove rows matching the filter values.
+        - If the filter column does not exist in the DataFrame, the filtering is skipped for that column.
+
+    This is a utility class that is dataset agnostic and can be used to filter any DataFrame based on the provided
+    filter configuration.
+    """
+
+    def __init__(self, filter_config: EvalFilterConfig):
+
+        self.filter_config = filter_config
+
+    def apply_filters(self, df) -> pd.DataFrame:
+
+        filtered_df = df.copy()
+
+        # Apply allowlist (only keep specified rows)
+        if self.filter_config.allowlist:
+            for column, values in self.filter_config.allowlist.field.items():
+                if column in filtered_df.columns:
+                    filtered_df = filtered_df[filtered_df[column].isin(values)]
+
+        # Apply denylist (remove specified rows)
+        if self.filter_config.denylist:
+            for column, values in self.filter_config.denylist.field.items():
+                if column in filtered_df.columns:
+                    filtered_df = filtered_df[~filtered_df[column].isin(values)]
+
+        return filtered_df

aiq/eval/dataset_handler/dataset_handler.py

@@ -0,0 +1,254 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import math
+
+import pandas as pd
+
+from aiq.data_models.dataset_handler import EvalDatasetConfig
+from aiq.data_models.dataset_handler import EvalDatasetJsonConfig
+from aiq.data_models.intermediate_step import IntermediateStep
+from aiq.data_models.intermediate_step import IntermediateStepType
+from aiq.eval.dataset_handler.dataset_downloader import DatasetDownloader
+from aiq.eval.dataset_handler.dataset_filter import DatasetFilter
+from aiq.eval.evaluator.evaluator_model import EvalInput
+from aiq.eval.evaluator.evaluator_model import EvalInputItem
+
+
+class DatasetHandler:
+    """
+    Read the datasets and pre-process (apply filters, deduplicate etc.) before turning them into EvalInput objects.
+    One DatasetHandler object is needed for each dataset to be evaluated.
+    """
+
+    def __init__(self,
+                 dataset_config: EvalDatasetConfig,
+                 reps: int,
+                 concurrency: int,
+                 num_passes: int | None = None,
+                 adjust_dataset_size: bool = False):
+        from aiq.eval.intermediate_step_adapter import IntermediateStepAdapter
+
+        self.dataset_config = dataset_config
+        self.dataset_filter = DatasetFilter(dataset_config.filter)
+        self.reps = reps
+
+        # number of passes at specific concurrency
+        self.concurrency = concurrency
+        self.num_passes = num_passes
+        self.adjust_dataset_size = adjust_dataset_size
+
+        # Helpers
+        self.intermediate_step_adapter = IntermediateStepAdapter()
+
+    def is_structured_input(self) -> bool:
+        '''Check if the input is structured or unstructured'''
+        return not self.dataset_config.structure.disable
+
+    @property
+    def id_key(self) -> str:
+        return self.dataset_config.id_key
+
+    @property
+    def question_key(self) -> str:
+        return self.dataset_config.structure.question_key
+
+    @property
+    def answer_key(self) -> str:
+        return self.dataset_config.structure.answer_key
+
+    @property
+    def generated_answer_key(self) -> str:
+        return self.dataset_config.structure.generated_answer_key
+
+    @property
+    def trajectory_key(self) -> str:
+        return self.dataset_config.structure.trajectory_key
+
+    @property
+    def expected_trajectory_key(self) -> str:
+        return self.dataset_config.structure.expected_trajectory_key
+
+    def get_eval_input_from_df(self, input_df: pd.DataFrame) -> EvalInput:
+
+        def create_eval_item(row: pd.Series, structured: bool) -> EvalInputItem:
+            """Helper function to create EvalInputItem."""
+            return EvalInputItem(
+                id=row.get(self.id_key, ""),
+                input_obj=row.to_json() if not structured else row.get(self.question_key, ""),
+                expected_output_obj=row.get(self.answer_key, "") if structured else "",
+                output_obj=row.get(self.generated_answer_key, "") if structured else "",
+                trajectory=row.get(self.trajectory_key, []) if structured else [],
+                expected_trajectory=row.get(self.expected_trajectory_key, []) if structured else [],
+                full_dataset_entry=row.to_dict(),
+            )
+
+        # if input dataframe is empty return an empty list
+        if input_df.empty:
+            return EvalInput(eval_input_items=[])
+
+        structured = self.is_structured_input()
+        if structured:
+            # For structured input, question is mandatory. Ignore rows with missing or empty questions
+            input_df = input_df[input_df[self.question_key].notnull() & input_df[self.question_key].str.strip().ne("")]
+        eval_input_items = [create_eval_item(row, structured) for _, row in input_df.iterrows()]
+
+        return EvalInput(eval_input_items=eval_input_items)
+
+    def setup_reps(self, input_df: pd.DataFrame) -> pd.DataFrame:
+        """replicate the rows and update the id to id_key + "_rep" + rep_number"""
+        # Replicate the rows
+        input_df = pd.concat([input_df] * self.reps, ignore_index=True)
+        # Compute repetition index
+        rep_index = input_df.groupby(self.dataset_config.id_key).cumcount().astype(str)
+        # Convert id_key to string (id can be integer) if needed and update IDs
+        input_df[self.dataset_config.id_key] = input_df[self.dataset_config.id_key].astype(str) + "_rep" + rep_index
+        # Ensure unique ID values after modification
+        input_df.drop_duplicates(subset=[self.dataset_config.id_key], inplace=True)
+
+        return input_df
+
+    def adjust_dataset(self, input_df: pd.DataFrame) -> pd.DataFrame:
+        """
+        Adjust the dataset so its length is a multiple of concurrency.
+
+        If num_passes > 0:
+            dataset size is adjusted to concurrency * num_passes
+        else:
+            dataset size is adjusted to the largest multiple of concurrency
+            that is less than or equal to the current dataset size
+        """
+        if self.concurrency <= 0:
+            raise ValueError("Concurrency must be > 0")
+
+        if self.num_passes < 0:
+            raise ValueError("num_passes must be >= 0")
+
+        original_size = input_df.shape[0]
+
+        # Calculate target size
+        if self.num_passes > 0:
+            # When num_passes is specified, always use concurrency * num_passes
+            # This respects the user's intent for exact number of passes
+            target_size = self.concurrency * self.num_passes
+        else:
+            # When num_passes = 0, use the largest multiple of concurrency <= original_size
+            # If original_size < concurrency, we need at least concurrency rows
+            if original_size >= self.concurrency:
+                target_size = (original_size // self.concurrency) * self.concurrency
+            else:
+                target_size = self.concurrency
+
+        if target_size == 0:
+            raise ValueError("Input dataset too small for even one batch at given concurrency.")
+
+        id_col = self.dataset_config.id_key
+
+        # If we need more rows than we have, replicate the dataset
+        if original_size < target_size:
+            # Clean existing _rep suffix if present
+            input_df[id_col] = input_df[id_col].astype(str).str.replace(r"_rep\d+$", "", regex=True)
+
+            # Calculate how many complete copies we need
+            copies_needed = math.ceil(target_size / original_size)
+
+            # Create the replicated dataframe
+            replicated_dfs = []
+            for i in range(copies_needed):
+                df_copy = input_df.copy()
+                if i > 0:  # Add suffix to all but the first copy
+                    df_copy[id_col] = df_copy[id_col].astype(str) + f"_rep{i}"
+                replicated_dfs.append(df_copy)
+
+            input_df = pd.concat(replicated_dfs, ignore_index=True)
+
+        # Return exactly the target size
+        return input_df.head(target_size)
+
+    def get_eval_input_from_dataset(self, dataset: str) -> EvalInput:
+        # read the dataset and convert it to EvalInput
+
+        # if a dataset file has been provided in the command line, use that
+        dataset_config = EvalDatasetJsonConfig(file_path=dataset) if dataset else self.dataset_config
+
+        # Download the dataset if it is remote
+        downloader = DatasetDownloader(dataset_config=dataset_config)
+        downloader.download_dataset()
+
+        parser, kwargs = dataset_config.parser()
+        # Parse the dataset into a DataFrame
+        input_df = parser(dataset_config.file_path, **kwargs)
+
+        # Apply filters and deduplicate
+        input_df = self.dataset_filter.apply_filters(input_df)
+        input_df.drop_duplicates(subset=[self.dataset_config.id_key], inplace=True)
+
+        if self.reps > 1 and self.adjust_dataset_size:
+            raise ValueError("reps and adjust_dataset_size are mutually exclusive")
+
+        # If more than one repetition is needed, replicate the rows
+        if self.reps > 1:
+            input_df = self.setup_reps(input_df)
+        elif self.adjust_dataset_size:
+            input_df = self.adjust_dataset(input_df)
+
+        # Convert the DataFrame to a list of EvalInput objects
+        return self.get_eval_input_from_df(input_df)
+
+    def filter_intermediate_steps(self,
+                                  intermediate_steps: list[IntermediateStep],
+                                  event_filter: list[IntermediateStepType] = None) -> list[dict]:
+        """
+        Filter out the intermediate steps that are not relevant for evaluation.
+        The output is written with with the intention of re-running the evaluation using the original config file.
+        """
+        if event_filter is None:
+            event_filter = self.intermediate_step_adapter.DEFAULT_EVENT_FILTER
+        filtered_steps = self.intermediate_step_adapter.filter_intermediate_steps(intermediate_steps, event_filter)
+        return self.intermediate_step_adapter.serialize_intermediate_steps(filtered_steps)
+
+    def publish_eval_input(self, eval_input, workflow_output_step_filter: list[IntermediateStepType] = None) -> str:
+        """
+        Convert the EvalInput object to a JSON output for storing in a file. Use the orginal keys to
+        allow re-running evaluation using the orignal config file and '--skip_workflow' option.
+        """
+
+        def parse_if_json_string(value):
+            if isinstance(value, str):
+                try:
+                    return json.loads(value)
+                except json.JSONDecodeError:
+                    return value
+            if hasattr(value, "model_dump"):
+                return value.model_dump()
+            return value
+
+        indent = 2
+        if self.is_structured_input():
+            # Extract structured data from EvalInputItems
+            data = [{
+                self.id_key: item.id,
+                self.question_key: item.input_obj,
+                self.answer_key: item.expected_output_obj,
+                self.generated_answer_key: item.output_obj,
+                self.trajectory_key: self.filter_intermediate_steps(item.trajectory, workflow_output_step_filter),
+                self.expected_trajectory_key: self.filter_intermediate_steps(item.expected_trajectory),
+            } for item in eval_input.eval_input_items]
+        else:
+            # Unstructured case: return only raw output objects as a JSON array
+            data = [parse_if_json_string(item.output_obj) for item in eval_input.eval_input_items]
+
+        return json.dumps(data, indent=indent, ensure_ascii=False, default=str)