nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/types/evaluation/benchmark_job_result_list_params.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import List
+from typing_extensions import Literal, TypedDict
+
+from .model_ref import ModelRef
+from .fileset_ref import FilesetRef
+from .benchmark_ref import BenchmarkRef
+from ..shared_params.datetime_filter import DatetimeFilter
+
+__all__ = ["BenchmarkJobResultListParams", "Filter"]
+
+
+class BenchmarkJobResultListParams(TypedDict, total=False):
+    workspace: str
+
+    aggregate_fields: List[
+        Literal[
+            "nan_count",
+            "sum",
+            "mean",
+            "min",
+            "max",
+            "std_dev",
+            "variance",
+            "score_type",
+            "percentiles",
+            "histogram",
+            "rubric_distribution",
+            "mode_category",
+        ]
+    ]
+    """Aggregate score fields to include in the response (comma-separated or repeated).
+
+    Default: ('nan_count', 'sum', 'mean', 'min', 'max'). Available: ('nan_count',
+    'sum', 'mean', 'min', 'max', 'std_dev', 'variance', 'score_type', 'percentiles',
+    'histogram', 'rubric_distribution', 'mode_category').
+    """
+
+    filter: Filter
+    """
+    Filter benchmark job results by name, benchmark, metrics, dataset, model, and
+    dates. Supports JSON filter syntax with operators: $eq, $like, $lt, $lte, $gt,
+    $gte, $in, $nin, $and, $or, $not. Also supports text filter syntax.
+    """
+
+    page: int
+    """Page number."""
+
+    page_size: int
+    """Page size."""
+
+    sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"]
+    """The field to sort by.
+
+    To sort in decreasing order, use `-` in front of the field name.
+    """
+
+
+class Filter(TypedDict, total=False):
+    """
+    Filter benchmark job results by name, benchmark, metrics, dataset, model, and dates. Supports JSON filter syntax with operators: $eq, $like, $lt, $lte, $gt, $gte, $in, $nin, $and, $or, $not. Also supports text filter syntax.
+    """
+
+    benchmark: BenchmarkRef
+    """Reference to a benchmark in the Benchmarks API.
+
+    A reference is a string with format 'workspace/benchmark-name' that points to a
+    persisted benchmark entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    created_at: DatetimeFilter
+    """Filter job results by creation date range."""
+
+    dataset: FilesetRef
+    """Reference to a Fileset in the Files API.
+
+    A reference is a string with format 'workspace/fileset-name' that points to a
+    persisted fileset entity. When used as a dataset source, all files within the
+    fileset will be downloaded to the job container.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    metrics: str
+    """Filter results by metric reference."""
+
+    model: ModelRef
+    """Reference to a Model in the Models API.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    name: str
+    """Filter job results by name."""

nemo_platform/types/evaluation/benchmark_job_result_retrieve_params.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import List
+from typing_extensions import Literal, TypedDict
+
+__all__ = ["BenchmarkJobResultRetrieveParams"]
+
+
+class BenchmarkJobResultRetrieveParams(TypedDict, total=False):
+    workspace: str
+
+    aggregate_fields: List[
+        Literal[
+            "nan_count",
+            "sum",
+            "mean",
+            "min",
+            "max",
+            "std_dev",
+            "variance",
+            "score_type",
+            "percentiles",
+            "histogram",
+            "rubric_distribution",
+            "mode_category",
+        ]
+    ]
+    """Aggregate score fields to include in the response (comma-separated or repeated).
+
+    Default: ('nan_count', 'sum', 'mean', 'min', 'max'). Available: ('nan_count',
+    'sum', 'mean', 'min', 'max', 'std_dev', 'variance', 'score_type', 'percentiles',
+    'histogram', 'rubric_distribution', 'mode_category').
+    """

nemo_platform/types/evaluation/benchmark_job_results_list_response.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+
+from ..._models import BaseModel
+from .benchmark_job_result import BenchmarkJobResult
+from ..shared.pagination_data import PaginationData
+
+__all__ = ["BenchmarkJobResultsListResponse"]
+
+
+class BenchmarkJobResultsListResponse(BaseModel):
+    data: List[BenchmarkJobResult]
+
+    filter: Optional[Dict[str, object]] = None
+    """Filtering information."""
+
+    pagination: Optional[PaginationData] = None
+    """Pagination information."""
+
+    sort: Optional[str] = None
+    """The field on which the results are sorted."""

nemo_platform/types/evaluation/benchmark_jobs/__init__.py

@@ -0,0 +1,18 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations

nemo_platform/types/evaluation/benchmark_jobs/results/__init__.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from .benchmark_metric_result import BenchmarkMetricResult as BenchmarkMetricResult
+from .row_score_download_params import RowScoreDownloadParams as RowScoreDownloadParams
+from .benchmark_evaluation_result import BenchmarkEvaluationResult as BenchmarkEvaluationResult

nemo_platform/types/evaluation/benchmark_jobs/results/benchmark_evaluation_result.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List
+
+from ....._models import BaseModel
+from .benchmark_metric_result import BenchmarkMetricResult
+
+__all__ = ["BenchmarkEvaluationResult"]
+
+
+class BenchmarkEvaluationResult(BaseModel):
+    """Aggregated results for a benchmark evaluation."""
+
+    results: List[BenchmarkMetricResult]
+    """Results for each metric in the benchmark."""

nemo_platform/types/evaluation/benchmark_jobs/results/benchmark_metric_result.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List, Union, Optional
+from typing_extensions import TypeAlias
+
+from ....._models import BaseModel
+from ...metric_ref import MetricRef
+from ...aggregate_range_score import AggregateRangeScore
+from ...aggregate_rubric_score import AggregateRubricScore
+
+__all__ = ["BenchmarkMetricResult", "Score"]
+
+Score: TypeAlias = Union[AggregateRangeScore, AggregateRubricScore]
+
+
+class BenchmarkMetricResult(BaseModel):
+    """Aggregated results for a single metric within a benchmark."""
+
+    scores: List[Score]
+    """The list of aggregated scores."""
+
+    metric: Optional[MetricRef] = None
+    """Reference to a metric in the Metrics API.
+
+    A reference is a string with format 'workspace/metric-name' that points to a
+    persisted metric entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """

nemo_platform/types/evaluation/benchmark_jobs/results/row_score_download_params.py

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["RowScoreDownloadParams"]
+
+
+class RowScoreDownloadParams(TypedDict, total=False):
+    workspace: str
+
+    limit: int

nemo_platform/types/evaluation/benchmark_list_params.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Literal, TypedDict
+
+from .fileset_ref import FilesetRef
+from ..shared_params.datetime_filter import DatetimeFilter
+
+__all__ = ["BenchmarkListParams", "Filter"]
+
+
+class BenchmarkListParams(TypedDict, total=False):
+    workspace: str
+
+    extended_response: bool
+    """Whether to return the extended benchmark."""
+
+    filter: Filter
+    """Filter benchmarks by name, description, dataset, project, and dates.
+
+    Supports JSON filter syntax with operators: $eq, $like, $lt, $lte, $gt, $gte,
+    $in, $nin, $and, $or, $not. Also supports text filter syntax.
+    """
+
+    page: int
+    """Page number."""
+
+    page_size: int
+    """Page size."""
+
+    sort: Literal["-created_at", "created_at", "-updated_at", "updated_at", "-name", "name"]
+    """The field to sort by.
+
+    To sort in decreasing order, use `-` in front of the field name.
+    """
+
+
+class Filter(TypedDict, total=False):
+    """Filter benchmarks by name, description, dataset, project, and dates.
+
+    Supports JSON filter syntax with operators: $eq, $like, $lt, $lte, $gt, $gte, $in, $nin, $and, $or, $not. Also supports text filter syntax.
+    """
+
+    created_at: DatetimeFilter
+    """Filter benchmarks by creation date range."""
+
+    dataset: FilesetRef
+    """Reference to a Fileset in the Files API.
+
+    A reference is a string with format 'workspace/fileset-name' that points to a
+    persisted fileset entity. When used as a dataset source, all files within the
+    fileset will be downloaded to the job container.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    description: str
+    """Filter benchmarks by description."""
+
+    name: str
+    """Filter benchmarks by name."""
+
+    project: str
+    """Filter benchmarks by project name."""
+
+    updated_at: DatetimeFilter
+    """Filter benchmarks by last update date range."""

nemo_platform/types/evaluation/benchmark_offline_job.py

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+from .run_config import RunConfig
+from .benchmark_ref import BenchmarkRef
+
+__all__ = ["BenchmarkOfflineJob"]
+
+
+class BenchmarkOfflineJob(BaseModel):
+    """Input for an offline benchmark evaluation job.
+
+    Evaluates the benchmark's dataset against all metrics in the benchmark.
+    """
+
+    benchmark: BenchmarkRef
+    """Reference to a benchmark in the Benchmarks API.
+
+    A reference is a string with format 'workspace/benchmark-name' that points to a
+    persisted benchmark entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    params: Optional[RunConfig] = None
+    """Job parameters."""

nemo_platform/types/evaluation/benchmark_offline_job_param.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+from .benchmark_ref import BenchmarkRef
+from .run_config_param import RunConfigParam
+
+__all__ = ["BenchmarkOfflineJobParam"]
+
+
+class BenchmarkOfflineJobParam(TypedDict, total=False):
+    """Input for an offline benchmark evaluation job.
+
+    Evaluates the benchmark's dataset against all metrics in the benchmark.
+    """
+
+    benchmark: Required[BenchmarkRef]
+    """Reference to a benchmark in the Benchmarks API.
+
+    A reference is a string with format 'workspace/benchmark-name' that points to a
+    persisted benchmark entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    params: RunConfigParam
+    """Job parameters."""

nemo_platform/types/evaluation/benchmark_online_agent_job.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+
+from .agent import Agent
+from ..._models import BaseModel
+from .benchmark_ref import BenchmarkRef
+from .run_config_online import RunConfigOnline
+
+__all__ = ["BenchmarkOnlineAgentJob"]
+
+
+class BenchmarkOnlineAgentJob(BaseModel):
+    """Input for an online benchmark evaluation job targeting an agent.
+
+    Evaluates an agent by prompting it with the benchmark's dataset and then evaluating
+    the responses against all metrics in the benchmark.
+    """
+
+    agent: Agent
+    """Agent definition for inference in online evaluation jobs.
+
+    An agent is an endpoint that accepts a request and returns a response,
+    potentially with a trajectory. Two formats are supported:
+
+    - `generic`: configurable HTTP POST with Jinja-templated body and JSONPath
+      extraction for response and trajectory.
+    - `nemo_agent_toolkit`: NeMo Agent Toolkit SSE streaming protocol
+      (`/generate/full?filter_steps=none`).
+    """
+
+    benchmark: BenchmarkRef
+    """Reference to a benchmark in the Benchmarks API.
+
+    A reference is a string with format 'workspace/benchmark-name' that points to a
+    persisted benchmark entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    prompt_template: Union[str, Dict[str, object]]
+    """The jinja template to prompt the agent for evaluation.
+
+    Can be either a simple string or a structured object (e.g., OpenAI messages
+    format). Use Jinja template variables like {{input}}, {{output}}, {{context}},
+    {{reference}} to reference input columns.
+    """
+
+    optional_fields: Optional[List[str]] = None
+    """
+    Prompt template fields that should remain available to the prompt template but
+    not be required by dataset schema validation.
+    """
+
+    params: Optional[RunConfigOnline] = None
+    """Job parameters for online evaluation."""

nemo_platform/types/evaluation/benchmark_online_agent_job_param.py

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, Union
+from typing_extensions import Required, TypedDict
+
+from ..._types import SequenceNotStr
+from .agent_param import AgentParam
+from .benchmark_ref import BenchmarkRef
+from .run_config_online_param import RunConfigOnlineParam
+
+__all__ = ["BenchmarkOnlineAgentJobParam"]
+
+
+class BenchmarkOnlineAgentJobParam(TypedDict, total=False):
+    """Input for an online benchmark evaluation job targeting an agent.
+
+    Evaluates an agent by prompting it with the benchmark's dataset and then evaluating
+    the responses against all metrics in the benchmark.
+    """
+
+    agent: Required[AgentParam]
+    """Agent definition for inference in online evaluation jobs.
+
+    An agent is an endpoint that accepts a request and returns a response,
+    potentially with a trajectory. Two formats are supported:
+
+    - `generic`: configurable HTTP POST with Jinja-templated body and JSONPath
+      extraction for response and trajectory.
+    - `nemo_agent_toolkit`: NeMo Agent Toolkit SSE streaming protocol
+      (`/generate/full?filter_steps=none`).
+    """
+
+    benchmark: Required[BenchmarkRef]
+    """Reference to a benchmark in the Benchmarks API.
+
+    A reference is a string with format 'workspace/benchmark-name' that points to a
+    persisted benchmark entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    prompt_template: Required[Union[str, Dict[str, object]]]
+    """The jinja template to prompt the agent for evaluation.
+
+    Can be either a simple string or a structured object (e.g., OpenAI messages
+    format). Use Jinja template variables like {{input}}, {{output}}, {{context}},
+    {{reference}} to reference input columns.
+    """
+
+    optional_fields: SequenceNotStr[str]
+    """
+    Prompt template fields that should remain available to the prompt template but
+    not be required by dataset schema validation.
+    """
+
+    params: RunConfigOnlineParam
+    """Job parameters for online evaluation."""