nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/types/evaluation/answer_accuracy_metric_response.py

@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from datetime import datetime
+from typing_extensions import Literal, TypeAlias
+
+from .model import Model
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .inference_params import InferenceParams
+
+__all__ = ["AnswerAccuracyMetricResponse", "JudgeModel"]
+
+JudgeModel: TypeAlias = Union[Model, ModelRef]
+
+
+class AnswerAccuracyMetricResponse(BaseModel):
+    """Response type for AnswerAccuracy metrics."""
+
+    judge_model: JudgeModel
+    """The judge model configuration."""
+
+    id: Optional[str] = None
+    """Entity name within the workspace"""
+
+    created_at: Optional[datetime] = None
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Optional[Dict[str, object]] = None
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    parent: Optional[str] = None
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["answer_accuracy"]] = None
+
+    updated_at: Optional[datetime] = None
+
+    workspace: Optional[str] = None
+    """Workspace identifier"""

nemo_platform/types/evaluation/benchmark.py

@@ -0,0 +1,81 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from datetime import datetime
+
+from ..._models import BaseModel
+from .metric_ref import MetricRef
+from .fileset_ref import FilesetRef
+from .field_mapping import FieldMapping
+
+__all__ = ["Benchmark"]
+
+
+class Benchmark(BaseModel):
+    """Benchmark response schema."""
+
+    id: str
+
+    created_at: datetime
+
+    created_by: Optional[str] = None
+
+    dataset: FilesetRef
+    """Reference to a Fileset in the Files API.
+
+    A reference is a string with format 'workspace/fileset-name' that points to a
+    persisted fileset entity. When used as a dataset source, all files within the
+    fileset will be downloaded to the job container.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    entity_id: str
+    """Alias for id for backwards compatibility."""
+
+    metrics: List[MetricRef]
+    """The metrics that comprise this benchmark (format: workspace/metric_name)."""
+
+    name: str
+    """Benchmark name"""
+
+    parent: str
+    """Parent entity ID for nested entities."""
+
+    updated_at: datetime
+
+    updated_by: Optional[str] = None
+
+    workspace: str
+    """Workspace identifier"""
+
+    description: Optional[str] = None
+    """Human-readable description of the benchmark."""
+
+    field_mapping: Optional[FieldMapping] = None
+    """
+    Maps canonical evaluator fields to raw dataset column paths. Example: {'input':
+    'question', 'output': 'answer', 'reference': 'gold', 'trajectory': 'steps'}
+    """
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""

nemo_platform/types/evaluation/benchmark_create_params.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict
+from typing_extensions import Required, TypedDict
+
+from ..._types import SequenceNotStr
+from .metric_ref import MetricRef
+from .fileset_ref import FilesetRef
+from .field_mapping_param import FieldMappingParam
+
+__all__ = ["BenchmarkCreateParams"]
+
+
+class BenchmarkCreateParams(TypedDict, total=False):
+    workspace: str
+
+    dataset: Required[FilesetRef]
+    """Reference to a Fileset in the Files API.
+
+    A reference is a string with format 'workspace/fileset-name' that points to a
+    persisted fileset entity. When used as a dataset source, all files within the
+    fileset will be downloaded to the job container.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    description: Required[str]
+    """The description of the benchmark."""
+
+    metrics: Required[SequenceNotStr[MetricRef]]
+    """The metrics that comprise this benchmark (format: workspace/metric_name)."""
+
+    name: Required[str]
+    """The name of the benchmark."""
+
+    extended_response: bool
+    """Whether to return the extended benchmark."""
+
+    field_mapping: FieldMappingParam
+    """
+    Maps canonical evaluator fields to raw dataset column paths. Example: {'input':
+    'question', 'output': 'answer', 'reference': 'gold', 'trajectory': 'steps'}
+    """
+
+    labels: Dict[str, str]
+    """Labels are key-value pairs that can be used for grouping and filtering."""

nemo_platform/types/evaluation/benchmark_create_response.py

@@ -0,0 +1,26 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Union
+from typing_extensions import TypeAlias
+
+from .benchmark import Benchmark
+from .extended_benchmark import ExtendedBenchmark
+
+__all__ = ["BenchmarkCreateResponse"]
+
+BenchmarkCreateResponse: TypeAlias = Union[Benchmark, ExtendedBenchmark]

nemo_platform/types/evaluation/benchmark_evaluation_job.py

@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, Union, Optional
+from datetime import datetime
+from typing_extensions import TypeAlias
+
+from ..._models import BaseModel
+from .benchmark_online_job import BenchmarkOnlineJob
+from .benchmark_offline_job import BenchmarkOfflineJob
+from .benchmark_online_agent_job import BenchmarkOnlineAgentJob
+from ..shared.platform_job_status import PlatformJobStatus
+from .system_benchmark_online_job import SystemBenchmarkOnlineJob
+from .system_benchmark_offline_job import SystemBenchmarkOfflineJob
+
+__all__ = ["BenchmarkEvaluationJob", "Spec"]
+
+Spec: TypeAlias = Union[
+    BenchmarkOfflineJob,
+    BenchmarkOnlineJob,
+    BenchmarkOnlineAgentJob,
+    SystemBenchmarkOfflineJob,
+    SystemBenchmarkOnlineJob,
+]
+
+
+class BenchmarkEvaluationJob(BaseModel):
+    name: str
+
+    spec: Spec
+    """Input for an offline benchmark evaluation job.
+
+    Evaluates the benchmark's dataset against all metrics in the benchmark.
+    """
+
+    id: Optional[str] = None
+
+    created_at: Optional[datetime] = None
+
+    custom_fields: Optional[Dict[str, object]] = None
+
+    description: Optional[str] = None
+
+    error_details: Optional[Dict[str, object]] = None
+
+    ownership: Optional[Dict[str, object]] = None
+
+    project: Optional[str] = None
+
+    status: Optional[PlatformJobStatus] = None
+    """Enumeration of possible job statuses.
+
+    This enum represents the various states a job can be in during its lifecycle,
+    from creation to a terminal state.
+    """
+
+    status_details: Optional[Dict[str, object]] = None
+
+    updated_at: Optional[datetime] = None
+
+    workspace: Optional[str] = None

nemo_platform/types/evaluation/benchmark_evaluation_jobs_list_filter_param.py

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+from ..shared.platform_job_status import PlatformJobStatus
+from ..shared_params.datetime_filter import DatetimeFilter
+
+__all__ = ["BenchmarkEvaluationJobsListFilterParam"]
+
+
+class BenchmarkEvaluationJobsListFilterParam(TypedDict, total=False):
+    created_at: DatetimeFilter
+    """Jobs created at 'gte' datetime or 'lte' datetime."""
+
+    name: str
+    """Name of the job."""
+
+    project: str
+    """Project containing the job."""
+
+    status: PlatformJobStatus
+    """Enumeration of possible job statuses.
+
+    This enum represents the various states a job can be in during its lifecycle,
+    from creation to a terminal state.
+    """
+
+    updated_at: DatetimeFilter
+    """Jobs updated at 'gte' datetime or 'lte' datetime."""
+
+    workspace: str
+    """Workspace of the job."""

nemo_platform/types/evaluation/benchmark_evaluation_jobs_page.py

@@ -0,0 +1,37 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+
+from ..._models import BaseModel
+from ..shared.pagination_data import PaginationData
+from .benchmark_evaluation_job import BenchmarkEvaluationJob
+
+__all__ = ["BenchmarkEvaluationJobsPage"]
+
+
+class BenchmarkEvaluationJobsPage(BaseModel):
+    data: List[BenchmarkEvaluationJob]
+
+    filter: Optional[Dict[str, object]] = None
+    """Filtering information."""
+
+    pagination: Optional[PaginationData] = None
+    """Pagination information."""
+
+    sort: Optional[str] = None
+    """The field on which the results are sorted."""

nemo_platform/types/evaluation/benchmark_evaluation_jobs_sort_field.py

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing_extensions import Literal, TypeAlias
+
+__all__ = ["BenchmarkEvaluationJobsSortField"]
+
+BenchmarkEvaluationJobsSortField: TypeAlias = Literal["created_at", "-created_at", "updated_at", "-updated_at"]

nemo_platform/types/evaluation/benchmark_job_create_params.py

@@ -0,0 +1,58 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, Union
+from typing_extensions import Required, TypeAlias, TypedDict
+
+from .benchmark_online_job_param import BenchmarkOnlineJobParam
+from .benchmark_offline_job_param import BenchmarkOfflineJobParam
+from .benchmark_online_agent_job_param import BenchmarkOnlineAgentJobParam
+from .system_benchmark_online_job_param import SystemBenchmarkOnlineJobParam
+from .system_benchmark_offline_job_param import SystemBenchmarkOfflineJobParam
+
+__all__ = ["BenchmarkJobCreateParams", "Spec"]
+
+
+class BenchmarkJobCreateParams(TypedDict, total=False):
+    workspace: str
+
+    spec: Required[Spec]
+    """Input for an offline benchmark evaluation job.
+
+    Evaluates the benchmark's dataset against all metrics in the benchmark.
+    """
+
+    custom_fields: Dict[str, object]
+
+    description: str
+
+    name: str
+
+    ownership: Dict[str, object]
+
+    project: str
+
+
+Spec: TypeAlias = Union[
+    BenchmarkOfflineJobParam,
+    BenchmarkOnlineJobParam,
+    BenchmarkOnlineAgentJobParam,
+    SystemBenchmarkOfflineJobParam,
+    SystemBenchmarkOnlineJobParam,
+]

nemo_platform/types/evaluation/benchmark_job_get_logs_params.py

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["BenchmarkJobGetLogsParams"]
+
+
+class BenchmarkJobGetLogsParams(TypedDict, total=False):
+    workspace: str
+
+    limit: int
+
+    page_cursor: str

nemo_platform/types/evaluation/benchmark_job_list_params.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+from .benchmark_evaluation_jobs_sort_field import BenchmarkEvaluationJobsSortField
+from .benchmark_evaluation_jobs_list_filter_param import BenchmarkEvaluationJobsListFilterParam
+
+__all__ = ["BenchmarkJobListParams"]
+
+
+class BenchmarkJobListParams(TypedDict, total=False):
+    workspace: str
+
+    filter: BenchmarkEvaluationJobsListFilterParam
+    """Filter jobs on various criteria."""
+
+    page: int
+    """Page number."""
+
+    page_size: int
+    """Page size."""
+
+    sort: BenchmarkEvaluationJobsSortField
+    """The field to sort by.
+
+    To sort in decreasing order, use `-` in front of the field name.
+    """

nemo_platform/types/evaluation/benchmark_job_result.py

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from datetime import datetime
+
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .metric_ref import MetricRef
+from .fileset_ref import FilesetRef
+from .benchmark_ref import BenchmarkRef
+from .benchmark_jobs.results.benchmark_metric_result import BenchmarkMetricResult
+
+__all__ = ["BenchmarkJobResult"]
+
+
+class BenchmarkJobResult(BaseModel):
+    """Response type for benchmark job result."""
+
+    id: str
+
+    benchmark: BenchmarkRef
+    """Reference to a benchmark in the Benchmarks API.
+
+    A reference is a string with format 'workspace/benchmark-name' that points to a
+    persisted benchmark entity. See
+    [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    created_at: datetime
+
+    created_by: Optional[str] = None
+
+    entity_id: str
+    """Alias for id for backwards compatibility."""
+
+    parent: str
+    """Parent entity ID for nested entities."""
+
+    results: List[BenchmarkMetricResult]
+    """Results for each metric in the benchmark."""
+
+    updated_at: datetime
+
+    updated_by: Optional[str] = None
+
+    workspace: str
+    """Workspace identifier"""
+
+    dataset: Optional[FilesetRef] = None
+    """Reference to a Fileset in the Files API.
+
+    A reference is a string with format 'workspace/fileset-name' that points to a
+    persisted fileset entity. When used as a dataset source, all files within the
+    fileset will be downloaded to the job container.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    metrics: Optional[List[MetricRef]] = None
+    """The list of metrics used for the evaluation job to generate the result."""
+
+    model: Optional[ModelRef] = None
+    """Reference to a Model in the Models API.
+
+    See [Entity references](docs/get-started/concepts/entity-references.md) for the
+    general entity reference pattern used across the platform.
+    """
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""