nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/types/evaluation/rubric_score.py

@@ -0,0 +1,54 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import List, Union, Optional
+from typing_extensions import TypeAlias
+
+from .rubric import Rubric
+from ..._models import BaseModel
+from .json_score_parser import JsonScoreParser
+from .regex_score_parser import RegexScoreParser
+
+__all__ = ["RubricScore", "Parser"]
+
+Parser: TypeAlias = Union[JsonScoreParser, RegexScoreParser]
+
+
+class RubricScore(BaseModel):
+    """Score definition for a rubric with optional parser.
+
+    If no parser is set, JSONScoreParser is the default parser inferred from the score parameters
+    """
+
+    name: str
+    """The name of the score.
+
+    Only lowercase letters, numbers, and underscores allowed.
+    """
+
+    rubric: List[Rubric]
+    """The rubric for the score."""
+
+    description: Optional[str] = None
+    """Human-readable description of the score."""
+
+    parser: Optional[Parser] = None
+    """The method to parse the score.
+
+    When used with llm-judge metric, and no parser is set, JSONScoreParser is the
+    default parser inferred from the score parameters.
+    """

nemo_platform/types/evaluation/rubric_score_param.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Union, Iterable
+from typing_extensions import Required, TypeAlias, TypedDict
+
+from .rubric_param import RubricParam
+from .json_score_parser_param import JsonScoreParserParam
+from .regex_score_parser_param import RegexScoreParserParam
+
+__all__ = ["RubricScoreParam", "Parser"]
+
+Parser: TypeAlias = Union[JsonScoreParserParam, RegexScoreParserParam]
+
+
+class RubricScoreParam(TypedDict, total=False):
+    """Score definition for a rubric with optional parser.
+
+    If no parser is set, JSONScoreParser is the default parser inferred from the score parameters
+    """
+
+    name: Required[str]
+    """The name of the score.
+
+    Only lowercase letters, numbers, and underscores allowed.
+    """
+
+    rubric: Required[Iterable[RubricParam]]
+    """The rubric for the score."""
+
+    description: str
+    """Human-readable description of the score."""
+
+    parser: Parser
+    """The method to parse the score.
+
+    When used with llm-judge metric, and no parser is set, JSONScoreParser is the
+    default parser inferred from the score parameters.
+    """

nemo_platform/types/evaluation/rubric_score_stat.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+
+__all__ = ["RubricScoreStat"]
+
+
+class RubricScoreStat(BaseModel):
+    """Rubric score with count statistics."""
+
+    label: str
+    """The label to use for the level of the rubric grading criteria."""
+
+    value: float
+    """The score value to assign for the criteria."""
+
+    count: Optional[int] = None
+    """The number of samples evaluated with the rubric level."""
+
+    description: Optional[str] = None
+    """Describe the semantic meaning of each criteria for the given rubric."""

nemo_platform/types/evaluation/run_config.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+
+__all__ = ["RunConfig"]
+
+
+class RunConfig(BaseModel):
+    """Job parameters."""
+
+    limit_samples: Optional[int] = None
+    """
+    Limit number of evaluation samples, taking the first `limit` samples from the
+    dataset.
+    """
+
+    parallelism: Optional[int] = None
+    """Parallelism to be used for the evaluation job.
+
+    Typically, this represents the maximum number of concurrent requests made to the
+    model.
+    """

nemo_platform/types/evaluation/run_config_online.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+
+__all__ = ["RunConfigOnline"]
+
+
+class RunConfigOnline(BaseModel):
+    """Job parameters for online evaluation."""
+
+    ignore_request_failure: Optional[bool] = None
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    limit_samples: Optional[int] = None
+    """
+    Limit number of evaluation samples, taking the first `limit` samples from the
+    dataset.
+    """
+
+    max_retries: Optional[int] = None
+    """Maximum number of retries for failed requests."""
+
+    parallelism: Optional[int] = None
+    """Parallelism to be used for the evaluation job.
+
+    Typically, this represents the maximum number of concurrent requests made to the
+    model.
+    """
+
+    request_timeout: Optional[int] = None
+    """The timeout to be used for requests made to the model."""

nemo_platform/types/evaluation/run_config_online_model.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, Optional
+
+from ..._models import BaseModel
+from .inference_params import InferenceParams
+from .reasoning_params import ReasoningParams
+
+__all__ = ["RunConfigOnlineModel"]
+
+
+class RunConfigOnlineModel(BaseModel):
+    """Job parameters for model online evaluation."""
+
+    ignore_request_failure: Optional[bool] = None
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    limit_samples: Optional[int] = None
+    """
+    Limit number of evaluation samples, taking the first `limit` samples from the
+    dataset.
+    """
+
+    max_retries: Optional[int] = None
+    """Maximum number of retries for failed requests."""
+
+    parallelism: Optional[int] = None
+    """Parallelism to be used for the evaluation job.
+
+    Typically, this represents the maximum number of concurrent requests made to the
+    model.
+    """
+
+    reasoning: Optional[ReasoningParams] = None
+    """Custom settings that control the model's reasoning behavior."""
+
+    request_timeout: Optional[int] = None
+    """The timeout to be used for requests made to the model."""
+
+    structured_output: Optional[Dict[str, object]] = None
+    """JSON schema to apply structured output for the model."""
+
+    system_prompt: Optional[str] = None
+    """
+    Initial instructions that define the model's role and behavior for the
+    conversation.
+    """

nemo_platform/types/evaluation/run_config_online_model_param.py

@@ -0,0 +1,75 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict
+from typing_extensions import TypedDict
+
+from .inference_params_param import InferenceParamsParam
+from .reasoning_params_param import ReasoningParamsParam
+
+__all__ = ["RunConfigOnlineModelParam"]
+
+
+class RunConfigOnlineModelParam(TypedDict, total=False):
+    """Job parameters for model online evaluation."""
+
+    ignore_request_failure: bool
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    inference: InferenceParamsParam
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    limit_samples: int
+    """
+    Limit number of evaluation samples, taking the first `limit` samples from the
+    dataset.
+    """
+
+    max_retries: int
+    """Maximum number of retries for failed requests."""
+
+    parallelism: int
+    """Parallelism to be used for the evaluation job.
+
+    Typically, this represents the maximum number of concurrent requests made to the
+    model.
+    """
+
+    reasoning: ReasoningParamsParam
+    """Custom settings that control the model's reasoning behavior."""
+
+    request_timeout: int
+    """The timeout to be used for requests made to the model."""
+
+    structured_output: Dict[str, object]
+    """JSON schema to apply structured output for the model."""
+
+    system_prompt: str
+    """
+    Initial instructions that define the model's role and behavior for the
+    conversation.
+    """

nemo_platform/types/evaluation/run_config_online_param.py

@@ -0,0 +1,51 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["RunConfigOnlineParam"]
+
+
+class RunConfigOnlineParam(TypedDict, total=False):
+    """Job parameters for online evaluation."""
+
+    ignore_request_failure: bool
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    limit_samples: int
+    """
+    Limit number of evaluation samples, taking the first `limit` samples from the
+    dataset.
+    """
+
+    max_retries: int
+    """Maximum number of retries for failed requests."""
+
+    parallelism: int
+    """Parallelism to be used for the evaluation job.
+
+    Typically, this represents the maximum number of concurrent requests made to the
+    model.
+    """
+
+    request_timeout: int
+    """The timeout to be used for requests made to the model."""

nemo_platform/types/evaluation/run_config_param.py

@@ -0,0 +1,39 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import TypedDict
+
+__all__ = ["RunConfigParam"]
+
+
+class RunConfigParam(TypedDict, total=False):
+    """Job parameters."""
+
+    limit_samples: int
+    """
+    Limit number of evaluation samples, taking the first `limit` samples from the
+    dataset.
+    """
+
+    parallelism: int
+    """Parallelism to be used for the evaluation job.
+
+    Typically, this represents the maximum number of concurrent requests made to the
+    model.
+    """

nemo_platform/types/evaluation/string_check_metric.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from ..._models import BaseModel
+
+__all__ = ["StringCheckMetric"]
+
+
+class StringCheckMetric(BaseModel):
+    """Persisted string check metric."""
+
+    id: str
+
+    created_at: datetime
+
+    created_by: Optional[str] = None
+
+    entity_id: str
+    """Alias for id for backwards compatibility."""
+
+    left_template: str
+    """
+    The template to use for rendering the left value of the operator to compute the
+    metric.
+    """
+
+    operation: Literal["equals", "==", "!=", "<>", "not equals", "contains", "not contains", "startswith", "endswith"]
+    """The operation to compute for the metric."""
+
+    parent: str
+    """Parent entity ID for nested entities."""
+
+    right_template: str
+    """
+    The template to use for rendering the right value of the operator to compute the
+    metric.
+    """
+
+    updated_at: datetime
+
+    updated_by: Optional[str] = None
+
+    workspace: str
+    """Workspace identifier"""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["string-check"]] = None

nemo_platform/types/evaluation/string_check_metric_param.py

@@ -0,0 +1,59 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from typing_extensions import Literal
+
+from ..._models import BaseModel
+
+__all__ = ["StringCheckMetricParam"]
+
+
+class StringCheckMetricParam(BaseModel):
+    """Request type for StringCheckMetric.
+
+    String-comparison metric with operator-based checks.
+    """
+
+    left_template: str
+    """
+    The template to use for rendering the left value of the operator to compute the
+    metric.
+    """
+
+    operation: Literal["equals", "==", "!=", "<>", "not equals", "contains", "not contains", "startswith", "endswith"]
+    """The operation to compute for the metric."""
+
+    right_template: str
+    """
+    The template to use for rendering the right value of the operator to compute the
+    metric.
+    """
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["string-check"]] = None

nemo_platform/types/evaluation/string_check_metric_param_param.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, List
+from typing_extensions import Literal, Required, TypedDict
+
+__all__ = ["StringCheckMetricParamParam"]
+
+
+class StringCheckMetricParamParam(TypedDict, total=False):
+    """Request type for StringCheckMetric.
+
+    String-comparison metric with operator-based checks.
+    """
+
+    left_template: Required[str]
+    """
+    The template to use for rendering the left value of the operator to compute the
+    metric.
+    """
+
+    operation: Required[
+        Literal["equals", "==", "!=", "<>", "not equals", "contains", "not contains", "startswith", "endswith"]
+    ]
+    """The operation to compute for the metric."""
+
+    right_template: Required[str]
+    """
+    The template to use for rendering the right value of the operator to compute the
+    metric.
+    """
+
+    description: str
+    """Human-readable description of the metric."""
+
+    labels: Dict[str, str]
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    supported_job_types: List[Literal["online", "offline"]]
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Literal["string-check"]