nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/types/evaluation/remote_metric_response.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from ..._models import BaseModel
+from .remote_score import RemoteScore
+from ..files.secret_ref import SecretRef
+
+__all__ = ["RemoteMetricResponse"]
+
+
+class RemoteMetricResponse(BaseModel):
+    """Response type for RemoteMetric."""
+
+    body: Dict[str, object]
+    """Jinja template for request payload"""
+
+    scores: List[RemoteScore]
+    """List of scores to extract from the remote response"""
+
+    url: str
+    """The URL of the remote endpoint."""
+
+    id: Optional[str] = None
+    """Entity name within the workspace"""
+
+    api_key_secret: Optional[SecretRef] = None
+    """Reference to a secret.
+
+    Format: 'secret_name' (uses request workspace) or 'workspace/secret_name'
+    (explicit workspace).
+    """
+
+    created_at: Optional[datetime] = None
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    max_retries: Optional[int] = None
+    """Maximum number of retry attempts."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    parent: Optional[str] = None
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    timeout_seconds: Optional[float] = None
+    """Request timeout in seconds."""
+
+    type: Optional[Literal["remote"]] = None
+
+    updated_at: Optional[datetime] = None
+
+    workspace: Optional[str] = None
+    """Workspace identifier"""

nemo_platform/types/evaluation/remote_score.py

@@ -0,0 +1,49 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Optional
+
+from ..._models import BaseModel
+from .json_score_parser import JsonScoreParser
+
+__all__ = ["RemoteScore"]
+
+
+class RemoteScore(BaseModel):
+    """Score configuration for remote metrics.
+
+    Unlike RangeScore, minimum and maximum are optional (default to None = no bounds).
+    This avoids JSON serialization issues with infinity values.
+    """
+
+    name: str
+    """The name of the score.
+
+    Only lowercase letters, numbers, and underscores allowed.
+    """
+
+    description: Optional[str] = None
+    """Human-readable description of the score."""
+
+    maximum: Optional[float] = None
+    """Maximum value for the score range. Defaults to None (no upper bound)."""
+
+    minimum: Optional[float] = None
+    """Minimum value for the score range. Defaults to None (no lower bound)."""
+
+    parser: Optional[JsonScoreParser] = None
+    """Parse a score from JSON structured content."""

nemo_platform/types/evaluation/remote_score_param.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+from .json_score_parser_param import JsonScoreParserParam
+
+__all__ = ["RemoteScoreParam"]
+
+
+class RemoteScoreParam(TypedDict, total=False):
+    """Score configuration for remote metrics.
+
+    Unlike RangeScore, minimum and maximum are optional (default to None = no bounds).
+    This avoids JSON serialization issues with infinity values.
+    """
+
+    name: Required[str]
+    """The name of the score.
+
+    Only lowercase letters, numbers, and underscores allowed.
+    """
+
+    description: str
+    """Human-readable description of the score."""
+
+    maximum: float
+    """Maximum value for the score range. Defaults to None (no upper bound)."""
+
+    minimum: float
+    """Minimum value for the score range. Defaults to None (no lower bound)."""
+
+    parser: JsonScoreParserParam
+    """Parse a score from JSON structured content."""

nemo_platform/types/evaluation/response_groundedness_metric.py

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from .model import Model
+from ..._models import BaseModel
+from .inference_params import InferenceParams
+
+__all__ = ["ResponseGroundednessMetric"]
+
+
+class ResponseGroundednessMetric(BaseModel):
+    """RAGAS metric for measuring response groundedness."""
+
+    id: str
+
+    created_at: datetime
+
+    created_by: Optional[str] = None
+
+    entity_id: str
+    """Alias for id for backwards compatibility."""
+
+    judge_model: Model
+    """Model definition for use without persisting to the Models API."""
+
+    parent: str
+    """Parent entity ID for nested entities."""
+
+    updated_at: datetime
+
+    updated_by: Optional[str] = None
+
+    workspace: str
+    """Workspace identifier"""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Optional[Dict[str, object]] = None
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["response_groundedness"]] = None

nemo_platform/types/evaluation/response_groundedness_metric_param.py

@@ -0,0 +1,66 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from typing_extensions import Literal, TypeAlias
+
+from .model import Model
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .inference_params import InferenceParams
+
+__all__ = ["ResponseGroundednessMetricParam", "JudgeModel"]
+
+JudgeModel: TypeAlias = Union[Model, ModelRef]
+
+
+class ResponseGroundednessMetricParam(BaseModel):
+    """Request type for ResponseGroundedness metrics."""
+
+    judge_model: JudgeModel
+    """The judge model configuration."""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Optional[Dict[str, object]] = None
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["response_groundedness"]] = None

nemo_platform/types/evaluation/response_groundedness_metric_param_param.py

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, List, Union
+from typing_extensions import Literal, Required, TypeAlias, TypedDict
+
+from .model_ref import ModelRef
+from .model_param import ModelParam
+from .inference_params_param import InferenceParamsParam
+
+__all__ = ["ResponseGroundednessMetricParamParam", "JudgeModel"]
+
+JudgeModel: TypeAlias = Union[ModelParam, ModelRef]
+
+
+class ResponseGroundednessMetricParamParam(TypedDict, total=False):
+    """Request type for ResponseGroundedness metrics."""
+
+    judge_model: Required[JudgeModel]
+    """The judge model configuration."""
+
+    description: str
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: bool
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: InferenceParamsParam
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Dict[str, object]
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Dict[str, str]
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    supported_job_types: List[Literal["online", "offline"]]
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Literal["response_groundedness"]

nemo_platform/types/evaluation/response_groundedness_metric_response.py

@@ -0,0 +1,85 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from datetime import datetime
+from typing_extensions import Literal, TypeAlias
+
+from .model import Model
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .inference_params import InferenceParams
+
+__all__ = ["ResponseGroundednessMetricResponse", "JudgeModel"]
+
+JudgeModel: TypeAlias = Union[Model, ModelRef]
+
+
+class ResponseGroundednessMetricResponse(BaseModel):
+    """Response type for ResponseGroundedness metrics."""
+
+    judge_model: JudgeModel
+    """The judge model configuration."""
+
+    id: Optional[str] = None
+    """Entity name within the workspace"""
+
+    created_at: Optional[datetime] = None
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Optional[Dict[str, object]] = None
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    parent: Optional[str] = None
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["response_groundedness"]] = None
+
+    updated_at: Optional[datetime] = None
+
+    workspace: Optional[str] = None
+    """Workspace identifier"""

nemo_platform/types/evaluation/response_relevancy_metric.py

@@ -0,0 +1,95 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Optional
+from datetime import datetime
+from typing_extensions import Literal
+
+from .model import Model
+from ..._models import BaseModel
+from .inference_params import InferenceParams
+
+__all__ = ["ResponseRelevancyMetric"]
+
+
+class ResponseRelevancyMetric(BaseModel):
+    """RAGAS metric for measuring response relevancy."""
+
+    id: str
+
+    created_at: datetime
+
+    created_by: Optional[str] = None
+
+    embeddings_model: Model
+    """Model definition for use without persisting to the Models API."""
+
+    entity_id: str
+    """Alias for id for backwards compatibility."""
+
+    judge_model: Model
+    """Model definition for use without persisting to the Models API."""
+
+    parent: str
+    """Parent entity ID for nested entities."""
+
+    updated_at: datetime
+
+    updated_by: Optional[str] = None
+
+    workspace: str
+    """Workspace identifier"""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Optional[Dict[str, object]] = None
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    strictness: Optional[int] = None
+    """Number of parallel questions generated. NIM can only generate 1."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["response_relevancy"]] = None

nemo_platform/types/evaluation/response_relevancy_metric_param.py

@@ -0,0 +1,74 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from typing_extensions import Literal, TypeAlias
+
+from .model import Model
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .inference_params import InferenceParams
+
+__all__ = ["ResponseRelevancyMetricParam", "EmbeddingsModel", "JudgeModel"]
+
+EmbeddingsModel: TypeAlias = Union[Model, ModelRef]
+
+JudgeModel: TypeAlias = Union[Model, ModelRef]
+
+
+class ResponseRelevancyMetricParam(BaseModel):
+    """Request type for ResponseRelevancy metrics."""
+
+    embeddings_model: EmbeddingsModel
+    """The embeddings model configuration."""
+
+    judge_model: JudgeModel
+    """The judge model configuration."""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """
+    If True, request failures to the judge model are ignored and the metric result
+    is marked as NaN. Parse/output formatting failures are always converted to NaN.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    input_template: Optional[Dict[str, object]] = None
+    """Optional Jinja template for rendering the input payload for RAGAS evaluation."""
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    strictness: Optional[int] = None
+    """Number of parallel questions generated. NIM can only generate 1."""
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    type: Optional[Literal["response_relevancy"]] = None