nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/types/evaluation/json_score_parser_param.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing_extensions import Literal, Required, TypedDict
+
+__all__ = ["JsonScoreParserParam"]
+
+
+class JsonScoreParserParam(TypedDict, total=False):
+    """Parse a score from JSON structured content."""
+
+    json_path: Required[str]
+    """
+    The JSON path to parse the score from the judge response when using structured
+    output.
+    """
+
+    type: Literal["json"]

nemo_platform/types/evaluation/llm_judge_metric.py

@@ -0,0 +1,126 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from datetime import datetime
+from typing_extensions import Literal, TypeAlias
+
+from .model import Model
+from ..._models import BaseModel
+from .range_score import RangeScore
+from .rubric_score import RubricScore
+from .inference_params import InferenceParams
+from .reasoning_params import ReasoningParams
+
+__all__ = ["LLMJudgeMetric", "Score"]
+
+Score: TypeAlias = Union[RubricScore, RangeScore]
+
+
+class LLMJudgeMetric(BaseModel):
+    """Persisted LLM-as-a-Judge metric."""
+
+    id: str
+
+    created_at: datetime
+
+    created_by: Optional[str] = None
+
+    entity_id: str
+    """Alias for id for backwards compatibility."""
+
+    model: Model
+    """Model definition for use without persisting to the Models API."""
+
+    parent: str
+    """Parent entity ID for nested entities."""
+
+    scores: List[Score]
+    """Definitions of scores that will be extracted from the judge's output."""
+
+    updated_at: datetime
+
+    updated_by: Optional[str] = None
+
+    workspace: str
+    """Workspace identifier"""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    optional_fields: Optional[List[str]] = None
+    """
+    Prompt template fields that should remain in the inferred input schema but not
+    be required. Use this for fields like 'reference' when the metric can still run
+    without them.
+    """
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    prompt_template: Union[str, Dict[str, object], None] = None
+    """The prompt template for the judge.
+
+    Can be either a simple string or a structured object (e.g., OpenAI messages
+    format). Use Jinja template variables like {{sample.output_text}} to use the
+    model output within the template or {{item.xxx}} to reference input columns from
+    the dataset.
+    """
+
+    reasoning: Optional[ReasoningParams] = None
+    """Custom settings that control the model's reasoning behavior."""
+
+    structured_output: Optional[Dict[str, object]] = None
+    """JSON schema to apply structured output for the judge model evaluation.
+
+    Structured output is derived from scores when omitted. Use this option if there
+    are custom requirements for the output of the judge.
+    """
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    system_prompt: Optional[str] = None
+    """
+    Initial instructions that define the judge model's role and behavior for the
+    conversation. This is prepended to the messages as a system message.
+    """
+
+    type: Optional[Literal["llm-judge"]] = None

nemo_platform/types/evaluation/llm_judge_metric_param.py

@@ -0,0 +1,103 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from typing_extensions import Literal, TypeAlias
+
+from . import model
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .range_score import RangeScore
+from .rubric_score import RubricScore
+from .inference_params import InferenceParams
+from .reasoning_params import ReasoningParams
+
+__all__ = ["LLMJudgeMetricParam", "Model", "Score"]
+
+Model: TypeAlias = Union[model.Model, ModelRef]
+
+Score: TypeAlias = Union[RubricScore, RangeScore]
+
+
+class LLMJudgeMetricParam(BaseModel):
+    """Request type for creating LLM Judge metrics."""
+
+    model: Model
+    """The model configuration."""
+
+    scores: List[Score]
+    """Definitions of scores that will be extracted from the judge's output."""
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    optional_fields: Optional[List[str]] = None
+    """
+    Prompt template fields that should remain in the inferred input schema but not
+    be required. Use this for fields like 'reference' when the metric can still run
+    without them.
+    """
+
+    prompt_template: Union[str, Dict[str, object], None] = None
+    """The prompt template for the judge.
+
+    Can be either a simple string or a structured object (e.g., OpenAI messages
+    format). Use Jinja template variables like {{sample.output_text}} to use the
+    model output within the template or {{item.xxx}} to reference input columns from
+    the dataset.
+    """
+
+    reasoning: Optional[ReasoningParams] = None
+    """Custom settings that control the model's reasoning behavior."""
+
+    structured_output: Optional[Dict[str, object]] = None
+    """JSON schema to apply structured output for the judge model evaluation.
+
+    Structured output is derived from scores when omitted. Use this option if there
+    are custom requirements for the output of the judge.
+    """
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    system_prompt: Optional[str] = None
+    """
+    Initial instructions that define the judge model's role and behavior for the
+    conversation. This is prepended to the messages as a system message.
+    """
+
+    type: Optional[Literal["llm-judge"]] = None

nemo_platform/types/evaluation/llm_judge_metric_param_param.py

@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from __future__ import annotations
+
+from typing import Dict, List, Union, Iterable
+from typing_extensions import Literal, Required, TypeAlias, TypedDict
+
+from ..._types import SequenceNotStr
+from .model_ref import ModelRef
+from .model_param import ModelParam
+from .range_score_param import RangeScoreParam
+from .rubric_score_param import RubricScoreParam
+from .inference_params_param import InferenceParamsParam
+from .reasoning_params_param import ReasoningParamsParam
+
+__all__ = ["LLMJudgeMetricParamParam", "Model", "Score"]
+
+Model: TypeAlias = Union[ModelParam, ModelRef]
+
+Score: TypeAlias = Union[RubricScoreParam, RangeScoreParam]
+
+
+class LLMJudgeMetricParamParam(TypedDict, total=False):
+    """Request type for creating LLM Judge metrics."""
+
+    model: Required[Model]
+    """The model configuration."""
+
+    scores: Required[Iterable[Score]]
+    """Definitions of scores that will be extracted from the judge's output."""
+
+    description: str
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: bool
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    inference: InferenceParamsParam
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    labels: Dict[str, str]
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    optional_fields: SequenceNotStr[str]
+    """
+    Prompt template fields that should remain in the inferred input schema but not
+    be required. Use this for fields like 'reference' when the metric can still run
+    without them.
+    """
+
+    prompt_template: Union[str, Dict[str, object]]
+    """The prompt template for the judge.
+
+    Can be either a simple string or a structured object (e.g., OpenAI messages
+    format). Use Jinja template variables like {{sample.output_text}} to use the
+    model output within the template or {{item.xxx}} to reference input columns from
+    the dataset.
+    """
+
+    reasoning: ReasoningParamsParam
+    """Custom settings that control the model's reasoning behavior."""
+
+    structured_output: Dict[str, object]
+    """JSON schema to apply structured output for the judge model evaluation.
+
+    Structured output is derived from scores when omitted. Use this option if there
+    are custom requirements for the output of the judge.
+    """
+
+    supported_job_types: List[Literal["online", "offline"]]
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    system_prompt: str
+    """
+    Initial instructions that define the judge model's role and behavior for the
+    conversation. This is prepended to the messages as a system message.
+    """
+
+    type: Literal["llm-judge"]

nemo_platform/types/evaluation/llm_judge_metric_response.py

@@ -0,0 +1,120 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
+
+from typing import Dict, List, Union, Optional
+from datetime import datetime
+from typing_extensions import Literal, TypeAlias
+
+from . import model
+from ..._models import BaseModel
+from .model_ref import ModelRef
+from .range_score import RangeScore
+from .rubric_score import RubricScore
+from .inference_params import InferenceParams
+from .reasoning_params import ReasoningParams
+
+__all__ = ["LLMJudgeMetricResponse", "Model", "Score"]
+
+Model: TypeAlias = Union[model.Model, ModelRef]
+
+Score: TypeAlias = Union[RubricScore, RangeScore]
+
+
+class LLMJudgeMetricResponse(BaseModel):
+    model: Model
+    """The model configuration."""
+
+    scores: List[Score]
+    """Definitions of scores that will be extracted from the judge's output."""
+
+    id: Optional[str] = None
+    """Entity name within the workspace"""
+
+    created_at: Optional[datetime] = None
+
+    description: Optional[str] = None
+    """Human-readable description of the metric."""
+
+    ignore_request_failure: Optional[bool] = None
+    """If True, request failures will be ignored and the result will be marked as NaN.
+
+    If False (default), request failures will raise an exception.
+    """
+
+    inference: Optional[InferenceParams] = None
+    """Parameters for model inference.
+
+    Extra fields can be supplied for additional options applied to the inference
+    request directly. Fields not supported by the model may cause inference errors
+    during evaluation.
+    """
+
+    labels: Optional[Dict[str, str]] = None
+    """Labels are key-value pairs that can be used for grouping and filtering."""
+
+    name: Optional[str] = None
+    """Entity name within the workspace"""
+
+    optional_fields: Optional[List[str]] = None
+    """
+    Prompt template fields that should remain in the inferred input schema but not
+    be required. Use this for fields like 'reference' when the metric can still run
+    without them.
+    """
+
+    parent: Optional[str] = None
+
+    project: Optional[str] = None
+    """The name of the project associated with this entity."""
+
+    prompt_template: Union[str, Dict[str, object], None] = None
+    """The prompt template for the judge.
+
+    Can be either a simple string or a structured object (e.g., OpenAI messages
+    format). Use Jinja template variables like {{sample.output_text}} to use the
+    model output within the template or {{item.xxx}} to reference input columns from
+    the dataset.
+    """
+
+    reasoning: Optional[ReasoningParams] = None
+    """Custom settings that control the model's reasoning behavior."""
+
+    structured_output: Optional[Dict[str, object]] = None
+    """JSON schema to apply structured output for the judge model evaluation.
+
+    Structured output is derived from scores when omitted. Use this option if there
+    are custom requirements for the output of the judge.
+    """
+
+    supported_job_types: Optional[List[Literal["online", "offline"]]] = None
+    """
+    A metric can evaluate model outputs for online evaluations or pre-generated
+    outputs for offline evaluations.
+    """
+
+    system_prompt: Optional[str] = None
+    """
+    Initial instructions that define the judge model's role and behavior for the
+    conversation. This is prepended to the messages as a system message.
+    """
+
+    type: Optional[Literal["llm-judge"]] = None
+
+    updated_at: Optional[datetime] = None
+
+    workspace: Optional[str] = None
+    """Workspace identifier"""