nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/beta/evaluator/values/models.py

@@ -0,0 +1,157 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Model-related value types."""
+
+from __future__ import annotations
+
+import logging
+import os
+from functools import cached_property
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+from pydantic.config import JsonDict
+
+from nemo_platform.beta.evaluator.enums import ModelFormat
+from nemo_platform.beta.evaluator.values.common import SecretRef
+
+logger = logging.getLogger(__name__)
+
+_AUTH_HEADER_PATTERNS: tuple[str, ...] = (
+    "auth",
+    "token",
+    "key",
+    "secret",
+    "credential",
+    "bearer",
+    "cookie",
+    "set-cookie",
+)
+
+
+class ReasoningParams(BaseModel):
+    """Custom settings that control the model's reasoning behavior."""
+
+    end_token: str | None = Field(
+        default=None,
+        description="Configure the end token to trim reasoning context based on the model's reasoning API. Example for Nemotron models: '</think>'",
+    )
+    include_if_not_finished: bool | None = Field(
+        default=None,
+        description="Configure whether to include reasoning context if the model has not finished reasoning.",
+    )
+    effort: str | None = Field(
+        default=None,
+        description="Option for OpenAI models to specify low, medium, or high reasoning effort.",
+    )
+
+
+def _strip_internal_fields(schema: JsonDict) -> None:
+    """Remove internal-only fields from the JSON schema so they don't appear in the OpenAPI spec."""
+    props = schema.get("properties")
+    if isinstance(props, dict):
+        props.pop("default_headers", None)
+        props.pop("host_url", None)
+
+
+def normalize_header_name(header_name: str) -> str:
+    """Normalize a transport header name for policy checks."""
+    return header_name.strip().lower().replace("_", "-")
+
+
+def is_auth_header_name(header_name: str) -> bool:
+    """Return whether a header name appears to carry authentication material."""
+    normalized_header_name = normalize_header_name(header_name)
+    return any(pattern in normalized_header_name for pattern in _AUTH_HEADER_PATTERNS)
+
+
+def filter_auth_headers(headers: dict[str, str] | None) -> dict[str, str] | None:
+    """Return only non-auth headers from a header mapping."""
+    if headers is None:
+        return None
+
+    filtered_headers: dict[str, str] = {}
+    for header_name, header_value in headers.items():
+        if is_auth_header_name(header_name):
+            logger.debug(
+                f"Filtered header {header_name} because it was recognized as an auth header",
+            )
+            continue
+        filtered_headers[header_name] = header_value
+
+    return filtered_headers or None
+
+
+class Model(BaseModel):
+    """Model definition for use without persisting to the Models API."""
+
+    model_config = ConfigDict(extra="forbid", json_schema_extra=_strip_internal_fields)
+
+    url: str = Field(description="URL of the model.")
+    name: str = Field(description="Name of the model.")
+    default_headers: dict[str, str] | None = Field(
+        default=None,
+        exclude=True,
+        description="Runtime-only non-auth headers automatically applied to requests made with this model. "
+        "Authentication must be configured via model.api_key_secret; auth headers such as Authorization will be rejected.",
+    )
+    host_url: str | None = Field(
+        default=None,
+        description="Direct NIM endpoint URL (http://host:port). Populated when resolved from a ModelRef. "
+        "Used by EvalFactory containers that reject path-based URLs (e.g., Haystack NvidiaDocumentEmbedder).",
+    )
+    api_key_secret: SecretRef | None = Field(
+        default=None,
+        description="API key secret reference for the model. Format: workspace/secret_name or secret_name within the job workspace.",
+    )
+    format: Literal[ModelFormat.NVIDIA_NIM, ModelFormat.OPEN_AI, ModelFormat.LLAMA_STACK] = Field(
+        default=ModelFormat.NVIDIA_NIM, description="API format of the model."
+    )
+
+    @field_validator("default_headers")
+    @classmethod
+    def validate_default_headers(cls, value: dict[str, str] | None) -> dict[str, str] | None:
+        """Reject auth-style headers and direct users to api_key_secret for credentials."""
+        if value is None:
+            return None
+
+        for header_name in value:
+            if is_auth_header_name(header_name):
+                raise ValueError(
+                    f"Header {header_name} in model.default_headers is rejected because model.default_headers cannot include authentication headers (Authorization, X-API-Key, etc.). "
+                    f"Header names are not allowed to contain the following substrings: {', '.join(_AUTH_HEADER_PATTERNS)}. "
+                    f"Configure model auth via model.api_key_secret instead."
+                )
+        return value
+
+    @cached_property
+    def api_key_env(self) -> str | None:
+        if self.api_key_secret:
+            env_name = self.api_key_secret.root
+            if env_name[0].isdigit():
+                env_name = f"_{env_name}"  # prefix with valid character for environment variable
+            return env_name.replace("-", "_").replace("/", "_")
+
+    @cached_property
+    def api_key(self) -> str | None:
+        if self.api_key_secret:
+            api_key_env = self.api_key_env
+            assert api_key_env is not None
+            return os.getenv(api_key_env) or os.getenv(api_key_env.upper())
+
+    def with_default_headers(self, headers: dict[str, str] | None) -> "Model":
+        """Return a copy of the model with merged runtime default headers."""
+        if not headers:
+            return self
+
+        return self.model_copy(
+            update={
+                "default_headers": {
+                    **(self.default_headers or {}),
+                    **headers,
+                }
+            },
+            # flat dict, no deep copy is needed
+            deep=False,
+        )

nemo_platform/beta/evaluator/values/multi_metric_results.py

@@ -0,0 +1,359 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Helpers for assembling multi-metric evaluator results."""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from pydantic import BaseModel
+
+from nemo_platform.beta.evaluator.metrics.protocol import MetricOutput
+from nemo_platform.beta.evaluator.values.results import (
+    AggregatedMetricResult,
+    AggregateFieldName,
+    EvaluationResult,
+    ResultView,
+    RowScore,
+    flatten_dict,
+    format_error_details,
+    format_table,
+    row_error_text,
+    row_status,
+    serialize_value,
+    summary_aggregate_record,
+    summary_header,
+    summary_row_base_record,
+)
+
+
+def _filter_aggregate_fields(
+    aggregate_scores: AggregatedMetricResult, aggregate_fields: tuple[AggregateFieldName, ...] | None
+) -> AggregatedMetricResult:
+    """Trim aggregate score payloads to the requested field subset.
+
+    Args:
+        aggregate_scores: Full aggregate metric output for one or more metrics.
+        aggregate_fields: Optional field names to retain on each score object.
+
+    Returns:
+        The original aggregate scores when no filtering is requested, otherwise
+        a copy that keeps only the selected fields.
+    """
+    if not aggregate_fields:
+        return aggregate_scores
+
+    filtered_scores = [score.with_fields(frozenset(aggregate_fields)) for score in aggregate_scores.scores]
+    return AggregatedMetricResult(scores=filtered_scores)
+
+
+def _extract_metric_outputs(row_score: RowScore, expected_key: str) -> list[MetricOutput]:
+    """Resolve the output list for one metric from a row result.
+
+    This exists because local and remote execution paths may return row scores
+    either already keyed by the final metric key or as a single unnamed metric
+    payload that still needs to be attached to that key.
+
+    Args:
+        row_score: Row-level result payload to inspect.
+        expected_key: Metric key the caller expects to find on the row.
+
+    Returns:
+        The output list for the requested metric key, or an empty list when the
+        row has no metric output because evaluation failed.
+
+    Raises:
+        ValueError: If the row contains multiple metric entries and none match
+            the requested key.
+    """
+    if expected_key in row_score.metrics:
+        return row_score.metrics[expected_key]
+    if not row_score.metrics:
+        return []
+    if len(row_score.metrics) == 1:
+        return next(iter(row_score.metrics.values()))
+    raise ValueError(f"Unable to resolve row metric outputs for key {expected_key!r}")
+
+
+def _extract_metric_error(row_score: RowScore, expected_key: str) -> str | None:
+    """Resolve one metric error message from a row result."""
+    metric_errors = row_score.metric_errors
+    if metric_errors and expected_key in metric_errors:
+        return metric_errors[expected_key]
+    if metric_errors:
+        if len(metric_errors) == 1:
+            return next(iter(metric_errors.values()))
+        raise ValueError(f"Unable to resolve row metric error for key {expected_key!r}")
+    return row_score.error
+
+
+def _row_identity(row_score: RowScore, fallback_index: int) -> int:
+    """Resolve the stable row identity used when combining metric results.
+
+    Args:
+        row_score: Row-level result payload.
+        fallback_index: Positional fallback when no explicit row identity is present.
+
+    Returns:
+        Stable row identity for result alignment.
+    """
+    return row_score.row_index if row_score.row_index is not None else fallback_index
+
+
+def namespace_result(
+    metric_key: str,
+    result: EvaluationResult,
+    aggregate_fields: tuple[AggregateFieldName, ...] | None,
+) -> EvaluationResult:
+    """Rewrite a single-metric result to use a stable `v4` metric namespace.
+
+    Args:
+        metric_key: Public metric key assigned by the evaluator.
+        result: Raw single-metric evaluation result from one backend.
+        aggregate_fields: Optional aggregate field subset to keep.
+
+    Returns:
+        A result whose row metrics and aggregate score names are prefixed with
+        the evaluator-assigned metric key.
+    """
+    row_scores = [
+        RowScore(
+            row_index=row_score.row_index,
+            item=row_score.item,
+            sample=row_score.sample,
+            metrics={metric_key: _extract_metric_outputs(row_score, metric_key)},
+            requests=row_score.requests,
+            metric_errors={metric_key: error} if (error := _extract_metric_error(row_score, metric_key)) else None,
+        )
+        for row_score in result.row_scores
+    ]
+    aggregate_scores = AggregatedMetricResult(
+        scores=[
+            score.model_copy(update={"name": f"{metric_key}.{score.name}"}) for score in result.aggregate_scores.scores
+        ]
+    )
+    aggregate_scores = _filter_aggregate_fields(aggregate_scores, aggregate_fields)
+    return EvaluationResult(row_scores=row_scores, aggregate_scores=aggregate_scores)
+
+
+def collapse_results(
+    results_by_key: dict[str, EvaluationResult],
+    aggregate_fields: tuple[AggregateFieldName, ...] | None,
+) -> BenchmarkEvaluationResult:
+    """Merge multiple single-metric results into one multi-metric view.
+
+    Args:
+        results_by_key: Namespaced single-metric results keyed by metric name.
+        aggregate_fields: Optional aggregate field subset to keep.
+
+    Returns:
+        A combined multi-metric result with merged row scores, merged aggregate
+        scores, and the original per-metric mapping.
+
+    Raises:
+        ValueError: If the provided metric results do not all contain the same
+            number of rows.
+    """
+    if not results_by_key:
+        empty = AggregatedMetricResult(scores=[])
+        return BenchmarkEvaluationResult(row_scores=[], aggregate_scores=empty, per_metric={})
+
+    ordered_keys = list(results_by_key.keys())
+    row_count = len(results_by_key[ordered_keys[0]].row_scores)
+    baseline_identities = [
+        _row_identity(row_score, index) for index, row_score in enumerate(results_by_key[ordered_keys[0]].row_scores)
+    ]
+    for metric_key in ordered_keys[1:]:
+        result = results_by_key[metric_key]
+        if len(result.row_scores) != row_count:
+            raise ValueError(f"Cannot combine metric results with different row counts: {metric_key}")
+        current_identities = [_row_identity(row_score, index) for index, row_score in enumerate(result.row_scores)]
+        if current_identities != baseline_identities:
+            raise ValueError(f"Cannot combine metric results with different row identities: {metric_key}")
+
+    combined_rows: list[RowScore] = []
+    for index in range(row_count):
+        first_row = results_by_key[ordered_keys[0]].row_scores[index]
+        metrics: dict[str, list[MetricOutput]] = {}
+        requests: list[dict[str, Any]] = []
+        metric_errors: dict[str, str] = {}
+        for metric_key in ordered_keys:
+            row_score = results_by_key[metric_key].row_scores[index]
+            metrics[metric_key] = _extract_metric_outputs(row_score, metric_key)
+            requests.extend(row_score.requests)
+            if row_score.metric_errors:
+                metric_errors.update(row_score.metric_errors)
+            elif row_score.error:
+                metric_errors[metric_key] = row_score.error
+        combined_rows.append(
+            RowScore(
+                row_index=_row_identity(first_row, index),
+                item=first_row.item,
+                sample=first_row.sample,
+                metrics=metrics,
+                requests=requests,
+                metric_errors=metric_errors or None,
+            )
+        )
+
+    aggregate_scores = AggregatedMetricResult(
+        scores=[score for result in results_by_key.values() for score in result.aggregate_scores.scores]
+    )
+    aggregate_scores = _filter_aggregate_fields(aggregate_scores, aggregate_fields)
+    return BenchmarkEvaluationResult(
+        row_scores=combined_rows,
+        aggregate_scores=aggregate_scores,
+        per_metric=results_by_key,
+    )
+
+
+class BenchmarkEvaluationResult(BaseModel):
+    """Unified benchmark evaluation result."""
+
+    row_scores: list[RowScore]
+    aggregate_scores: AggregatedMetricResult
+    per_metric: dict[str, EvaluationResult]
+
+    def metric_result(self, metric_key: str) -> EvaluationResult:
+        """Return the original single-metric result for one metric key.
+
+        Args:
+            metric_key: Metric key to retrieve from the combined result.
+
+        Returns:
+            The single-metric result stored for that key.
+        """
+        return self.per_metric[metric_key]
+
+    def to_records(self, view: ResultView = "rows") -> list[dict[str, Any]]:
+        """Convert the result into flat dictionaries for export or inspection.
+
+        Args:
+            view: Which logical result view to flatten, either `"rows"` or
+                `"aggregate"`.
+
+        Returns:
+            A list of flat dictionaries suitable for tabular rendering.
+
+        Raises:
+            ValueError: If `view` is not one of the supported options.
+        """
+        if view == "rows":
+            records: list[dict[str, Any]] = []
+            for index, row_score in enumerate(self.row_scores):
+                record: dict[str, Any] = {
+                    "row_index": row_score.row_index if row_score.row_index is not None else index,
+                    "status": row_status(row_score),
+                }
+                flatten_dict("item", serialize_value(row_score.item), record)
+                flatten_dict("sample", serialize_value(row_score.sample), record)
+                if error_text := row_error_text(row_score):
+                    record["error"] = error_text
+                for metric_key, metric_scores in row_score.metrics.items():
+                    for output in metric_scores:
+                        record[f"output.{metric_key}.{output.name}"] = serialize_value(output.value)
+                records.append(record)
+            return records
+
+        if view == "aggregate":
+            records = []
+            for score in self.aggregate_scores.scores:
+                record = {}
+                for key, value in score.model_dump(mode="json").items():
+                    if key == "percentiles" and isinstance(value, dict):
+                        flatten_dict("percentiles", value, record)
+                    elif key == "histogram" and value is not None:
+                        record[key] = json.dumps(value, sort_keys=True)
+                    else:
+                        record[key] = value
+                records.append(record)
+            return records
+
+        raise ValueError(f"Unsupported view {view!r}. Expected 'rows' or 'aggregate'.")
+
+    def to_table(self, view: ResultView = "rows"):
+        """Convert the result into a `pyarrow.Table`.
+
+        Args:
+            view: Which logical result view to materialize.
+
+        Returns:
+            A PyArrow table built from the flattened result records.
+        """
+        import pyarrow as pa
+
+        return pa.Table.from_pylist(self.to_records(view=view))
+
+    def to_pandas(self, view: ResultView = "rows"):
+        """Convert the result into a pandas `DataFrame`.
+
+        Args:
+            view: Which logical result view to materialize.
+
+        Returns:
+            A pandas dataframe built from the flattened result records.
+        """
+        import pandas as pd
+
+        return pd.DataFrame.from_records(self.to_records(view=view))
+
+    def format_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> str:
+        """Build a human-readable text summary of the result.
+
+        Args:
+            max_rows: Maximum number of row records to include in the preview.
+            max_error_rows: Maximum number of failed rows included in the full
+                error-details section. Defaults to ``max_rows``.
+
+        Returns:
+            A formatted multiline string summary.
+        """
+        if max_error_rows is None:
+            max_error_rows = max_rows
+        aggregate_records = [summary_aggregate_record(score) for score in self.aggregate_scores.scores]
+        parts = [
+            summary_header("BenchmarkEvaluationResult", self.row_scores, len(self.aggregate_scores.scores)),
+            "",
+            "Aggregate scores",
+            format_table(aggregate_records),
+        ]
+        for metric_key, metric_result in self.per_metric.items():
+            preview_records = []
+            for index, row_score in enumerate(metric_result.row_scores[:max_rows]):
+                record = summary_row_base_record(row_score, index)
+                for metric_scores in row_score.metrics.values():
+                    for score in metric_scores:
+                        record[f"score.{score.name}"] = serialize_value(score.value)
+                preview_records.append(record)
+            if not preview_records:
+                continue
+            parts.extend(
+                [
+                    "",
+                    f"Row preview for metric '{metric_key}' (first {len(preview_records)} of {len(metric_result.row_scores)})",
+                    format_table(preview_records),
+                ]
+            )
+        parts.extend(
+            format_error_details(
+                self.row_scores,
+                max_error_rows=max_error_rows,
+                label_metric_errors=True,
+            )
+        )
+        return "\n".join(parts)
+
+    def print_summary(self, max_rows: int = 10, *, max_error_rows: int | None = None) -> None:
+        """Print the formatted summary to standard output.
+
+        Args:
+            max_rows: Maximum number of row records to include in the preview.
+            max_error_rows: Maximum number of failed rows included in the full
+                error-details section. Defaults to ``max_rows``.
+
+        Returns:
+            None.
+        """
+        print(self.format_summary(max_rows=max_rows, max_error_rows=max_error_rows))

nemo_platform/beta/evaluator/values/params.py

@@ -0,0 +1,109 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Shared execution parameter types for evaluator SDK and service runtimes."""
+
+from typing import Self
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from nemo_platform.beta.evaluator.values.models import ReasoningParams
+
+
+class InferenceParams(BaseModel):
+    """
+    Parameters for model inference. Extra fields can be supplied for additional options applied to the inference request directly. Fields not supported by the model may cause inference errors during evaluation.
+    """
+
+    model_config = ConfigDict(extra="allow")
+
+    temperature: float | None = Field(
+        default=None,
+        ge=0,
+        le=2,
+        description="Float value between 0 and 1. temp of 0 indicates greedy decoding, "
+        "where the token with highest prob is chosen. Temperature can't be set to 0.0 currently",
+    )
+    max_tokens: int | None = Field(default=None, ge=1, description="Max tokens to generate")
+    max_completion_tokens: int | None = Field(default=None, ge=1, description="Max tokens to generate")
+    top_p: float | None = Field(
+        default=None,
+        ge=0,
+        le=1,
+        description="Float value between 0 and 1; limits to the top tokens within a certain "
+        "probability. top_p=0 means the model will only consider the single most likely "
+        "token for the next prediction",
+    )
+    stop: list[str] | None = Field(default=None)
+
+    @model_validator(mode="after")
+    def check_max_tokens(self) -> Self:
+        if self.max_tokens and self.max_completion_tokens:
+            raise ValueError(
+                "max_tokens and max_completion_tokens cannot both be configured. "
+                "Choose the appropriate tokens parameter for the model."
+            )
+        return self
+
+
+class RunConfig(BaseModel):
+    """Job parameters."""
+
+    model_config = ConfigDict(extra="forbid")
+
+    parallelism: int = Field(
+        default=8,
+        ge=1,
+        description="Parallelism to be used for the evaluation job. "
+        "Typically, this represents the maximum number of concurrent requests made to the model.",
+    )
+    limit_samples: int | None = Field(
+        default=None,
+        ge=1,
+        description="Limit number of evaluation samples, taking the first `limit` samples from the dataset.",
+    )
+
+
+class RunConfigOnline(RunConfig):
+    """Job parameters for online evaluation."""
+
+    ignore_request_failure: bool = Field(
+        default=False,
+        description="If True, request failures will be ignored and the result will be marked as NaN. "
+        "If False (default), request failures will raise an exception.",
+    )
+    request_timeout: int | None = Field(
+        default=None, description="The timeout to be used for requests made to the model."
+    )
+    max_retries: int = Field(default=3, ge=0, description="Maximum number of retries for failed requests.")
+
+
+class RunConfigOnlineModel(RunConfigOnline):
+    """Job parameters for model online evaluation."""
+
+    inference: InferenceParams | None = Field(
+        default=None,
+        description="Custom settings that control the model's text generation behavior.",
+    )
+    system_prompt: str | None = Field(
+        default=None,
+        description="Initial instructions that define the model's role and behavior for the conversation.",
+    )
+    reasoning: ReasoningParams | None = Field(
+        default=None, description="Custom settings that control the model's reasoning behavior."
+    )
+    structured_output: dict | None = Field(
+        default=None,
+        description="JSON schema to apply structured output for the model.",
+    )
+
+    @field_validator("inference", mode="before")
+    @classmethod
+    def coerce_inference_params(cls, value: object) -> object:
+        """Normalize equivalent Pydantic inference models before SDK validation runs."""
+        # Compatibility shim for service-side inference models. This converts other
+        # Pydantic models into plain data and lets SDK InferenceParams validation
+        # decide whether the payload is compatible.
+        if isinstance(value, BaseModel) and not isinstance(value, InferenceParams):
+            return value.model_dump(mode="python", exclude_none=True)
+        return value