nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/beta/evaluator/execution/pipeline.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Pipeline infrastructure types for online execution."""
+
+from __future__ import annotations
+
+import asyncio
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any, Protocol
+
+from nemo_platform.beta.evaluator.metrics.protocol import MetricResult
+from nemo_platform.beta.evaluator.values.results import RowScore
+
+
+@dataclass
+class GeneratedSampleEvent:
+    """Carries one generated sample row from producer workers to scorer workers."""
+
+    row_index: int
+    item: MappingProxyType
+    sample: MappingProxyType
+    requests_log: list[dict[str, Any]]
+
+
+class GeneratedSampleScoringPipeline(Protocol):
+    """Internal contract for pipelines that generate a sample before scoring."""
+
+    rows: list[dict[str, Any]]
+    parallelism: int
+
+    async def generate_sample(self, index: int, row: dict[str, Any]) -> dict[str, Any]:
+        """Prepare the generated sample payload for one input row."""
+        ...
+
+    def handle_generation_error(
+        self,
+        index: int,
+        row: dict[str, Any],
+        error: Exception,
+        generation_requests: list[dict[str, Any]],
+    ) -> tuple[int, MetricResult | None, RowScore]:
+        """Convert a generation failure into a completed row result or raise."""
+        ...
+
+    async def score_row(
+        self,
+        index: int,
+        row: dict[str, Any],
+        sample: dict[str, Any],
+        generation_requests: list[dict[str, Any]],
+    ) -> tuple[int, MetricResult | None, RowScore]:
+        """Score one prepared sample and return the completed row result."""
+        ...
+
+
+@dataclass
+class PipelineRuntime:
+    """Mutable queue-worker state shared by the generic pipeline helpers."""
+
+    pipeline: GeneratedSampleScoringPipeline
+    sample_queue: asyncio.Queue[GeneratedSampleEvent | object]
+    results: list[tuple[int, MetricResult | None, RowScore] | None]

nemo_platform/beta/evaluator/execution/runs.py

@@ -0,0 +1,143 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Job-oriented run handle types for evaluator backends."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult
+from nemo_platform.beta.evaluator.values.results import EvaluationResult
+
+
+class JobEvaluationRun(Protocol):
+    """Structural contract for job-backed single-metric run handles."""
+
+    async def status(self) -> str:
+        """Return the current lifecycle status for the run.
+
+        Returns:
+            A backend-defined status string such as `created`, `active`, or
+            `completed`.
+        """
+        ...
+
+    async def result(
+        self,
+        *,
+        timeout_s: float | None = None,
+        poll_interval_s: float = 1.0,
+    ) -> EvaluationResult:
+        """Wait for the run to finish and return its metric result.
+
+        Args:
+            timeout_s: Optional maximum wait time before raising `TimeoutError`.
+            poll_interval_s: Poll interval used by backends that require
+                repeated status checks.
+
+        Returns:
+            The completed single-metric evaluation result.
+        """
+        ...
+
+    def result_sync(
+        self,
+        *,
+        timeout_s: float | None = None,
+        poll_interval_s: float = 1.0,
+    ) -> EvaluationResult:
+        """Synchronously wait for the run to finish and return its result.
+
+        Args:
+            timeout_s: Optional maximum wait time before raising `TimeoutError`.
+            poll_interval_s: Poll interval used by backends that require
+                repeated status checks.
+
+        Returns:
+            The completed single-metric evaluation result.
+        """
+        ...
+
+    def job(self) -> Any | None:
+        """Return the primary backend job object, if one exists.
+
+        Returns:
+            A backend-specific job object for single-job runs, or `None` for
+            purely local completed runs.
+        """
+        ...
+
+    def jobs(self) -> list[Any]:
+        """Return all backend job objects associated with the run.
+
+        Returns:
+            One or more backend-specific job objects.
+        """
+        ...
+
+
+class JobBenchmarkEvaluationRun(Protocol):
+    """Structural contract for job-backed multi-metric run handles."""
+
+    async def status(self) -> str:
+        """Return the current lifecycle status for the combined run.
+
+        Returns:
+            A backend-defined status string aggregated across all underlying
+            metric runs.
+        """
+        ...
+
+    async def result(
+        self,
+        *,
+        timeout_s: float | None = None,
+        poll_interval_s: float = 1.0,
+    ) -> BenchmarkEvaluationResult:
+        """Wait for all metrics to finish and return the combined result.
+
+        Args:
+            timeout_s: Optional maximum wait time before raising `TimeoutError`.
+            poll_interval_s: Poll interval used by backends that require
+                repeated status checks.
+
+        Returns:
+            The completed multi-metric evaluation result.
+        """
+        ...
+
+    def result_sync(
+        self,
+        *,
+        timeout_s: float | None = None,
+        poll_interval_s: float = 1.0,
+    ) -> BenchmarkEvaluationResult:
+        """Synchronously wait for all metrics to finish and return the result.
+
+        Args:
+            timeout_s: Optional maximum wait time before raising `TimeoutError`.
+            poll_interval_s: Poll interval used by backends that require
+                repeated status checks.
+
+        Returns:
+            The completed multi-metric evaluation result.
+        """
+        ...
+
+    def job(self) -> Any | None:
+        """Return a primary backend job object when one exists.
+
+        Returns:
+            A backend-specific job object, or `None` when the run is backed by
+            multiple jobs or no remote jobs at all.
+        """
+        ...
+
+    def jobs(self) -> list[Any]:
+        """Return all backend job objects associated with the run.
+
+        Returns:
+            One or more backend-specific job objects.
+        """
+        ...

nemo_platform/beta/evaluator/execution/scoring.py

@@ -0,0 +1,224 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Shared row-scoring and result-finalization primitives used during execution."""
+
+from collections.abc import Iterable, Sequence
+from logging import Logger, getLogger
+from typing import Any
+
+from nemo_platform.beta.evaluator.execution.values import EvaluationError, EvaluationPhase
+from nemo_platform.beta.evaluator.inference import requests_log_var
+from nemo_platform.beta.evaluator.metrics.aggregation import (
+    add_corpus_scores,
+    aggregate_metrics,
+    is_aggregateable_output_spec,
+    rubric_definitions_from_metric,
+)
+from nemo_platform.beta.evaluator.metrics.protocol import (
+    CandidateOutput,
+    CorpusMetric,
+    DatasetRow,
+    Metric,
+    MetricInput,
+    MetricOutput,
+    MetricOutputSpec,
+    MetricResult,
+    validate_metric_result,
+)
+from nemo_platform.beta.evaluator.metrics.utils import metric_type_name
+from nemo_platform.beta.evaluator.values import (
+    EvaluationResult,
+    RowScore,
+)
+
+logger = getLogger(__name__)
+
+
+_CANDIDATE_SAMPLE_FIELDS = frozenset({"output_text", "response", "trajectory"})
+
+
+def build_metric_input(row: dict[str, Any], sample: dict[str, Any], index: int | None = None) -> MetricInput:
+    """Build the metric protocol input from dataset row and generated sample payloads."""
+    output_text = sample.get("output_text")
+    metadata = {
+        key: value
+        for key, value in sample.items()
+        if key not in _CANDIDATE_SAMPLE_FIELDS or (key == "output_text" and not isinstance(output_text, str))
+    }
+    return MetricInput(
+        row=DatasetRow(row_index=index, data=row),
+        candidate=CandidateOutput(
+            output_text=output_text if isinstance(output_text, str) else None,
+            response=sample.get("response"),
+            trajectory=sample.get("trajectory"),
+            metadata=metadata,
+        ),
+    )
+
+
+def nan_metric_result(outputs: Iterable[MetricOutputSpec]) -> MetricResult:
+    """Build the NaN output payload used for ignored scoring failures.
+
+    Only aggregateable outputs receive NaN placeholders; non-score outputs are
+    not synthesized for failed rows.
+    """
+    return MetricResult(
+        outputs=[
+            MetricOutput(name=output.name, value=float("nan"))
+            for output in outputs
+            if is_aggregateable_output_spec(output)
+        ]
+    )
+
+
+def corpus_output_spec(metric: Metric, fallback: list[MetricOutputSpec] | None = None) -> list[MetricOutputSpec]:
+    """Return corpus-level output specs when a metric declares them."""
+    corpus_spec = getattr(metric, "corpus_output_spec", None)
+    if callable(corpus_spec):
+        return list(corpus_spec())
+    return list(fallback if fallback is not None else metric.output_spec())
+
+
+CompletedRowEvaluation = tuple[int, MetricResult | None, RowScore]
+
+
+def empty_evaluation_result() -> EvaluationResult:
+    """Return the canonical empty evaluation result payload."""
+    return EvaluationResult(row_scores=[], aggregate_scores=aggregate_metrics([], []))
+
+
+async def finalize_evaluation_result(
+    metric: Metric,
+    eval_results: Sequence[CompletedRowEvaluation],
+    *,
+    skip_errored: bool = False,
+) -> EvaluationResult:
+    """Build the final evaluation result from eval_results row-level outputs.
+
+    Callers are expected to pass ``eval_results`` in the original row order; the
+    upstream pipeline (``run_indexed_tasks`` / ``run_generated_sample_scoring_pipeline``)
+    already writes results by index, so no re-sorting is performed here.
+
+    When ``skip_errored`` is true, rows whose ``RowScore.metric_errors`` is
+    populated are excluded from aggregation so the NaN placeholder produced
+    for ignored failures does not contribute to ``nan_count``, and the same
+    rows are excluded from the ``items``/``samples`` passed to
+    :meth:`CorpusMetric.compute_corpus_scores` so corpus-level aggregation
+    (e.g. BLEU/ROUGE-corpus) isn't skewed by failed rows with empty samples.
+    Errored rows still appear in ``row_scores``.
+    """
+    valid_eval_results = [
+        (result, row_score)
+        for _, result, row_score in eval_results
+        if result is not None and not (skip_errored and row_score.metric_errors)
+    ]
+    metric_results = [result for result, _ in valid_eval_results]
+    # Keep all rows in the reported ``row_scores`` (including errored ones); only
+    # aggregation and corpus inputs honor ``skip_errored``.
+    row_scores = [row_score for _, _, row_score in eval_results]
+
+    output_spec = metric.output_spec()
+    rubric_definitions = rubric_definitions_from_metric(metric)
+    if rubric_definitions:
+        aggregated_result = aggregate_metrics(metric_results, output_spec, rubric_definitions=rubric_definitions)
+    else:
+        aggregated_result = aggregate_metrics(metric_results, output_spec)
+
+    if valid_eval_results and isinstance(metric, CorpusMetric):
+        corpus_metric_result = await metric.compute_corpus_scores(
+            inputs=[
+                build_metric_input(row_score.item, row_score.sample, row_score.row_index)
+                for _, row_score in valid_eval_results
+            ],
+        )
+        if corpus_metric_result:
+            add_corpus_scores(aggregated_result, corpus_metric_result, corpus_output_spec(metric, output_spec))
+
+    return EvaluationResult(
+        row_scores=row_scores,
+        aggregate_scores=aggregated_result,
+    )
+
+
+async def score_row(
+    metric: Metric,
+    row: dict[str, Any],
+    sample: dict[str, Any],
+    index: int,
+    metric_key: str,
+    fail_fast: bool,
+    generation_requests: list[dict[str, Any]],
+    logger: Logger | None = None,
+) -> tuple[int, MetricResult | None, RowScore]:
+    """Score an already-prepared sample for one row.
+
+    Args:
+        metric: Metric object used for scoring.
+        row: Input row from the dataset.
+        sample: Prepared sample payload passed to the metric.
+        index: Row position in the original dataset.
+        metric_key: Key used to place score output in ``RowScore.metrics``.
+        fail_fast: Whether metric errors should raise immediately. When
+            ``True``, the exception is wrapped in ``EvaluationError`` and
+            raised. When ``False``, a metric exception yields a NaN score row.
+        generation_requests: Requests collected before metric scoring,
+            such as online generation requests.
+        logger: Optional logger override for row-scoring logs.
+    Returns:
+        Tuple of ``(index, metric_result_or_none, row_score_payload)``.
+
+    Raises:
+        EvaluationError: If row evaluation fails and ``fail_fast`` is ``True``.
+    """
+
+    metric_requests: list[dict[str, Any]] = []
+    requests_log_var.set(metric_requests)
+    active_logger = logger or globals()["logger"]
+
+    try:
+        output_spec = metric.output_spec()
+        result = validate_metric_result(
+            await metric.compute_scores(build_metric_input(row, sample, index)), output_spec
+        )
+        active_logger.debug(
+            "Computed metric",
+            extra={
+                "item_index": index,
+                "metric_type": metric_type_name(metric),
+                "outputs": [output.model_dump() for output in result.outputs],
+            },
+        )
+        return (
+            index,
+            result,
+            RowScore(
+                row_index=index,
+                item=row,
+                sample=sample,
+                metrics={metric_key: result.outputs},
+                requests=[*generation_requests, *metric_requests],
+            ),
+        )
+    except Exception as e:
+        if fail_fast:
+            raise EvaluationError(
+                index,
+                str(e),
+                phase=EvaluationPhase.METRIC_SCORING,
+                metric_key=metric_key,
+            ) from e
+        active_logger.warning("Evaluation failed, marking as NaN", extra={"item_index": index, "error": str(e)})
+        result = nan_metric_result(metric.output_spec())
+        return (
+            index,
+            result,
+            RowScore(
+                row_index=index,
+                item=row,
+                sample=sample,
+                metrics={metric_key: result.outputs},
+                requests=[*generation_requests, *metric_requests],
+                metric_errors={metric_key: str(e)},
+            ),
+        )

nemo_platform/beta/evaluator/execution/utils.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Local metric preparation helpers for evaluator SDK runtime."""
+
+from __future__ import annotations
+
+import copy
+import os
+from collections.abc import Sequence
+from typing import cast
+
+from nemo_platform.beta.evaluator.execution._protocols import JobParamsConfigurableMetric
+from nemo_platform.beta.evaluator.metrics.protocol import Metric, MetricWithPreflight, MetricWithSecrets
+from nemo_platform.beta.evaluator.metrics.utils import metric_type_name
+from nemo_platform.beta.evaluator.values.params import RunConfig
+from pydantic import BaseModel
+
+
+def unique_metric_keys(metrics: Sequence[Metric]) -> list[str]:
+    """Assign stable unique keys to a sequence of metrics.
+
+    Args:
+        metrics: Metrics submitted in one evaluator call.
+
+    Returns:
+        Unique metric keys in the same order as the input metrics.
+    """
+
+    seen: dict[str, int] = {}
+    keys: list[str] = []
+    for metric in metrics:
+        base = metric_type_name(metric)
+        seen[base] = seen.get(base, 0) + 1
+        suffix = seen[base]
+        keys.append(base if suffix == 1 else f"{base}_{suffix}")
+    return keys
+
+
+def _copy_metric(metric: Metric) -> Metric:
+    """Create a best-effort isolated copy of a metric instance.
+
+    Preparation mutates metrics in-place (run-config overrides, secret
+    resolution, preflight) so we copy first to avoid side-effects on the
+    caller's original instance. Pydantic models are copied via
+    ``model_copy(deep=True)``; regular Python metric classes are also
+    supported when ``copy.deepcopy()`` works, for example by implementing
+    ``__deepcopy__``.
+    """
+    if isinstance(metric, BaseModel):
+        return cast(Metric, metric.model_copy(deep=True))
+
+    try:
+        return copy.deepcopy(metric)
+    except Exception as exc:
+        raise TypeError(
+            f"Cannot copy metric {type(metric).__name__}; use a Pydantic model or ensure the metric "
+            "supports copy.deepcopy() (for example, by implementing __deepcopy__)."
+        ) from exc
+
+
+def _candidate_env_names(secret_name: str) -> list[str]:
+    """Generate environment variable names that may contain one secret."""
+    names = [secret_name, secret_name.upper()]
+    normalized = secret_name.replace("-", "_").replace("/", "_")
+    names.extend([normalized, normalized.upper()])
+    if normalized and normalized[0].isdigit():
+        prefixed = f"_{normalized}"
+        names.extend([prefixed, prefixed.upper()])
+    return list(dict.fromkeys(names))
+
+
+async def _resolve_secret_from_env(secret_name: str) -> str | None:
+    """Resolve one secret value from environment variables.
+
+    Async to satisfy the ``SecretResolver`` protocol expected by metrics.
+    """
+    for candidate in _candidate_env_names(secret_name):
+        value = os.getenv(candidate)
+        if value:
+            return value
+    return None
+
+
+def _apply_runtime_params(metric: Metric, params: RunConfig) -> None:
+    """Apply runtime execution params to one prepared metric.
+
+    Args:
+        metric: Copied metric instance to mutate for this run only.
+        params: Materialized execution params for this run.
+
+    Returns:
+        None.
+    """
+    if isinstance(metric, JobParamsConfigurableMetric):
+        metric.apply_evaluation_job_params(params)
+
+
+async def prepare_metric_for_local_execution(metric: Metric, params: RunConfig) -> Metric:
+    """Prepare one metric for execution.
+
+    Args:
+        metric: User-provided metric instance.
+        params: Materialized execution params for this run.
+
+    Returns:
+        A metric object ready for execution in the selected backend.
+    """
+    prepared = _copy_metric(metric)
+    _apply_runtime_params(prepared, params)
+
+    if isinstance(prepared, MetricWithSecrets):
+        await prepared.resolve_secrets(_resolve_secret_from_env)
+    if isinstance(prepared, MetricWithPreflight):
+        await prepared.preflight()
+    return prepared

nemo_platform/beta/evaluator/execution/values.py

@@ -0,0 +1,53 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Value types used during pipeline execution."""
+
+from enum import Enum
+
+
+class EvaluationPhase(str, Enum):
+    """Phase where a row failure is raised instead of converted to NaN."""
+
+    SAMPLE_GENERATION = "sample_generation"
+    METRIC_SCORING = "metric_scoring"
+
+
+class EvaluationError(Exception):
+    """Raised when evaluation fails with ``fail_fast=True``.
+
+    ``metric_key`` is the public metric identifier. For single-metric
+    pipelines it is the metric type name; for multi-metric benchmark
+    pipelines it is the fully-qualified metric ref.
+    """
+
+    def __init__(
+        self,
+        index: int,
+        message: str,
+        *,
+        phase: EvaluationPhase = EvaluationPhase.METRIC_SCORING,
+        metric_key: str | None = None,
+    ) -> None:
+        """Create an evaluation error that keeps sample index context.
+
+        Strict mode means ``fail_fast=True``: row failures abort evaluation
+        instead of being converted into NaN row results.
+
+        Args:
+            index: Position of the failing sample in the input list.
+            message: Original exception message.
+            phase: Execution phase where the row failed.
+            metric_key: Public metric identifier for the failing metric.
+        """
+        self.index = index
+        self.message = message
+        self.phase = phase
+        self.metric_key = metric_key
+        super().__init__(self._format_message())
+
+    def _format_message(self) -> str:
+        """Return a concise user-facing error message."""
+        phase = self.phase.value.replace("_", " ")
+        metric = f" for metric {self.metric_key!r}" if self.metric_key else ""
+        return f"Evaluation failed during {phase}{metric} on row {self.index}: {self.message}"