nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/beta/evaluator/execution/config.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public configuration types for the v4 evaluator API."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, TypeAlias
+
+from nemo_platform.beta.evaluator.inference import PostprocessResponse, PreprocessRequest
+from nemo_platform.beta.evaluator.values import (
+    Agent,
+    Model,
+    RunConfig,
+    RunConfigOnline,
+    RunConfigOnlineModel,
+)
+from nemo_platform.beta.evaluator.values.datasets import DatasetInput
+from nemo_platform.beta.evaluator.values.results import AggregateFieldName
+
+_RunConfigT: TypeAlias = RunConfig | RunConfigOnline | RunConfigOnlineModel
+
+
+def normalize_params(
+    params: _RunConfigT | None = None,
+    target: Model | Agent | None = None,
+) -> _RunConfigT:
+    if isinstance(target, Model):
+        if not isinstance(params, RunConfigOnlineModel):
+            params = RunConfigOnlineModel(**params.model_dump()) if params else RunConfigOnlineModel()
+    elif isinstance(target, Agent):
+        if not isinstance(params, RunConfigOnline):
+            params = RunConfigOnline(**params.model_dump()) if params else RunConfigOnline()
+    else:
+        params = params or RunConfig()
+    return params
+
+
+def fail_fast_from_params(params: _RunConfigT) -> bool:
+    """
+    Return whether row failures should abort execution for the given params.
+    When params is not an online params, return fail_fast is True - always fail fast.
+    """
+    return not (isinstance(params, RunConfigOnline) and params.ignore_request_failure)
+
+
+@dataclass(frozen=True, slots=True)
+class EvaluationRequest:
+    """Normalized evaluator request passed from the public API to backends."""
+
+    dataset: DatasetInput | str | Path
+    params: _RunConfigT = field(default_factory=normalize_params)
+    target: Model | Agent | None = None
+    dataset_glob_pattern: str | None = None
+    prompt_template: str | dict[str, Any] | None = None
+    aggregate_fields: tuple[AggregateFieldName, ...] | None = None
+    preprocess_hooks: tuple[PreprocessRequest, ...] | None = None
+    postprocess_hooks: tuple[PostprocessResponse, ...] | None = None
+
+    def __post_init__(self) -> None:
+        """Normalize params to a concrete type compatible with the configured target."""
+        object.__setattr__(self, "params", normalize_params(self.params, self.target))

nemo_platform/beta/evaluator/execution/evaluator.py

@@ -0,0 +1,289 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Public evaluator entrypoint for completed-result execution."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any, TypeGuard, overload
+
+import nemo_platform.beta.evaluator.inference as inference
+from nemo_platform.beta.evaluator.execution.metric_execution import run_sync
+from nemo_platform.beta.evaluator.metrics.protocol import Metric
+from nemo_platform.beta.evaluator.values.agents import Agent
+from nemo_platform.beta.evaluator.values.datasets import DatasetInput
+from nemo_platform.beta.evaluator.values.models import Model
+from nemo_platform.beta.evaluator.values.multi_metric_results import BenchmarkEvaluationResult
+from nemo_platform.beta.evaluator.values.params import RunConfig
+from nemo_platform.beta.evaluator.values.results import AggregateFieldName, EvaluationResult
+
+from .backends.base import EvaluationBackend, SyncEvaluationBackend
+from .backends.local.backend import LocalBackend
+from .config import EvaluationRequest, normalize_params
+
+BackendClient = EvaluationBackend | SyncEvaluationBackend
+
+
+def _validate_backend_client(client: BackendClient) -> None:
+    """Validate that a backend client exposes callable evaluator methods.
+
+    Do not use runtime-checkable protocols for this check. ``EvaluationBackend``
+    and ``SyncEvaluationBackend`` share method names, and runtime protocol
+    checks cannot distinguish async methods from sync methods.
+
+    Args:
+        client: Backend client to validate.
+
+    Raises:
+        TypeError: If the backend client does not expose the evaluator backend methods.
+    """
+    missing = [
+        method_name
+        for method_name in ("evaluate", "evaluate_benchmark")
+        if not callable(getattr(client, method_name, None))
+    ]
+    if missing:
+        raise TypeError(
+            f"client must provide callable evaluate and evaluate_benchmark methods; missing: {', '.join(missing)}"
+        )
+
+
+def _is_async_backend(client: BackendClient) -> TypeGuard[EvaluationBackend]:
+    """Return whether the validated backend client exposes async evaluator methods."""
+    return inspect.iscoroutinefunction(client.evaluate) and inspect.iscoroutinefunction(client.evaluate_benchmark)
+
+
+def _is_sync_backend(client: BackendClient) -> TypeGuard[SyncEvaluationBackend]:
+    """Return whether the validated backend client exposes sync evaluator methods."""
+    return not inspect.iscoroutinefunction(client.evaluate) and not inspect.iscoroutinefunction(
+        client.evaluate_benchmark
+    )
+
+
+class _SyncBackendAdapter:
+    """Expose a sync evaluator backend through the async backend contract."""
+
+    def __init__(self, backend: SyncEvaluationBackend) -> None:
+        """Store the sync backend to execute off the event loop."""
+        self._backend = backend
+
+    async def evaluate(
+        self,
+        *,
+        metric: Metric,
+        request: EvaluationRequest,
+    ) -> EvaluationResult:
+        """Evaluate one metric by running the sync backend in a worker thread."""
+        return await asyncio.to_thread(self._backend.evaluate, metric=metric, request=request)
+
+    async def evaluate_benchmark(
+        self,
+        *,
+        metrics: Sequence[Metric],
+        request: EvaluationRequest,
+    ) -> BenchmarkEvaluationResult:
+        """Evaluate multiple metrics by running the sync backend in a worker thread."""
+        return await asyncio.to_thread(self._backend.evaluate_benchmark, metrics=metrics, request=request)
+
+
+class Evaluator:
+    """Evaluator convenience API for backends that return completed results.
+
+    ``Evaluator`` evaluates metrics locally by default. When constructed with an
+    evaluator backend object, it delegates completed-result execution to that
+    backend. Sync backends are adapted to the async backend contract.
+
+    Examples:
+        Local evaluation uses `run` directly:
+
+        ```python
+        evaluator = Evaluator()
+        result = await evaluator.run(
+            metrics=ExactMatchMetric(reference="{{item.reference}}"),
+            dataset=[{"reference": "Paris", "output_text": "Paris"}],
+        )
+        ```
+    """
+
+    def __init__(self, client: BackendClient | None = None) -> None:
+        """Create an evaluator for completed-result backends.
+
+        Args:
+            client: Optional evaluator backend. Async backends are used directly;
+                sync backends are adapted to the async backend contract. When
+                omitted, the evaluator runs metrics in-process via ``LocalBackend``.
+        """
+        if client is None:
+            self._backend: EvaluationBackend = LocalBackend()
+            return
+
+        _validate_backend_client(client)
+
+        if _is_async_backend(client):
+            self._backend = client
+        elif _is_sync_backend(client):
+            self._backend = _SyncBackendAdapter(client)
+        else:
+            raise TypeError(
+                "client must implement either async evaluate/evaluate_benchmark "
+                "or sync evaluate/evaluate_benchmark; "
+                "mixed sync/async clients are not supported"
+            )
+
+    @overload
+    async def run(
+        self,
+        metrics: Metric,
+        dataset: DatasetInput | str | Path,
+        *,
+        config: RunConfig | None = None,
+        target: Model | Agent | None = None,
+        dataset_glob_pattern: str | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> EvaluationResult: ...
+
+    @overload
+    async def run(
+        self,
+        metrics: Sequence[Metric],
+        dataset: DatasetInput | str | Path,
+        *,
+        config: RunConfig | None = None,
+        target: Model | Agent | None = None,
+        dataset_glob_pattern: str | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> BenchmarkEvaluationResult: ...
+
+    async def run(
+        self,
+        metrics: Metric | Sequence[Metric],
+        dataset: DatasetInput | str | Path,
+        *,
+        config: RunConfig | None = None,
+        target: Model | Agent | None = None,
+        dataset_glob_pattern: str | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> EvaluationResult | BenchmarkEvaluationResult:
+        """Evaluate metrics and return the finished result.
+
+        Args:
+            metrics: One metric or a sequence of metrics to execute.
+            dataset: Dataset input for the configured backend.
+            config: Optional run-level execution configuration.
+            model: Optional model used for online generation.
+            dataset_glob_pattern: Optional file selector within the provided dataset path.
+            prompt_template: Optional prompt template to use for online model generation.
+            aggregate_fields: Optional aggregate score fields to keep in the returned result.
+            preprocess_hooks: Optional request preprocess hooks for online execution.
+            postprocess_hooks: Optional response postprocess hooks for online execution.
+
+        Returns:
+            A single-metric or multi-metric result, matching the input metric
+            shape.
+        """
+        eval_request = EvaluationRequest(
+            dataset=dataset,
+            params=normalize_params(config, target),
+            target=target,
+            dataset_glob_pattern=dataset_glob_pattern,
+            prompt_template=prompt_template,
+            aggregate_fields=aggregate_fields,
+            preprocess_hooks=tuple(preprocess_hooks) if preprocess_hooks is not None else None,
+            postprocess_hooks=tuple(postprocess_hooks) if postprocess_hooks is not None else None,
+        )
+        if isinstance(metrics, Sequence):
+            return await self._backend.evaluate_benchmark(
+                metrics=list(metrics),
+                request=eval_request,
+            )
+        return await self._backend.evaluate(
+            metric=metrics,
+            request=eval_request,
+        )
+
+    @overload
+    def run_sync(
+        self,
+        metrics: Metric,
+        dataset: DatasetInput | str | Path,
+        *,
+        config: RunConfig | None = None,
+        target: Model | Agent | None = None,
+        dataset_glob_pattern: str | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> EvaluationResult: ...
+
+    @overload
+    def run_sync(
+        self,
+        metrics: Sequence[Metric],
+        dataset: DatasetInput | str | Path,
+        *,
+        config: RunConfig | None = None,
+        target: Model | Agent | None = None,
+        dataset_glob_pattern: str | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> BenchmarkEvaluationResult: ...
+
+    def run_sync(
+        self,
+        metrics: Metric | Sequence[Metric],
+        dataset: DatasetInput | str | Path,
+        *,
+        config: RunConfig | None = None,
+        target: Model | Agent | None = None,
+        dataset_glob_pattern: str | None = None,
+        prompt_template: str | dict[str, Any] | None = None,
+        aggregate_fields: tuple[AggregateFieldName, ...] | None = None,
+        preprocess_hooks: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess_hooks: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> EvaluationResult | BenchmarkEvaluationResult:
+        """Synchronously evaluate metrics and return the finished result.
+
+        Args:
+            metrics: One metric or a sequence of metrics to execute.
+            dataset: Dataset input for the configured backend.
+            config: Optional run-level execution configuration.
+            model: Optional model used for online generation.
+            dataset_glob_pattern: Optional file selector within the provided dataset path.
+            prompt_template: Optional prompt template for online execution.
+            aggregate_fields: Optional aggregate score fields to keep in the returned result.
+            preprocess_hooks: Optional request preprocess hooks for online execution.
+            postprocess_hooks: Optional response postprocess hooks for online execution.
+
+        Returns:
+            A single-metric or multi-metric result, matching the input metric
+            shape.
+        """
+        return run_sync(
+            lambda: self.run(
+                metrics=metrics,
+                dataset=dataset,
+                config=config,
+                target=target,
+                dataset_glob_pattern=dataset_glob_pattern,
+                prompt_template=prompt_template,
+                aggregate_fields=aggregate_fields,
+                preprocess_hooks=preprocess_hooks,
+                postprocess_hooks=postprocess_hooks,
+            )
+        )

nemo_platform/beta/evaluator/execution/job_poll.py

@@ -0,0 +1,98 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Async job polling aligned with ``nmp.testing.e2e.jobs.poll_until_terminal``.
+
+The E2E helper lives in ``nmp_testing``; the evaluator SDK cannot depend on that
+package at runtime, so this module mirrors its timeout semantics (pending time
+excluded from the job timeout, separate image-pull cap).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import time
+from collections.abc import Awaitable, Callable, Mapping
+from typing import TypeVar
+
+log = logging.getLogger(__name__)
+
+_StatusT = TypeVar("_StatusT")
+
+
+async def _async_pause(seconds: float) -> None:
+    await asyncio.sleep(seconds)
+
+
+async def async_poll_until_terminal(
+    get_status: Callable[[], Awaitable[_StatusT]],
+    *,
+    status_value: Callable[[_StatusT], str],
+    details_value: Callable[[_StatusT], Mapping[str, object] | None] | None = None,
+    job_name: str,
+    terminal: frozenset[str],
+    timeout: float,
+    pending_timeout: float,
+    poll_interval: float,
+) -> _StatusT:
+    """Poll *get_status* until it returns a value in *terminal* or a timeout fires.
+
+    *status_value* must return a **lowercase** status string for each response
+    returned by *get_status*.
+
+    *details_value* may return status details to include as a nested JSON object
+    in each progress log line.
+
+    Time spent in ``pending`` status is not counted against *timeout*; it is
+    instead capped by the separate *pending_timeout*.
+
+    Returns:
+        The terminal response returned by *get_status*.
+
+    Raises:
+        TimeoutError: When *timeout* is exceeded (excluding pending time) or
+            *pending_timeout* is exceeded while in pending status.
+    """
+    elapsed = 0.0
+    pending_elapsed = 0.0
+
+    while True:
+        poll_start = time.monotonic()
+        status_response = await get_status()
+        status = status_value(status_response)
+        log_payload: dict[str, object] = {
+            "job_name": job_name,
+            "status": status or "<empty>",
+            "elapsed_s": round(elapsed, 1),
+            "poll_interval_s": poll_interval,
+        }
+        if status == "pending":
+            log_payload["pending_elapsed_s"] = round(pending_elapsed, 1)
+        if details_value is not None:
+            details = details_value(status_response)
+            if details:
+                log_payload["status_details"] = dict(details)
+        log.info(json.dumps(log_payload, separators=(",", ":")))
+
+        if status in terminal:
+            return status_response
+
+        poll_duration = time.monotonic() - poll_start
+        if status == "pending":
+            pending_elapsed += poll_duration
+            if pending_elapsed >= pending_timeout:
+                raise TimeoutError(f"'{job_name}' stuck in pending after {pending_timeout}s.")
+        else:
+            elapsed += poll_duration
+            if elapsed >= timeout:
+                raise TimeoutError(f"'{job_name}' timed out after {timeout}s. Status: {status}")
+
+        sleep_start = time.monotonic()
+        await _async_pause(poll_interval)
+        sleep_duration = time.monotonic() - sleep_start
+        if status == "pending":
+            pending_elapsed += sleep_duration
+        else:
+            elapsed += sleep_duration