nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/beta/evaluator/metrics/aggregation.py

@@ -0,0 +1,451 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Aggregation data structures and computations for metric results."""
+
+# Migrated from: services/evaluator/src/nmp/evaluator/app/metrics/aggregation.py
+
+import math
+from collections import OrderedDict, defaultdict
+from collections.abc import Mapping, Sequence
+from typing import Protocol, cast, runtime_checkable
+
+from nemo_platform.beta.evaluator.metrics.protocol import (
+    BooleanValue,
+    ContinuousScore,
+    DiscreteScore,
+    MetricOutput,
+    MetricOutputSpec,
+    MetricResult,
+)
+from nemo_platform.beta.evaluator.values.results import (
+    AggregatedMetricResult,
+    AggregateRangeScore,
+    AggregateRubricScore,
+    AggregateScore,
+    Histogram,
+    HistogramBin,
+    MetricScore,
+    Percentiles,
+    RubricScoreStat,
+    ScoreStats,
+)
+from nemo_platform.beta.evaluator.values.scores import RubricScore, Score
+
+
+def is_aggregateable_output_spec(output_spec: MetricOutputSpec) -> bool:
+    """Return whether an output should contribute aggregate statistics."""
+    return issubclass(output_spec.value_schema, (ContinuousScore, DiscreteScore, BooleanValue))
+
+
+@runtime_checkable
+class MetricWithScores(Protocol):
+    """Metric config protocol for metrics that carry rubric score definitions."""
+
+    @property
+    def scores(self) -> Sequence[Score]: ...
+
+
+def _coerce_aggregate_output(output: MetricOutput, output_spec: MetricOutputSpec) -> MetricScore | None:
+    """Convert one declared aggregateable metric output into MetricScore form."""
+    if not is_aggregateable_output_spec(output_spec):
+        return None
+    if (
+        issubclass(output_spec.value_schema, BooleanValue)
+        and isinstance(output.value, float)
+        and math.isnan(output.value)
+    ):
+        return MetricScore(name=output.name, value=output.value)
+    coerced = cast(ContinuousScore | DiscreteScore | BooleanValue, output_spec.coerce_output(output))
+    value = coerced.root
+    if isinstance(value, bool):
+        value = 1.0 if value else 0.0
+    return MetricScore(name=output.name, value=value)
+
+
+def _attach_rubric_stats(
+    score: MetricScore,
+    output_by_name: Mapping[str, MetricOutput],
+    rubric_definitions: Mapping[str, Sequence[RubricScoreStat]],
+) -> MetricScore:
+    """Attach per-row rubric bucket stats from companion label outputs."""
+    rubric_definition = rubric_definitions.get(score.name)
+    if not rubric_definition:
+        return score
+
+    label_output = output_by_name.get(f"{score.name}.label")
+    selected_label = label_output.value if label_output is not None else None
+    if isinstance(score.value, float) and math.isnan(score.value):
+        selected_label = None
+
+    rubric_distribution = [
+        RubricScoreStat(
+            label=rubric.label,
+            description=rubric.description,
+            value=rubric.value,
+            count=int(isinstance(selected_label, str) and selected_label == rubric.label),
+        )
+        for rubric in rubric_definition
+    ]
+    return MetricScore(
+        name=score.name,
+        value=score.value,
+        stats=ScoreStats(rubric_distribution=rubric_distribution),
+    )
+
+
+def _aggregateable_scores(
+    result: MetricResult,
+    output_specs: list[MetricOutputSpec],
+    rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None,
+) -> list[MetricScore]:
+    """Extract score-like outputs from a metric result using declared output specs."""
+    specs_by_name = {output_spec.name: output_spec for output_spec in output_specs}
+    output_by_name = {output.name: output for output in result.outputs}
+    scores: list[MetricScore] = []
+    for output in result.outputs:
+        output_spec = specs_by_name.get(output.name)
+        if output_spec is None:
+            continue
+        score = _coerce_aggregate_output(output, output_spec)
+        if score is not None:
+            score = _attach_rubric_stats(score, output_by_name, rubric_definitions or {})
+            scores.append(score)
+    return scores
+
+
+def add_corpus_scores(
+    aggregated_result: AggregatedMetricResult,
+    corpus_result: MetricResult,
+    output_specs: list[MetricOutputSpec],
+) -> None:
+    """Append corpus-level scores using aggregate-score schema fields.
+
+    Args:
+        aggregated_result: Aggregate result object to mutate.
+        corpus_result: Corpus-level metric output with one or more scores.
+
+    Returns:
+        ``None``. The ``aggregated_result`` object is updated in place.
+    """
+    for score in _aggregateable_scores(corpus_result, output_specs):
+        value = score.value
+        # Corpus-level metrics contribute one already-aggregated value, so
+        # expose them through the same aggregate schema with count=1.
+        corpus_score = AggregateRangeScore(
+            name=score.name,
+            count=1,
+            nan_count=0,
+            sum=value,
+            mean=value,
+            min=value,
+            max=value,
+            std_dev=0.0,
+            variance=0.0,
+            percentiles=Percentiles(
+                p10=value,
+                p20=value,
+                p30=value,
+                p40=value,
+                p50=value,
+                p60=value,
+                p70=value,
+                p80=value,
+                p90=value,
+                p100=value,
+            ),
+            histogram=Histogram(bins=[HistogramBin(lower_bound=value, upper_bound=value, count=1)]),
+        )
+        aggregated_result.scores.append(corpus_score)
+
+
+def _compute_percentile(sorted_values: list[float], percentile: float) -> float:
+    """Compute a percentile using linearly interpolated rank position.
+
+    The rank position is computed as ``(p/100) * (n + 1) - 1`` and then
+    interpolated between neighboring points when the position is fractional.
+
+    Args:
+        sorted_values: Score values sorted in ascending order.
+        percentile: Percentile in the inclusive range ``[0, 100]``.
+
+    Returns:
+        The interpolated percentile value, or ``0.0`` when no values exist.
+    """
+    if not sorted_values:
+        return 0.0
+    n = len(sorted_values)
+    pos = (percentile / 100.0) * (n + 1) - 1
+    if pos <= 0:
+        return sorted_values[0]
+    if pos >= n - 1:
+        return sorted_values[-1]
+    lower_idx = int(pos)
+    frac = pos - lower_idx
+    return sorted_values[lower_idx] + frac * (sorted_values[lower_idx + 1] - sorted_values[lower_idx])
+
+
+def _compute_percentiles(sorted_values: list[float]) -> Percentiles:
+    """Compute the fixed percentile set used by SDK aggregate output.
+
+    Args:
+        sorted_values: Score values sorted in ascending order.
+
+    Returns:
+        ``Percentiles`` containing p10 through p100.
+    """
+    return Percentiles(
+        p10=_compute_percentile(sorted_values, 10),
+        p20=_compute_percentile(sorted_values, 20),
+        p30=_compute_percentile(sorted_values, 30),
+        p40=_compute_percentile(sorted_values, 40),
+        p50=_compute_percentile(sorted_values, 50),
+        p60=_compute_percentile(sorted_values, 60),
+        p70=_compute_percentile(sorted_values, 70),
+        p80=_compute_percentile(sorted_values, 80),
+        p90=_compute_percentile(sorted_values, 90),
+        p100=_compute_percentile(sorted_values, 100),
+    )
+
+
+def _compute_histogram(values: list[float], num_bins: int = 10) -> Histogram:
+    """Build a fixed-width histogram for numeric score distribution.
+
+    The algorithm uses equal-width bins between the global minimum and maximum.
+    All bins except the last are half-open ``[lower, upper)``, while the final
+    bin is closed ``[lower, upper]`` so the maximum value is always counted.
+
+    Args:
+        values: Numeric score values to bucket.
+        num_bins: Number of equally sized bins.
+
+    Returns:
+        Histogram object containing ordered bin counts.
+    """
+    if not values:
+        return Histogram(bins=[])
+
+    min_val = min(values)
+    max_val = max(values)
+    if min_val == max_val:
+        return Histogram(bins=[HistogramBin(lower_bound=min_val, upper_bound=max_val, count=len(values))])
+
+    bin_width = (max_val - min_val) / num_bins
+    bins: list[HistogramBin] = []
+    for i in range(num_bins):
+        lower = min_val + i * bin_width
+        upper = min_val + (i + 1) * bin_width
+        if i == num_bins - 1:
+            count = sum(1 for v in values if lower <= v <= upper)
+        else:
+            count = sum(1 for v in values if lower <= v < upper)
+        bins.append(HistogramBin(lower_bound=lower, upper_bound=upper, count=count))
+    return Histogram(bins=bins)
+
+
+def aggregate_metrics(
+    items: list[MetricResult],
+    output_specs: list[MetricOutputSpec],
+    rubric_definitions: Mapping[str, Sequence[RubricScoreStat]] | None = None,
+) -> AggregatedMetricResult:
+    """Aggregate row-level metric results into range or rubric summaries.
+
+    This function performs two logical passes:
+    1. Incremental accumulation of count/sum/min/max and rubric occurrences.
+    2. Finalization of derived statistics (variance, percentiles, histogram,
+       and rubric mode category) once complete value sets are known.
+
+    Args:
+        items: Row-level metric results to aggregate.
+        output_specs: Declared outputs for the metric. Only continuous,
+            discrete, and boolean output values contribute to aggregate scores.
+        rubric_definitions: Optional rubric bucket definitions keyed by numeric
+            output name. This is aggregation metadata, not metric protocol
+            metadata, and is usually derived from LLM judge score config.
+
+    Returns:
+        Aggregate metric result with one aggregate score per score name.
+    """
+    score_values: dict[str, list[float]] = defaultdict(list)
+
+    aggregated_results: dict[str, MetricScore] = {}
+    rubric_distribution: dict[str, dict[str, dict]] = defaultdict(lambda: defaultdict(OrderedDict))
+    has_rubric: dict[str, bool] = {}
+
+    for item in items:
+        for score in _aggregateable_scores(item, output_specs, rubric_definitions):
+            if score.name not in aggregated_results:
+                # Keep one running accumulator per score name; distribution
+                # details are materialized in a second pass once all values exist.
+                aggregated_results[score.name] = MetricScore(
+                    name=score.name,
+                    value=0.0,
+                    stats=ScoreStats(
+                        count=0,
+                        sum=0,
+                        sum_squared=0,
+                        min=None,
+                        max=None,
+                        mean=0,
+                        variance=None,
+                        stddev=None,
+                        nan_count=0,
+                    ),
+                )
+                has_rubric[score.name] = bool(score.stats and score.stats.rubric_distribution)
+                if score.stats and score.stats.rubric_distribution:
+                    for rubric_stat in score.stats.rubric_distribution:
+                        rubric_distribution[score.name][rubric_stat.label] = {
+                            "label": rubric_stat.label,
+                            "value": rubric_stat.value,
+                            "count": 0,
+                        }
+
+            if score.stats and score.stats.rubric_distribution:
+                for rubric_stat in score.stats.rubric_distribution:
+                    if rubric_stat.count:
+                        rubric_distribution[score.name][rubric_stat.label]["count"] += 1
+
+            results = aggregated_results[score.name]
+            assert results.stats is not None
+
+            # int values can never be NaN in Python, so skipping int and
+            # float checks is sufficient.
+            # float('nan') is the only way to get a NaN in Python.
+            if isinstance(score.value, float) and math.isnan(score.value):
+                results.stats.nan_count = (results.stats.nan_count or 0) + 1
+                continue
+
+            score_values[score.name].append(score.value)
+            results.stats.count = (results.stats.count or 0) + 1
+            results.stats.sum = (results.stats.sum or 0) + score.value
+            results.stats.sum_squared = (results.stats.sum_squared or 0) + score.value**2
+            results.stats.mean = results.stats.sum / results.stats.count
+
+            if results.stats.min is None or score.value < results.stats.min:
+                results.stats.min = score.value
+            if results.stats.max is None or score.value > results.stats.max:
+                results.stats.max = score.value
+
+            results.value = results.stats.mean
+
+    for score_name, values in score_values.items():
+        if not values:
+            continue
+        results = aggregated_results[score_name]
+        assert results.stats is not None
+
+        n = len(values)
+        mean = results.stats.mean or 0
+        # Use population variance because these rows are the full evaluation set,
+        # not a sample intended to estimate a larger population.
+        variance = sum((v - mean) ** 2 for v in values) / n if n > 0 else 0
+        results.stats.variance = variance
+        results.stats.stddev = math.sqrt(variance)
+
+    aggregated_scores: list[AggregateScore] = []
+    for score_name, metric_score in aggregated_results.items():
+        stats = metric_score.stats
+        assert stats is not None
+
+        values = score_values[score_name]
+        base_name = metric_score.name
+        base_count = stats.count or 0
+        base_nan_count = stats.nan_count or 0
+        base_mean = stats.mean
+        base_sum = stats.sum if stats.sum is not None else (None if base_mean is None else (base_mean * base_count))
+        base_min = stats.min if stats.min is not None else base_mean
+        base_max = stats.max if stats.max is not None else base_mean
+        base_variance = stats.variance if stats.variance is not None else (None if base_mean is None else 0.0)
+        base_std_dev = stats.stddev if stats.stddev is not None else (None if base_mean is None else 0.0)
+
+        if base_count == 0:
+            base_sum = None
+            base_mean = None
+            base_min = None
+            base_max = None
+            base_variance = None
+            base_std_dev = None
+
+        if has_rubric.get(score_name):
+            rubric_dist = [
+                RubricScoreStat(label=r["label"], value=r["value"], count=r["count"])
+                for r in rubric_distribution[score_name].values()
+            ]
+            mode_category = None
+            if rubric_dist:
+                # Break ties deterministically so aggregate output is stable.
+                max_count = max(r.count for r in rubric_dist)
+                tied = sorted(r.label for r in rubric_dist if r.count == max_count)
+                mode_category = tied[0]
+
+            aggregated_scores.append(
+                AggregateRubricScore(
+                    name=base_name,
+                    count=base_count,
+                    nan_count=base_nan_count,
+                    sum=base_sum,
+                    mean=base_mean,
+                    min=base_min,
+                    max=base_max,
+                    variance=base_variance,
+                    std_dev=base_std_dev,
+                    rubric_distribution=rubric_dist,
+                    mode_category=mode_category,
+                )
+            )
+        else:
+            if values:
+                # Range scores get richer distribution metadata than rubric scores.
+                sorted_values = sorted(values)
+                percentiles = _compute_percentiles(sorted_values)
+                histogram = _compute_histogram(values)
+            else:
+                percentiles = None
+                histogram = Histogram(bins=[])
+
+            aggregated_scores.append(
+                AggregateRangeScore(
+                    name=base_name,
+                    count=base_count,
+                    nan_count=base_nan_count,
+                    sum=base_sum,
+                    mean=base_mean,
+                    min=base_min,
+                    max=base_max,
+                    variance=base_variance,
+                    std_dev=base_std_dev,
+                    percentiles=percentiles,
+                    histogram=histogram,
+                )
+            )
+
+    return AggregatedMetricResult(scores=aggregated_scores)
+
+
+def rubric_definitions_from_scores(scores: Sequence[Score]) -> dict[str, list[RubricScoreStat]]:
+    """Return declared rubric buckets keyed by score name."""
+    definitions: dict[str, list[RubricScoreStat]] = {}
+    for score in scores:
+        if not isinstance(score, RubricScore):
+            continue
+        definitions[score.name] = [
+            RubricScoreStat(
+                label=rubric.label,
+                description=rubric.description,
+                value=rubric.value,
+                count=0,
+            )
+            for rubric in score.rubric
+        ]
+    return definitions
+
+
+def rubric_definitions_from_metric(metric: object) -> dict[str, list[RubricScoreStat]]:
+    """Return rubric bucket definitions for metrics that carry score config."""
+    if not isinstance(metric, MetricWithScores):
+        return {}
+    scores = metric.scores
+    if not isinstance(scores, Sequence) or isinstance(scores, (str, bytes)):
+        return {}
+    return rubric_definitions_from_scores(scores)

nemo_platform/beta/evaluator/metrics/bleu.py

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""BLEU metric runtime implementation."""
+
+import sacrebleu
+from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_platform.beta.evaluator.metrics.template_rendering import (
+    TemplateSample,
+    build_template_context,
+    render_default_output_text_candidate_or_raise,
+    render_template_or_raise,
+    template_metric_repr,
+)
+from nemo_platform.beta.evaluator.values.metrics import BLEU
+
+__all__ = ["BLEUMetric"]
+
+
+class BLEUMetric(BLEU):
+    """BLEU metric for sentence- and corpus-level n-gram overlap.
+
+    Evaluator-driven runs render references from dataset fields exposed through
+    ``item``. Candidate text can come from explicit dataset fields or from
+    ``sample.output_text`` when the evaluator generates model outputs online.
+    """
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        """Return row-level outputs emitted by this metric."""
+        return [MetricOutputSpec.continuous_score("sentence")]
+
+    def corpus_output_spec(self) -> list[MetricOutputSpec]:
+        """Return corpus-level outputs emitted by this metric."""
+        return [MetricOutputSpec.continuous_score("corpus")]
+
+    def _render_references(self, item: dict, sample: TemplateSample) -> list[str]:
+        """Render all reference templates for one row."""
+        context = build_template_context(item, sample)
+        metric_repr = template_metric_repr(self)
+        references: list[str] = []
+        for index, reference in enumerate(self.references):
+            rendered_reference = render_template_or_raise(
+                template_name=f"references[{index}]",
+                template=reference,
+                context=context,
+                item=item,
+                sample=sample,
+                metric_repr=metric_repr,
+            )
+            if not isinstance(rendered_reference, str):
+                raise TypeError("The reference must be a string.")
+            references.append(rendered_reference)
+        return references
+
+    def _render_candidate(self, item: dict, sample: TemplateSample) -> str:
+        """Render the candidate text for one row."""
+        if self.candidate:
+            context = build_template_context(item, sample)
+            prediction = render_template_or_raise(
+                template_name="candidate",
+                template=self.candidate,
+                context=context,
+                item=item,
+                sample=sample,
+                metric_repr=template_metric_repr(self),
+            )
+        else:
+            prediction = render_default_output_text_candidate_or_raise(
+                sample=sample,
+                metric_name=self.__class__.__name__,
+            )
+
+        if not isinstance(prediction, str):
+            raise TypeError("The candidate must be a string.")
+        return prediction
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        """Compute the scores for the metric."""
+        item = input.row.data
+        sample = input.candidate
+        references = self._render_references(item, sample)
+        candidate = self._render_candidate(item, sample)
+        score = sacrebleu.sentence_bleu(candidate, references).score.real
+        return MetricResult(outputs=[MetricOutput(name="sentence", value=score)])
+
+    async def compute_corpus_scores(self, inputs: list[MetricInput]) -> MetricResult | None:
+        """Compute the corpus-level BLEU metric."""
+        item_sample_pairs = [(input.row.data, input.candidate) for input in inputs]
+        references_raw = [self._render_references(item, sample) for item, sample in item_sample_pairs]
+        # NOTE: because of the bug in sacrebleu, we need to flatten the references
+        references = [[reference_set[0] for reference_set in references_raw]]
+        candidates = [self._render_candidate(item, sample) for item, sample in item_sample_pairs]
+        score = sacrebleu.corpus_bleu(candidates, references)
+        return MetricResult(outputs=[MetricOutput(name="corpus", value=score.score.real)])

nemo_platform/beta/evaluator/metrics/exact_match.py

@@ -0,0 +1,52 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Exact-match metric runtime implementation."""
+
+from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr
+from nemo_platform.beta.evaluator.metrics.utils import normalize_text
+from nemo_platform.beta.evaluator.values.metrics import ExactMatch
+
+__all__ = ["ExactMatchMetric"]
+
+
+class ExactMatchMetric(ExactMatch):
+    """Exact-match metric runtime for evaluator-driven execution.
+
+    Evaluator-driven runs expose dataset values through ``item`` and generated
+    model outputs through ``sample.output_text`` for online execution.
+    """
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        """Return outputs emitted by this metric."""
+        return [MetricOutputSpec.continuous_score(self.type.value)]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        """Compute structured metric output for one item/sample pair.
+
+        The algorithm renders reference and candidate text from templates, then
+        normalizes both strings (case, punctuation, articles, and whitespace)
+        before equality comparison.
+
+        Args:
+            input: Original dataset row paired with candidate output.
+
+        Returns:
+            ``MetricResult`` with one exact-match score entry.
+
+        Raises:
+            TypeError: If rendered reference or candidate is not a string.
+            ValueError: If template rendering fails or ``candidate`` is omitted
+                and ``sample.output_text`` is missing.
+        """
+        ground_truth, prediction = render_reference_and_candidate(
+            metric_repr=template_metric_repr(self),
+            metric_name=self.__class__.__name__,
+            reference_template=self.reference,
+            candidate_template=self.candidate,
+            item=input.row.data,
+            sample=input.candidate,
+        )
+        score = int(normalize_text(prediction) == normalize_text(ground_truth))
+        return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)])

nemo_platform/beta/evaluator/metrics/f1.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""F1 metric runtime implementation."""
+
+import collections
+
+from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_platform.beta.evaluator.metrics.template_rendering import render_reference_and_candidate, template_metric_repr
+from nemo_platform.beta.evaluator.metrics.utils import normalize_text
+from nemo_platform.beta.evaluator.values.metrics import F1
+
+__all__ = ["F1Metric"]
+
+
+class F1Metric(F1):
+    """F1 metric for token-overlap similarity scoring."""
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        """Return outputs emitted by this metric."""
+        return [MetricOutputSpec.continuous_score(self.type.value)]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        """Compute the scores for the metric."""
+        ground_truth, prediction = render_reference_and_candidate(
+            metric_repr=template_metric_repr(self),
+            metric_name=self.__class__.__name__,
+            reference_template=self.reference,
+            candidate_template=self.candidate,
+            item=input.row.data,
+            sample=input.candidate,
+        )
+
+        prediction_tokens = normalize_text(prediction).split()
+        ground_truth_tokens = normalize_text(ground_truth).split()
+
+        # If either token list is empty, the F1 is 1.0 if they agree, 0.0 otherwise.
+        if len(ground_truth_tokens) == 0 or len(prediction_tokens) == 0:
+            score = float(ground_truth_tokens == prediction_tokens)
+            return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)])
+
+        common = collections.Counter(prediction_tokens) & collections.Counter(ground_truth_tokens)
+        num_same = sum(common.values())
+        if num_same == 0:
+            return MetricResult(outputs=[MetricOutput(name=self.type.value, value=0.0)])
+
+        precision = 1.0 * num_same / len(prediction_tokens)
+        recall = 1.0 * num_same / len(ground_truth_tokens)
+        score = (2 * precision * recall) / (precision + recall)
+        return MetricResult(outputs=[MetricOutput(name=self.type.value, value=score)])

nemo_platform/beta/evaluator/metrics/hooks.py

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Hook capability mixin for V2 SDK metrics."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Self
+
+import nemo_platform.beta.evaluator.inference as inference
+from pydantic import BaseModel, PrivateAttr
+
+
+class HooksBase(BaseModel):
+    """Reusable hook state and helpers for V2 metrics."""
+
+    _preprocess_hooks: list[inference.PreprocessRequest] = PrivateAttr(default_factory=list)
+    _postprocess_hooks: list[inference.PostprocessResponse] = PrivateAttr(default_factory=list)
+
+    def with_hooks(
+        self,
+        *,
+        preprocess: Sequence[inference.PreprocessRequest] | None = None,
+        postprocess: Sequence[inference.PostprocessResponse] | None = None,
+    ) -> Self:
+        """Attach preprocess and postprocess hooks to this metric instance."""
+        self._preprocess_hooks = list(preprocess or [])
+        self._postprocess_hooks = list(postprocess or [])
+        return self
+
+    def _apply_preprocess_hooks(self, request: dict, *, id: str | None = None) -> dict:
+        """Apply preprocess hooks in order or return the request unchanged."""
+        return inference.preprocess_request(request, hooks=self._preprocess_hooks, id=id)
+
+    def _apply_postprocess_hooks(self, response: dict, *, id: str | None = None) -> dict:
+        """Apply postprocess hooks in order or return the response unchanged."""
+        processed = response
+        for hook in self._postprocess_hooks:
+            processed = hook.postprocess(processed, id=id)
+        return processed