nemo-platform-plugin 0.1.0__py3-none-any.whl

nemo_platform/beta/evaluator/metrics/ragas/imports.py

@@ -0,0 +1,192 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Apply Git patch before any other imports that might use GitPython (like ragas)
+from nemo_platform.beta.evaluator.metrics.ragas.git_patch import apply_git_patch
+
+apply_git_patch()
+
+# ruff: noqa: E402
+"""Lazy imports for RAGAS library to avoid slow startup times.
+
+RAGAS and its dependencies (langchain, etc.) take 20-30 seconds to import.
+This module provides lazy loading so that the import cost is only paid when
+RAGAS metrics are actually used, not when the evaluator service starts.
+
+Usage:
+    # Instead of: from ragas import EvaluationDataset
+    # Use: EvaluationDataset = get_evaluation_dataset_class()
+"""
+import os
+from functools import cache
+
+# Disable RAGAS telemetry/tracking before importing ragas
+os.environ["RAGAS_DO_NOT_TRACK"] = "true"
+
+
+@cache
+def _load_ragas():
+    """Load all RAGAS modules. Cached so imports only happen once."""
+    import ragas
+    from ragas import EvaluationDataset, RunConfig, evaluate
+    from ragas.embeddings.base import LangchainEmbeddingsWrapper
+    from ragas.llms.base import LangchainLLMWrapper
+    from ragas.metrics import (
+        AgentGoalAccuracyWithoutReference,
+        AgentGoalAccuracyWithReference,
+        AnswerAccuracy,
+        AnswerCorrectness,
+        AnswerSimilarity,
+        ContextEntityRecall,
+        ContextPrecision,
+        ContextRecall,
+        ContextRelevance,
+        Faithfulness,
+        NoiseSensitivity,
+        ResponseGroundedness,
+        ResponseRelevancy,
+        ToolCallAccuracy,
+        TopicAdherenceScore,
+    )
+
+    # Return as a dict for easy access
+    return {
+        "ragas": ragas,
+        "EvaluationDataset": EvaluationDataset,
+        "RunConfig": RunConfig,
+        "evaluate": evaluate,
+        "LangchainEmbeddingsWrapper": LangchainEmbeddingsWrapper,
+        "LangchainLLMWrapper": LangchainLLMWrapper,
+        "AgentGoalAccuracyWithoutReference": AgentGoalAccuracyWithoutReference,
+        "AgentGoalAccuracyWithReference": AgentGoalAccuracyWithReference,
+        "AnswerAccuracy": AnswerAccuracy,
+        "AnswerCorrectness": AnswerCorrectness,
+        "AnswerSimilarity": AnswerSimilarity,
+        "ContextEntityRecall": ContextEntityRecall,
+        "ContextPrecision": ContextPrecision,
+        "ContextRecall": ContextRecall,
+        "ContextRelevance": ContextRelevance,
+        "Faithfulness": Faithfulness,
+        "NoiseSensitivity": NoiseSensitivity,
+        "ResponseGroundedness": ResponseGroundedness,
+        "ResponseRelevancy": ResponseRelevancy,
+        "ToolCallAccuracy": ToolCallAccuracy,
+        "TopicAdherenceScore": TopicAdherenceScore,
+    }
+
+
+# =============================================================================
+# Lazy accessor functions
+# =============================================================================
+
+
+def get_evaluation_dataset_class() -> type:
+    """Get the EvaluationDataset class."""
+    return _load_ragas()["EvaluationDataset"]
+
+
+def get_run_config_class() -> type:
+    """Get the RunConfig class."""
+    return _load_ragas()["RunConfig"]
+
+
+def get_evaluate_function():
+    """Get the evaluate function."""
+    return _load_ragas()["evaluate"]
+
+
+def get_langchain_embeddings_wrapper_class() -> type:
+    """Get the LangchainEmbeddingsWrapper class."""
+    return _load_ragas()["LangchainEmbeddingsWrapper"]
+
+
+def get_langchain_llm_wrapper_class() -> type:
+    """Get the LangchainLLMWrapper class."""
+    return _load_ragas()["LangchainLLMWrapper"]
+
+
+def get_topic_adherence_score_class() -> type:
+    """Get the TopicAdherenceScore metric class."""
+    return _load_ragas()["TopicAdherenceScore"]
+
+
+def get_tool_call_accuracy_class() -> type:
+    """Get the ToolCallAccuracy metric class."""
+    return _load_ragas()["ToolCallAccuracy"]
+
+
+def get_agent_goal_accuracy_with_reference_class() -> type:
+    """Get the AgentGoalAccuracyWithReference metric class."""
+    return _load_ragas()["AgentGoalAccuracyWithReference"]
+
+
+def get_agent_goal_accuracy_without_reference_class() -> type:
+    """Get the AgentGoalAccuracyWithoutReference metric class."""
+    return _load_ragas()["AgentGoalAccuracyWithoutReference"]
+
+
+def get_answer_accuracy_class() -> type:
+    """Get the AnswerAccuracy metric class."""
+    return _load_ragas()["AnswerAccuracy"]
+
+
+def get_context_relevance_class() -> type:
+    """Get the ContextRelevance metric class."""
+    return _load_ragas()["ContextRelevance"]
+
+
+def get_response_groundedness_class() -> type:
+    """Get the ResponseGroundedness metric class."""
+    return _load_ragas()["ResponseGroundedness"]
+
+
+def get_context_recall_class() -> type:
+    """Get the ContextRecall metric class."""
+    return _load_ragas()["ContextRecall"]
+
+
+def get_context_precision_class() -> type:
+    """Get the ContextPrecision metric class."""
+    return _load_ragas()["ContextPrecision"]
+
+
+def get_context_entity_recall_class() -> type:
+    """Get the ContextEntityRecall metric class."""
+    return _load_ragas()["ContextEntityRecall"]
+
+
+def get_response_relevancy_class() -> type:
+    """Get the ResponseRelevancy metric class."""
+    return _load_ragas()["ResponseRelevancy"]
+
+
+def get_faithfulness_class() -> type:
+    """Get the Faithfulness metric class."""
+    return _load_ragas()["Faithfulness"]
+
+
+def get_noise_sensitivity_class() -> type:
+    """Get the NoiseSensitivity metric class."""
+    return _load_ragas()["NoiseSensitivity"]
+
+
+__all__ = [
+    "get_evaluation_dataset_class",
+    "get_run_config_class",
+    "get_evaluate_function",
+    "get_langchain_embeddings_wrapper_class",
+    "get_langchain_llm_wrapper_class",
+    "get_topic_adherence_score_class",
+    "get_tool_call_accuracy_class",
+    "get_agent_goal_accuracy_with_reference_class",
+    "get_agent_goal_accuracy_without_reference_class",
+    "get_answer_accuracy_class",
+    "get_context_relevance_class",
+    "get_response_groundedness_class",
+    "get_context_recall_class",
+    "get_context_precision_class",
+    "get_context_entity_recall_class",
+    "get_response_relevancy_class",
+    "get_faithfulness_class",
+    "get_noise_sensitivity_class",
+]

nemo_platform/beta/evaluator/metrics/ragas/metrics.py

@@ -0,0 +1,183 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+# Type-only imports for static analysis (not imported at runtime)
+if TYPE_CHECKING:
+    from ragas import EvaluationDataset
+    from ragas.llms import LangchainLLMWrapper
+
+from nemo_platform.beta.evaluator.enums import MetricType
+from nemo_platform.beta.evaluator.metrics.ragas.base import BaseRAGASMetric
+
+# Lazy imports for RAGAS metric classes - these are getter functions that defer
+# the expensive RAGAS/langchain imports (~20-30s) until first use.
+from nemo_platform.beta.evaluator.metrics.ragas.imports import (
+    get_agent_goal_accuracy_with_reference_class,
+    get_agent_goal_accuracy_without_reference_class,
+    get_answer_accuracy_class,
+    get_context_entity_recall_class,
+    get_context_precision_class,
+    get_context_recall_class,
+    get_context_relevance_class,
+    get_faithfulness_class,
+    get_noise_sensitivity_class,
+    get_response_groundedness_class,
+    get_response_relevancy_class,
+    get_tool_call_accuracy_class,
+    get_topic_adherence_score_class,
+)
+from nemo_platform.beta.evaluator.values import metrics
+
+log = logging.getLogger(__name__)
+
+
+# Agentic metrics
+
+
+class TopicAdherenceMetric(metrics.TopicAdherence, BaseRAGASMetric):
+    """Metric for measuring topic adherence."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        metric_mode = self.metric_mode
+        TopicAdherenceScore = get_topic_adherence_score_class()
+        metric = TopicAdherenceScore(llm=llm_judge, mode=metric_mode)
+        return self._run_evaluate(data, [metric])
+
+
+class ToolCallAccuracyMetric(metrics.ToolCallAccuracy, BaseRAGASMetric):
+    """Metric for measuring tool call accuracy."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        ToolCallAccuracy = get_tool_call_accuracy_class()
+        metric = ToolCallAccuracy()
+        return self._run_evaluate(data, [metric])
+
+
+class AgentGoalAccuracyMetric(metrics.AgentGoalAccuracy, BaseRAGASMetric):
+    """Metric for measuring agent goal accuracy."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        # Choose between with/without reference based on params
+        if self.use_reference:
+            metric_class = get_agent_goal_accuracy_with_reference_class()
+        else:
+            metric_class = get_agent_goal_accuracy_without_reference_class()
+        metric = metric_class(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+# Nvidia metrics
+
+
+class AnswerAccuracyMetric(metrics.AnswerAccuracy, BaseRAGASMetric):
+    """Metric for measuring answer accuracy."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        AnswerAccuracy = get_answer_accuracy_class()
+        metric = AnswerAccuracy(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+class ContextRelevanceMetric(metrics.ContextRelevance, BaseRAGASMetric):
+    """Metric for measuring context relevance."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        ContextRelevance = get_context_relevance_class()
+        metric = ContextRelevance(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+class ResponseGroundednessMetric(metrics.ResponseGroundedness, BaseRAGASMetric):
+    """Metric for measuring response groundedness."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        ResponseGroundedness = get_response_groundedness_class()
+        metric = ResponseGroundedness(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+# RAG Metrics
+
+
+class ContextRecallMetric(metrics.ContextRecall, BaseRAGASMetric):
+    """Metric for measuring context recall."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        ContextRecall = get_context_recall_class()
+        metric = ContextRecall(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+class ContextPrecisionMetric(metrics.ContextPrecision, BaseRAGASMetric):
+    """Metric for measuring context precision."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        ContextPrecision = get_context_precision_class()
+        metric = ContextPrecision(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+class ContextEntityRecallMetric(metrics.ContextEntityRecall, BaseRAGASMetric):
+    """Metric for measuring context entity recall."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        ContextEntityRecall = get_context_entity_recall_class()
+        metric = ContextEntityRecall(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+class ResponseRelevancyMetric(metrics.ResponseRelevancy, BaseRAGASMetric):
+    """Metric for measuring response relevancy."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        embeddings_client = self._get_embeddings_client()
+        # Strictness defines number of parallel questions generated. NIM can only generate 1.
+        # Having a configurable parameter allows computation with non-NIM judges.
+        ResponseRelevancy = get_response_relevancy_class()
+        metric = ResponseRelevancy(llm=llm_judge, embeddings=embeddings_client, strictness=self.strictness)
+        return self._run_evaluate(data, [metric])
+
+
+class FaithfulnessMetric(metrics.Faithfulness, BaseRAGASMetric):
+    """Metric for measuring faithfulness."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        Faithfulness = get_faithfulness_class()
+        metric = Faithfulness(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+class NoiseSensitivityMetric(metrics.NoiseSensitivity, BaseRAGASMetric):
+    """Metric for measuring noise sensitivity."""
+
+    def _metric(self, data: EvaluationDataset, llm_judge: LangchainLLMWrapper | None) -> dict[str, float]:
+        NoiseSensitivity = get_noise_sensitivity_class()
+        metric = NoiseSensitivity(llm=llm_judge)
+        return self._run_evaluate(data, [metric])
+
+
+# List of all RAGAS metrics
+ragas_metrics = [
+    AgentGoalAccuracyMetric,
+    AnswerAccuracyMetric,
+    ContextEntityRecallMetric,
+    ContextPrecisionMetric,
+    ContextRecallMetric,
+    ContextRelevanceMetric,
+    FaithfulnessMetric,
+    NoiseSensitivityMetric,
+    ResponseGroundednessMetric,
+    ResponseRelevancyMetric,
+    ToolCallAccuracyMetric,
+    TopicAdherenceMetric,
+]
+
+# Map of Metric enum values to class for all RAGAS metrics
+RAGAS_METRIC_CLASSES: dict[MetricType, type[BaseRAGASMetric]] = {
+    metric.model_fields["type"].default: metric for metric in ragas_metrics
+}

nemo_platform/beta/evaluator/metrics/remote.py

@@ -0,0 +1,250 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Remote metric runtime implementation."""
+
+import logging
+import os
+from abc import ABC, abstractmethod
+from collections.abc import Awaitable, Callable
+from typing import Any, cast
+
+import httpx
+from httpx import Timeout
+from jsonpath_ng import parse as jsonpath_parse
+from jsonpath_ng.exceptions import JsonPathParserError
+from nemo_platform.beta.evaluator.inference import requests_log_var
+from nemo_platform.beta.evaluator.metrics.protocol import MetricInput, MetricOutput, MetricOutputSpec, MetricResult
+from nemo_platform.beta.evaluator.metrics.template_rendering import (
+    TemplateSample,
+    build_template_context,
+    render_template_or_raise,
+    template_metric_repr,
+)
+from nemo_platform.beta.evaluator.resilience.api import run_with_resilience
+from nemo_platform.beta.evaluator.resilience.classifier import endpoint_identity
+from nemo_platform.beta.evaluator.values.common import SecretRef
+from nemo_platform.beta.evaluator.values.metrics import NemoAgentToolkitRemote, Remote, _RemoteBase
+from nemo_platform.beta.evaluator.values.scores import RemoteScore
+from pydantic import Field, SecretStr, field_validator
+
+__all__ = ["RemoteMetric", "NemoAgentToolkitRemoteMetric", "SecretRef"]
+
+_logger = logging.getLogger(__name__)
+
+
+async def _post_to_remote_endpoint(
+    url: str,
+    payload: dict[str, Any],
+    api_key: str | None = None,
+    timeout: float = 30.0,
+    max_retries: int = 0,
+    log: logging.Logger = _logger,
+) -> dict[str, Any]:
+    """Make a POST request to the remote endpoint."""
+    headers = {"Content-Type": "application/json"}
+    if api_key:
+        headers["Authorization"] = f"Bearer {api_key}"
+
+    log.debug("Calling remote metric url: %r", url)
+
+    endpoint_key = endpoint_identity(url, model_id="remote-metric", auth_identity=api_key)
+    max_attempts = max(1, max_retries + 1)
+
+    async with httpx.AsyncClient(timeout=Timeout(timeout)) as client:
+
+        async def _invoke_post() -> dict[str, Any]:
+            response = await client.post(url, json=payload, headers=headers)
+            response.raise_for_status()
+            return response.json()
+
+        try:
+            return await run_with_resilience(endpoint_key, _invoke_post, max_attempts=max_attempts)
+        except Exception:
+            log.exception("Remote metric request failed after %d attempts", max_attempts)
+            raise
+
+
+class _RemoteMetricBase(_RemoteBase, ABC):
+    """Shared runtime lifecycle for metrics backed by remote HTTP endpoints."""
+
+    metric_threshold_score: str | None = Field(default=None)
+    _api_key: SecretStr | None = None
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize private API key from env when configured."""
+        if self.api_key_secret:
+            env_var_name = self.api_key_secret.root.replace("-", "_")
+            self._set_api_key(os.getenv(env_var_name))
+        return super().model_post_init(__context)
+
+    def _set_api_key(self, api_key: str | None) -> None:
+        """Store API key as SecretStr private attribute."""
+        self._api_key = SecretStr(api_key) if api_key else None
+
+    def _get_api_key(self) -> str | None:
+        """Read API key value from SecretStr private attribute."""
+        return self._api_key.get_secret_value() if self._api_key else None
+
+    def _append_request_log(self, *, payload: dict[str, Any], response: dict[str, Any]) -> None:
+        """Append the remote request and response payloads to the shared request log."""
+        requests_log = requests_log_var.get([])
+        requests_log.append({"request": payload, "response": response})
+
+    async def _post_payload(self, payload: dict[str, Any]) -> dict[str, Any]:
+        """Send one rendered payload to the remote endpoint and log the result."""
+        result_data = await _post_to_remote_endpoint(
+            url=self.url,
+            payload=payload,
+            api_key=self._get_api_key(),
+            timeout=self.timeout_seconds,
+            max_retries=self.max_retries,
+            log=_logger,
+        )
+        self._append_request_log(payload=payload, response=result_data)
+        return result_data
+
+    async def resolve_secrets(self, secret_resolver: Callable[[str], Awaitable[str | None]]) -> None:
+        """Resolve API key secret if configured. Must be called before live evaluation."""
+        if self.api_key_secret:
+            secret_name = self.api_key_secret.root
+            resolved_key = await secret_resolver(secret_name)
+            if resolved_key:
+                self._set_api_key(resolved_key)
+            elif not self._get_api_key():
+                raise ValueError(
+                    f"Missing secret '{secret_name}' for API key authentication with remote metric server."
+                )
+
+    def secrets(self) -> dict[str, SecretRef]:
+        """Return secret env mappings required by this metric."""
+        if self.api_key_secret:
+            env_var_name = self.api_key_secret.root.replace("-", "_")
+            return {env_var_name: self.api_key_secret}
+        return {}
+
+    def _select_metric_score(self, metric_result: MetricResult) -> float:
+        """Select the default score value for one-row metric results."""
+        return float(metric_result.outputs[0].value)
+
+    @abstractmethod
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        """Compute structured score output for one item/sample pair."""
+        ...
+
+
+class RemoteMetric(Remote, _RemoteMetricBase):
+    """A metric that computes scores via a remote endpoint."""
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        """Return outputs emitted by this metric."""
+        return [MetricOutputSpec.continuous_score(score.name) for score in self.scores]
+
+    @field_validator("scores")
+    @classmethod
+    def _validate_scores(cls, scores: list[RemoteScore]) -> list[RemoteScore]:
+        for score in scores:
+            try:
+                jsonpath_parse(score.parser.json_path)
+            except JsonPathParserError as error:
+                raise ValueError(
+                    f"Score '{score.name}' has invalid JSONPath expression '{score.parser.json_path}': {error}"
+                ) from error
+        return scores
+
+    def _select_metric_score(self, metric_result: MetricResult) -> float:
+        """Select the score value used for single-score consumers."""
+        if self.metric_threshold_score:
+            output_names = [output.name for output in metric_result.outputs]
+            if self.metric_threshold_score not in output_names:
+                raise ValueError(
+                    f"Score name '{self.metric_threshold_score}' not found in remote metric response. "
+                    f"Available scores: {output_names}"
+                )
+            return float(
+                next(output for output in metric_result.outputs if output.name == self.metric_threshold_score).value
+            )
+
+        if len(metric_result.outputs) == 1:
+            return float(metric_result.outputs[0].value)
+
+        raise ValueError(
+            f"Remote metric returned multiple scores {[output.name for output in metric_result.outputs]}. "
+            "Please set metric_threshold_score to specify which score to use."
+        )
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        """Compute structured score output via the remote endpoint."""
+        item = input.row.data
+        sample: TemplateSample = input.candidate
+        context = build_template_context(item, sample)
+        rendered_args = render_template_or_raise(
+            template_name="body",
+            template=self.body,
+            context=context,
+            item=item,
+            sample=sample,
+            metric_repr=template_metric_repr(self),
+        )
+        payload = cast(dict[str, Any], rendered_args) if isinstance(rendered_args, dict) else {"args": rendered_args}
+        result_data = await self._post_payload(payload)
+
+        try:
+            _logger.debug("Remote metric result received for payload: %r", payload)
+            outputs: list[MetricOutput] = []
+            for score_config in self.scores:
+                jsonpath_expr = jsonpath_parse(score_config.parser.json_path)
+                matches = jsonpath_expr.find(result_data)
+                if not matches:
+                    _logger.warning(
+                        "Could not extract score %r from path: %r, setting to NaN",
+                        score_config.name,
+                        score_config.parser.json_path,
+                    )
+                    outputs.append(MetricOutput(name=score_config.name, value=float("nan")))
+                else:
+                    score_value = matches[0].value
+                    outputs.append(MetricOutput(name=score_config.name, value=float(score_value)))
+
+            return MetricResult(outputs=outputs)
+        except Exception:
+            _logger.exception("Error validating remote metric response")
+            raise
+
+
+class NemoAgentToolkitRemoteMetric(NemoAgentToolkitRemote, _RemoteMetricBase):
+    """A remote metric that interfaces with NeMo Agent Toolkit evaluators."""
+
+    _RESULT_SCORE_JSONPATH = jsonpath_parse("$.result.score")
+
+    def output_spec(self) -> list[MetricOutputSpec]:
+        """Return outputs emitted by this metric."""
+        return [MetricOutputSpec.continuous_score(self.evaluator_name)]
+
+    async def compute_scores(self, input: MetricInput) -> MetricResult:
+        """Compute structured score output via the NeMo Agent Toolkit evaluator endpoint."""
+        item = input.row.data
+        sample: TemplateSample = input.candidate
+        context = build_template_context(item, sample)
+        rendered_item = render_template_or_raise(
+            template_name="body.item",
+            template="{{ item | tojson }}",
+            context=context,
+            item=item,
+            sample=sample,
+            metric_repr=template_metric_repr(self),
+        )
+        payload = {
+            "evaluator_name": self.evaluator_name,
+            "item": rendered_item,
+        }
+        result_data = await self._post_payload(payload)
+
+        matches = self._RESULT_SCORE_JSONPATH.find(result_data)
+        if not matches:
+            _logger.warning("Could not extract NeMo Agent Toolkit score from response, setting to NaN")
+            score = float("nan")
+        else:
+            score = float(matches[0].value)
+
+        return MetricResult(outputs=[MetricOutput(name=self.evaluator_name, value=score)])