ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/metrics/answer_similarity/answer_similarity_metric.py

@@ -0,0 +1,219 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from lazy_imports import LazyModule, load
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.base_classes import Error
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.llm_judge import LLMJudge
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
+from ibm_watsonx_gov.utils.validation_util import (validate_llm_as_judge,
+                                                   validate_output,
+                                                   validate_reference,
+                                                   validate_unitxt_method)
+
+unitxt_provider = LazyModule(
+    "from ibm_watsonx_gov.providers.unitxt_provider import UnitxtProvider",
+    name="lazy_unitxt_provider"
+)
+load(unitxt_provider)
+UnitxtProvider = unitxt_provider.UnitxtProvider
+
+logger = GovSDKLogger.get_logger(__name__)
+UNITXT_METRIC_NAME = "answer_correctness"
+
+unitxt_methods = ["token_recall",
+                  "bert_score_recall",
+                  "sentence_bert_mini_lm",
+                  "llm_as_judge",
+                  ]
+
+
+class AnswerSimilarityMetric(GenAIMetric):
+    """
+    Defines the Answer Similarity metric class.
+
+    The Answer Similarity metric measures the similarity between the generated text and the ground truth.
+    It can be computed using the below methods:
+
+    1. token_recall (default)
+    2. bert_score_recall
+    3. sentence_bert_mini_lm
+    4. llm_as_judge
+
+    Examples:
+        1. Create Answer Similarity metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = AnswerSimilarityMetric()
+                result = MetricsEvaluator().evaluate(data={"generated_text": "...", "ground_truth": "..."}, 
+                                                    metrics=[metric])
+
+        2. Create Answer Similarity metric with a custom threshold and method.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                method = "sentence_bert_mini_lm"
+                metric = AnswerSimilarityMetric(method=method, threshold=threshold)
+
+        3. Create Answer Similarity metric with llm_as_judge method.
+            .. code-block:: python
+
+                # Define LLM Judge using watsonx.ai
+                # To use other frameworks and models as llm_judge, see :module:`ibm_watsonx_gov.entities.foundation_model`
+                llm_judge = LLMJudge(model=WxAIFoundationModel(
+                                            model_id="ibm/granite-3-3-8b-instruct",
+                                            project_id="<PROJECT_ID>"
+                                    ))
+                metric = AnswerSimilarityMetric(llm_judge=llm_judge)
+    """
+    name: Annotated[Literal["answer_similarity"],
+                    Field(title="Name",
+                          description="The answer similarity metric name.",
+                          default="answer_similarity", frozen=True)]
+    display_name: Annotated[Literal["Answer Similarity"],
+                            Field(title="Display Name",
+                                  description="The answer similarity metric display name.",
+                                  default="Answer Similarity", frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=[TaskType.RAG, TaskType.QA])]
+    is_reference_free: Annotated[bool,
+                                 Field(title="Is Reference free",
+                                       description="The flag to indicate whether this metric needs a reference for computation. This metric needs reference value to compute.",
+                                       default=False, frozen=True)]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="lower_limit", value=0.7)])]
+    method: Annotated[Literal["token_recall", "bert_score_recall", "sentence_bert_mini_lm", "llm_as_judge"],
+                      Field(title="Method",
+                            description="The method used to compute the metric. This field is optional and when `llm_judge` is provided, the method would be set to `llm_as_judge`.",
+                            default="token_recall")]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.ANSWER_QUALITY, frozen=True)]
+    llm_judge: Annotated[LLMJudge | None,
+                         Field(title="LLM Judge",
+                               description="The LLM judge used to compute the metric.",
+                               default=None)]
+
+    @model_validator(mode="after")
+    def set_llm_judge_default_method(self) -> Self:
+        # If llm_judge is set, set the method to llm_as_judge
+        if self.llm_judge:
+            self.method = "llm_as_judge"
+        return self
+
+    def evaluate(self,
+                 data: pd.DataFrame,
+                 configuration: GenAIConfiguration | AgenticAIConfiguration,
+                 **kwargs) -> AggregateMetricResult:
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )
+
+    async def evaluate_async(self, data: pd.DataFrame,
+                             configuration: GenAIConfiguration | AgenticAIConfiguration,
+                             **kwargs) -> AggregateMetricResult:
+
+        data_cols = data.columns.to_list()
+
+        try:
+            validate_output(data_cols, configuration)
+            validate_reference(data_cols, configuration)
+            validate_unitxt_method(self.name, self.method, unitxt_methods)
+            validate_llm_as_judge(self.name, self.method,
+                                  self.llm_judge, configuration.llm_judge)
+        except ValueError as ve:
+            if kwargs.get("ignore_validation_errors"):
+                message = f"Skipping '{self.name}' computation because the validation failed. Details: {str(ve)}"
+                logger.warning(message)
+                return
+            raise ve
+
+        # Separate the data into a dataframe with no None values and a dataframe with None values
+        required_fields = configuration.output_fields + configuration.reference_fields
+        mask_has_none = data[required_fields].isna().any(axis=1)
+        df_with_none = data[mask_has_none]
+        df_without_none = data[mask_has_none == False]
+
+        # Compute the metrics only for the dataframe with no None values
+        aggregated_metric_result = None
+        if not df_without_none.empty:
+            provider = UnitxtProvider(configuration=configuration,
+                                      metric_name=self.name,
+                                      metric_display_name=self.display_name,
+                                      metric_method=self.method,
+                                      metric_prefix="metrics.rag.external_rag",
+                                      metric_alias=UNITXT_METRIC_NAME,
+                                      metric_group=self.group,
+                                      llm_judge=self.llm_judge,
+                                      thresholds=self.thresholds,
+                                      **kwargs)
+
+            aggregated_metric_result = await provider.evaluate_async(data=df_without_none)
+
+        # Update the metric result with record level metrics results for the records with missing values
+        if not df_with_none.empty:
+            # Create None results for records with missing values
+            none_results = []
+            for _, row in df_with_none.iterrows():
+                record_result = RecordMetricResult(
+                    name=self.name,
+                    display_name=self.display_name,
+                    method=self.method,
+                    group=self.group,
+                    value=None,
+                    record_id=row[configuration.record_id_field],
+                    thresholds=self.thresholds,
+                    errors=[Error(
+                        code="BAD_REQUEST", message_en="The value of required fields output or reference is None.")]
+                )
+                none_results.append(record_result)
+
+            # Merge the results
+            if aggregated_metric_result:
+                all_record_results = aggregated_metric_result.record_level_metrics + none_results
+                aggregated_metric_result.record_level_metrics = all_record_results
+                aggregated_metric_result.total_records = len(
+                    all_record_results)
+            else:
+                aggregated_metric_result = AggregateMetricResult(
+                    name=self.name,
+                    display_name=self.display_name,
+                    method=self.method,
+                    group=self.group,
+                    value=None,
+                    total_records=len(none_results),
+                    record_level_metrics=none_results,
+                    min=None,
+                    max=None,
+                    mean=None,
+                    thresholds=self.thresholds
+                )
+
+        return aggregated_metric_result

ibm_watsonx_gov/metrics/average_precision/average_precision_decorator.py

@@ -0,0 +1,62 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.average_precision.average_precision_metric import \
+    AveragePrecisionMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.context_relevance.context_relevance_metric import \
+    ContextRelevanceMetric
+
+
+class AveragePrecisionDecorator(BaseMetricDecorator):
+    def evaluate_average_precision(self,
+                                   func: Optional[Callable] = None,
+                                   *,
+                                   configuration: Optional[AgenticAIConfiguration] = None,
+                                   metrics: list[GenAIMetric] = []
+                                   ) -> dict:
+        """
+        An evaluation decorator for computing average precision metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_average_precision, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [AveragePrecisionMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(AveragePrecisionMetric, ContextRelevanceMetric))
+
+                metric_inputs = [EvaluatorFields.INPUT_FIELDS]
+                metric_outputs = [EvaluatorFields.CONTEXT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating average precision metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/average_precision/average_precision_metric.py

@@ -0,0 +1,174 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Any, Literal
+
+import pandas as pd
+from pydantic import Field, TypeAdapter, field_validator
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.metrics.context_relevance.context_relevance_metric import (
+    CONTEXT_RELEVANCE, ContextRelevanceMetric, ContextRelevanceResult)
+
+AVERAGE_PRECISION = "average_precision"
+AVERAGE_PRECISION_DISPLAY_NAME = "Average Precision"
+
+
+class AveragePrecisionResult(RecordMetricResult):
+    name: str = AVERAGE_PRECISION
+    display_name: str = AVERAGE_PRECISION_DISPLAY_NAME
+    group: MetricGroup = MetricGroup.RETRIEVAL_QUALITY
+
+
+class AveragePrecisionMetric(GenAIMetric):
+    """
+    Defines the Average Precision metric class.
+
+    The Average Precision metric measures the quality of how a retrieval system ranks relevant contexts.
+    The Context Relevance metric is computed as a pre requisite to compute this metric.
+
+    Examples:
+        1. Create Average Precision metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = AveragePrecisionMetric()
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", "context": "..."}, 
+                                                    metrics=[metric])
+                # A list of contexts can also be passed as shown below
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", "context": ["...", "..."]}, 
+                                                    metrics=[metric])
+
+        2. Create Average Precision metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = AveragePrecisionMetric(method=method, threshold=threshold)
+
+        3. Create Average Precision metric with llm_as_judge method.
+            .. code-block:: python
+
+                # Define LLM Judge using watsonx.ai
+                # To use other frameworks and models as llm_judge, see :module:`ibm_watsonx_gov.entities.foundation_model`
+                llm_judge = LLMJudge(model=WxAIFoundationModel(
+                                            model_id="ibm/granite-3-3-8b-instruct",
+                                            project_id="<PROJECT_ID>"
+                                    ))
+                cr_metric = ContextRelevanceMetric(llm_judge=llm_judge)
+                ap_metric = AveragePrecisionMetric()
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", "context": ["...", "..."]}, 
+                                                    metrics=[cr_metric, ap_metric])
+    """
+    name: Annotated[Literal["average_precision"],
+                    Field(title="Name",
+                          description="The average precision metric name.",
+                          default=AVERAGE_PRECISION, frozen=True)]
+    display_name: Annotated[Literal["Average Precision"],
+                            Field(title="Display Name",
+                                  description="The average precision metric display name.",
+                                  default=AVERAGE_PRECISION_DISPLAY_NAME, frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=[TaskType.RAG])]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="lower_limit", value=0.7)])]
+    metric_dependencies: Annotated[list[GenAIMetric],
+                                   Field(title="Metric dependencies",
+                                         description="The list of metric dependencies",
+                                         default=[ContextRelevanceMetric()])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.RETRIEVAL_QUALITY, frozen=True)]
+
+    @field_validator("metric_dependencies", mode="before")
+    @classmethod
+    def metric_dependencies_validator(cls, value: Any):
+        if value:
+            value = [TypeAdapter(Annotated[ContextRelevanceMetric, Field(
+                discriminator="name")]).validate_python(
+                m) for m in value]
+        return value
+
+    def evaluate(
+            self,
+            data: pd.DataFrame,
+            configuration: GenAIConfiguration | AgenticAIConfiguration,
+            metrics_result: list[AggregateMetricResult],
+            **kwargs,
+    ) -> AggregateMetricResult:
+        record_level_metrics = []
+        scores = []
+
+        context_relevance_result: list[ContextRelevanceResult] = next(
+            (metric_result.record_level_metrics for metric_result in metrics_result if metric_result.name == CONTEXT_RELEVANCE), None)
+
+        if context_relevance_result is None:
+            raise Exception(
+                f"Failed to evaluate {self.name} metric. Missing context relevance metric result")
+
+        for relevance_result in context_relevance_result:
+            score = self.__compute(
+                relevance_scores=relevance_result.additional_info.get(
+                    "contexts_values", []),
+                threshold=self.thresholds[0].value,
+            )
+            scores.append(score)
+            record_level_metrics.append(
+                AveragePrecisionResult(
+                    method="",
+                    provider="",
+                    record_id=relevance_result.record_id,
+                    value=score,
+                    thresholds=self.thresholds
+                )
+            )
+
+        mean = sum(scores) / len(scores)
+        aggregate_metric_score = AggregateMetricResult(
+            name=self.name,
+            display_name=self.display_name,
+            method="",
+            provider="",
+            group=self.group,
+            min=min(scores),
+            max=max(scores),
+            mean=mean,
+            value=mean,
+            total_records=len(record_level_metrics),
+            record_level_metrics=record_level_metrics,
+            thresholds=self.thresholds
+        )
+
+        return aggregate_metric_score
+
+    def __compute(self, relevance_scores: list[float], threshold: float) -> float:
+        relevancy_at_k = []
+        for i, score in enumerate(relevance_scores):
+            if score >= threshold:
+                relevancy_at_k.append(i + 1)
+        total_relevant_items = len(relevancy_at_k)
+        if total_relevant_items == 0:
+            return 0
+        precision_sum = 0
+        relevant_rank = 0
+        for k in relevancy_at_k:
+            relevant_rank += 1
+            precision_at_k = relevant_rank / k
+            precision_sum += precision_at_k
+        average_precision = precision_sum / total_relevant_items
+        average_precision_rounded = round(average_precision, 1)
+        return average_precision_rounded

ibm_watsonx_gov/metrics/base_metric_decorator.py

@@ -0,0 +1,193 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from json import dumps
+from threading import Lock
+from typing import Any, Callable, Set
+
+from ibm_watsonx_gov.clients.api_client import APIClient
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.agentic_app import MetricsConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields, MetricGroup
+from ibm_watsonx_gov.entities.evaluation_result import AgentMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.evaluators.impl.evaluate_metrics_impl import \
+    _evaluate_metrics_async
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.python_utils import get_argument_value
+
+try:
+    from ibm_agent_analytics.instrumentation.utils import (
+        AIEventRecorder, get_current_trace_id, record_span_attributes)
+except:
+    pass
+
+
+class BaseMetricDecorator():
+    """
+    Base class for all metric decorators
+    """
+
+    def __init__(self, api_client: APIClient = None, configuration: AgenticAIConfiguration = None,
+                 compute_real_time: bool = True, metric_results: list[AgentMetricResult] = [],
+                 execution_counts: dict[str, dict[str, int]] = {},
+                 nodes_being_run: dict[str, Set[str]] = {}, lock: Lock = None):
+        self.api_client = api_client
+        self.configuration = configuration
+        self.compute_real_time = compute_real_time
+        self.metric_results = metric_results
+        self.execution_counts = execution_counts
+        self.nodes_being_run = nodes_being_run
+        self.lock = lock
+
+    def validate(self, *, func: Callable, metrics: list[GenAIMetric], valid_metric_types: tuple[Any]):
+        if not metrics:
+            raise ValueError(
+                "The 'metrics' argument can not be empty.")
+
+        invalid_metrics = [metric.name for metric in metrics if not isinstance(
+            metric, valid_metric_types)]
+        if len(invalid_metrics):
+            raise ValueError(
+                f"The evaluator '{func.__name__}' is not applicable for "
+                f"computing the metrics: {', '.join(invalid_metrics)}")
+
+    def compute_helper(self, *, func: Callable,
+                       args: tuple,
+                       kwargs: dict[str, Any],
+                       configuration: AgenticAIConfiguration,
+                       metrics: list[GenAIMetric],
+                       metric_inputs: list[EvaluatorFields],
+                       metric_outputs: list[EvaluatorFields],
+                       metric_references: list[EvaluatorFields] = [],
+                       metric_groups: list[MetricGroup] = []) -> dict:
+        """
+        Helper method for computing metrics.
+
+        Does the following:
+            1. Computes node latency metric, and appends the result to the :py:attr:`AgenticEvaluation.metric_results` attribute.
+            2. Calls the original node.
+            3. Computes the list of metrics given, and appends the result to the :py:attr:`AgenticEvaluation.metric_results` attribute.
+            4. Returns the result of the original node without any changes.
+
+        Args:
+            func (Callable): The node on which the metric is to be computed
+            args (tuple): The tuple of positional arguments passed to the node
+            kwargs (dict[str, Any]): The dictionary of keyword arguments passed to the node
+            configuration (AgenticAIConfiguration): The node specific configuration
+            metrics (list[GenAIMetric]): The list of metrics to compute.
+            metric_inputs (list[EvaluatorFields]): The list of inputs for the metric.
+            metric_outputs (list[EvaluatorFields]): The list of outputs for the metric.
+            metric_references (list[EvaluatorFields], optional): The optional list of references for the metric. Defaults to [].
+
+        Raises:
+            ValueError: If the record id field is missing from the node inputs.
+
+        Returns:
+            dict: The result of the wrapped node.
+        """
+
+        get_arg_value = partial(
+            get_argument_value, func=func, args=args, kwargs=kwargs)
+
+        defaults = metric_inputs + metric_outputs + metric_references
+        _configuration = AgenticAIConfiguration.create_configuration(app_config=self.configuration,
+                                                                     method_config=configuration,
+                                                                     defaults=defaults)
+        _configuration.record_id_field = _configuration.message_id_field
+
+        _data = {}
+        # Add record id to the data
+        _field = getattr(_configuration, EvaluatorFields.MESSAGE_ID_FIELD.value,
+                         EvaluatorFields.get_default_fields_mapping()[EvaluatorFields.MESSAGE_ID_FIELD])
+
+        try:
+            _message_id_value = get_arg_value(
+                param_name=_field) or get_current_trace_id()
+        except ValueError:
+            _message_id_value = get_current_trace_id()
+
+        if _message_id_value is None:
+            raise ValueError(
+                f"The {_field} is required for evaluation. Please add it while invoking the application.")
+
+        _data[_field] = _message_id_value
+
+        if _message_id_value not in self.nodes_being_run:
+            self.nodes_being_run[_message_id_value] = set()
+        if _message_id_value not in self.execution_counts:
+            self.execution_counts[_message_id_value] = dict()
+
+        if func.__name__ not in self.nodes_being_run[_message_id_value]:
+            self.nodes_being_run[_message_id_value].add(func.__name__)
+            self.execution_counts[_message_id_value][func.__name__] = self.execution_counts[_message_id_value].get(
+                func.__name__, 0) + 1
+
+        original_result = func(*args, **kwargs)
+
+        metric_result = []
+        if self.compute_real_time:
+            for field in metric_inputs + metric_references:
+                _field = getattr(_configuration, field.value)
+                if not (isinstance(_field, list)):
+                    _field = [_field]
+                _data.update(dict(map(lambda f: (
+                    f, get_arg_value(param_name=f)), _field)))
+
+            for field in metric_outputs:
+                _field = getattr(_configuration, field.value)
+                if not (isinstance(_field, list)):
+                    _field = [_field]
+                _data.update(dict(map(lambda f: (
+                    f, original_result.get(f)), _field)))
+
+            metric_result = run_in_event_loop(
+                _evaluate_metrics_async,
+                configuration=_configuration,
+                data=_data,
+                metrics=metrics,
+                metric_groups=metric_groups,
+                api_client=self.api_client
+            )
+            metric_result = metric_result.to_dict()
+
+            for mr in metric_result:
+                node_result = {
+                    "applies_to": "node",
+                    "node_name": func.__name__,
+                    **mr
+                }
+                node_result["message_id"] = node_result["record_id"]
+                amr = AgentMetricResult(**node_result)
+
+                AIEventRecorder.record_metric(name=amr.name,
+                                              value=amr.value,
+                                              attributes={"wxgov.result.metric": amr.model_dump_json(exclude_unset=True)})
+                metrics_configuration = MetricsConfiguration(
+                    configuration=_configuration, metrics=metrics)
+                record_span_attributes({"wxgov.config.metrics."+str(type(self)).split(".")[2]: dumps({
+                    "metrics_configuration": metrics_configuration.model_dump(mode="json"),
+                    "compute_real_time": "true"
+                })})
+
+                with self.lock:
+                    self.metric_results.append(amr)
+
+        else:
+            metrics_configuration = MetricsConfiguration(
+                configuration=_configuration, metrics=metrics)
+            # Store the configuration of metrics to compute in traces
+            record_span_attributes({"wxgov.config.metrics."+str(type(self)).split(".")[2]: dumps({
+                "metrics_configuration": metrics_configuration.model_dump(mode="json"),
+                "compute_real_time": "false"
+            })})
+
+        return original_result

ibm_watsonx_gov/metrics/context_relevance/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------