ibm-watsonx-gov 1.3.3__cp313-cp313-win_amd64.whl

ibm_watsonx_gov/metrics/text_reading_ease/text_reading_ease_metric.py

@@ -0,0 +1,123 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+import textstat
+from pydantic import Field
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.utils.python_utils import replace_none_with_empty_string
+from ibm_watsonx_gov.utils.validation_util import validate_output
+
+TEXT_READING_EASE = "text_reading_ease"
+TEXT_READING_EASE_DISPLAY_NAME = "Text Reading Ease"
+FLESCH_READING_EASE = "flesch_reading_ease"
+TEXTSTAT = "textstat"
+
+
+class TextReadingEaseResult(RecordMetricResult):
+    name: str = TEXT_READING_EASE
+    display_name: str = TEXT_READING_EASE_DISPLAY_NAME
+    provider: str = TEXTSTAT
+    method: str = FLESCH_READING_EASE
+
+
+class TextReadingEaseMetric(GenAIMetric):
+    """
+    Defines the Text Reading Ease metric class.
+
+    The Text Reading Ease metric measures how readable the text is.
+    It is computed using the flesch_reading_ease method.
+    The score ranges broadly from 0 to 100, where a higher score indicates that a text is easier to read
+
+    Examples:
+        1. Create Text Reading Ease metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = TextReadingEaseMetric()
+                result = MetricsEvaluator().evaluate(data={"generated_text": "..."},
+                                                    metrics=[metric])
+
+        2. Create Text Reading Ease metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=70)
+                metric = TextReadingEaseMetric(thresholds=[threshold])
+    """
+    name: Annotated[Literal["text_reading_ease"],
+                    Field(title="name",
+                          description="The text reading ease metric name.",
+                          default=TEXT_READING_EASE, frozen=True)]
+    display_name: Annotated[Literal["Text Reading Ease"],
+                            Field(title="Display Name",
+                                  description="The text reading ease metric display name.",
+                                  default=TEXT_READING_EASE_DISPLAY_NAME, frozen=True)]
+    method: Annotated[Literal["flesch_reading_ease"],
+                      Field(title="Method",
+                            description="The method used to compute text reading ease metric.",
+                            default=FLESCH_READING_EASE)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="lower_limit", value=70)])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.READABILITY, frozen=True)]
+
+    def evaluate(
+            self,
+            data: pd.DataFrame,
+            configuration: GenAIConfiguration | AgenticAIConfiguration,
+            **kwargs,
+    ) -> list[AggregateMetricResult]:
+        from ibm_watsonx_gov.utils.aggregation_util import get_summaries
+
+        validate_output(data.columns.to_list(), configuration)
+        predictions = data[configuration.output_fields[0]].to_list()
+        record_ids = data[configuration.record_id_field].to_list()
+        replace_none_with_empty_string(predictions)
+
+        all_scores = self._compute(predictions=predictions)
+        record_level_metrics = [
+            TextReadingEaseResult(record_id=record_id,
+                                  value=score, thresholds=self.thresholds, group=MetricGroup.READABILITY.value)
+            for score, record_id in zip(all_scores, record_ids)
+        ]
+        summary = get_summaries(all_scores)
+        aggregate_metric_scores = AggregateMetricResult(
+            name=self.name,
+            display_name=self.display_name,
+            provider=TEXTSTAT,
+            method=self.method,
+            group=self.group,
+            min=summary.get("min"),
+            max=summary.get("max"),
+            mean=summary.get("mean"),
+            value=summary.get("mean"),
+            total_records=len(record_level_metrics),
+            record_level_metrics=record_level_metrics,
+            thresholds=self.thresholds,
+        )
+
+        return aggregate_metric_scores
+
+    def _compute(self, predictions: list) -> list:
+        return [textstat.flesch_reading_ease(pred) for pred in predictions]

ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_decorator.py

@@ -0,0 +1,67 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.tool_call_accuracy.tool_call_accuracy_metric import \
+    ToolCallAccuracyMetric
+from ibm_watsonx_gov.providers.tool_call_metric_provider import \
+    ToolCallMetricProvider
+
+
+class ToolCallAccuracyDecorator(BaseMetricDecorator):
+    def evaluate_tool_call_accuracy(self,
+                                    func: Optional[Callable] = None,
+                                    *,
+                                    configuration: Optional[AgenticAIConfiguration] = None,
+                                    metrics: list[GenAIMetric] = []
+                                    ) -> dict:
+        """
+        An evaluation decorator for computing tool call accuracy metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_tool_call_accuracy, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [ToolCallAccuracyMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(ToolCallAccuracyMetric,))
+
+                metric_inputs = [
+                    EvaluatorFields.INPUT_FIELDS
+                ]
+                metric_outputs = [
+                    EvaluatorFields.TOOL_CALLS_FIELD, EvaluatorFields.OUTPUT_FIELDS]
+
+                if isinstance(configuration.tools, list) and all(callable(item) for item in configuration.tools):
+                    configuration.tools = ToolCallMetricProvider.get_tools_list_schema(
+                        configuration.tools)
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating tool call accuracy metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/tool_call_accuracy/tool_call_accuracy_metric.py

@@ -0,0 +1,162 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.detectors_provider import DetectorsProvider
+from ibm_watsonx_gov.providers.tool_call_metric_provider import \
+    ToolCallMetricProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
+from ibm_watsonx_gov.utils.validation_util import validate_tool_calls, validate_input
+from pydantic import Field
+
+logger = GovSDKLogger.get_logger(__name__)
+TOOL_CALL_ACCURACY = "tool_call_accuracy"
+FUNCTION_CALL = "function_call"
+
+
+class ToolCallAccuracyMetric(GenAIMetric):
+    """
+    ToolCallAccuracyMetric checks whether the tool call in the LLM response is 
+    syntactically correct and semantically meaningful, given the user's query and 
+    the available tool definitions.
+
+    The ToolCallAccuracyMetric can be computed using the below methods:
+
+    1. syntactic (default)
+    2. granite_guardian
+
+    Examples:
+        1. Create ToolCallAccuracyMetric by passing the basic configuration.
+            .. code-block:: python
+
+                config = GenAIConfiguration(tools = [get_weather,fetch_stock_price])
+                evaluator = MetricsEvaluator(configuration=config)
+                df = pd.read_csv("")
+
+                metrics = [ToolCallAccuracyMetric()]
+                result = evaluator.evaluate(data=df, metrics=metrics)
+
+        2. Create ToolCallAccuracyMetric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="upper_limit", value=0.8)
+                metric = ToolCallAccuracyMetric(threshold=threshold)
+
+        3. Create ToolCallAccuracyMetric by passing custom tool calls field in configuration.
+            .. code-block:: python
+
+                test_data = {"input_text": "What's the latest on Tesla today?", 
+                "tools_used":[{"name": "get_weather", "args": {"location": "Tesla"}, "id": "0724", "type": "tool_call"}]}
+
+                config = GenAIConfiguration(tools = [get_weather,fetch_stock_price],
+                                            tool_calls_field="tools_used")
+                evaluator = MetricsEvaluator(configuration=config)
+                metrics = [ToolCallAccuracyMetric()]
+                result = evaluator.evaluate(data=test_data, metrics=metrics)
+
+        4. Create ToolCallAccuracyMetric by passing a list of dictionary items as tools field in configuration.
+            .. code-block:: python
+
+                available_tools = [{"type":"function","function":{"name":"f1_name","description":"f1_description.","parameters":{"parameter1":{"description":"parameter_description","type":"parameter_type","default":"default_value"}}}}]
+                config = GenAIConfiguration(tools = available_tools,
+                                            tool_calls_field="tools_used")
+                evaluator = MetricsEvaluator(configuration=config)
+                df = pd.read_csv("")
+
+                metrics = [ToolCallAccuracyMetric()]
+                result = evaluator.evaluate(data=df, metrics=metrics)
+    """
+
+    name: Annotated[Literal["tool_call_accuracy"], Field(title="Metric Name",
+                                                         description="The tool call accuracy metric name.",
+                                                         default=TOOL_CALL_ACCURACY)]
+    display_name: Annotated[Literal["Tool Call Accuracy"], Field(title="Display Name",
+                                                                 description="The tool call accuracy metric display name.",
+                                                                 default="Tool Call Accuracy", frozen=True)]
+    tasks: Annotated[list[TaskType], Field(title="Task Type",
+                                           description="The generative task type.",
+                                           default=[TaskType.RAG])]
+    group: Annotated[MetricGroup, Field(
+        default=MetricGroup.TOOL_CALL_QUALITY, frozen=True)]
+
+    method: Annotated[Literal["syntactic", "granite_guardian"], Field(title="Computation Method",
+                                                                      description="The method used to compute the metric.",
+                                                                      default="syntactic")]
+    thresholds: Annotated[list[MetricThreshold], Field(title="Metric threshold",
+                                                       description="Value that defines the violation limit for the metric",
+                                                       default=[MetricThreshold(
+                                                           type="lower_limit", value=0.7)]
+                                                       )]
+
+    async def evaluate_async(
+        self,
+        data: pd.DataFrame,
+        configuration: GenAIConfiguration | AgenticAIConfiguration,
+        **kwargs
+    ) -> list[AggregateMetricResult]:
+
+        data_cols = data.columns.to_list()
+
+        try:
+            validate_tool_calls(data_cols, configuration)
+            validate_input(data_cols, configuration)
+        except ValueError as ve:
+            if kwargs.get("ignore_validation_errors"):
+                message = f"Skipping '{self.name}' computation because the validation failed. Details: {str(ve)}"
+                logger.warning(message)
+                return
+            raise ve
+
+        if self.method == "granite_guardian":
+            kwargs["detector_params"] = {
+                "risk_name": FUNCTION_CALL, "threshold": 0.001}
+            tool_call_provider = DetectorsProvider(configuration=configuration,
+                                                   metric_name=self.name,
+                                                   metric_display_name=self.display_name,
+                                                   metric_method=self.method,
+                                                   metric_group=self.group,
+                                                   thresholds=self.thresholds,
+                                                   **kwargs)
+            metric_result = await tool_call_provider.evaluate_async(data=data)
+        elif self.method == "syntactic":
+            tool_call_provider = ToolCallMetricProvider(
+                configuration=configuration, metric=self)
+
+            # Compute the metrics
+            metric_result = await tool_call_provider.compute_metrics(data)
+        return metric_result
+
+    def evaluate(self, data: pd.DataFrame | dict,
+                 configuration: GenAIConfiguration | AgenticAIConfiguration,
+                 **kwargs):
+        """
+        Evaluate the data for ToolCallAccuracyMetric
+        Args:
+            data (pd.DataFrame | dict): Data to be evaluated
+            configuration (GenAIConfiguration | AgenticAIConfiguration): Metrics configuration
+            **kwargs: Additional keyword arguments
+
+        Returns:
+            AggregateMetricResult: The computed metrics
+        """
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_decorator.py

@@ -0,0 +1,68 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+from functools import partial
+from typing import Callable, Optional
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.tool_call_parameter_accuracy.tool_call_parameter_accuracy_metric import \
+    ToolCallParameterAccuracyMetric
+from ibm_watsonx_gov.providers.tool_call_metric_provider import \
+    ToolCallMetricProvider
+from wrapt import decorator
+
+
+class ToolCallParameterAccuracyDecorator(BaseMetricDecorator):
+    def evaluate_tool_call_parameter_accuracy(self,
+                                              func: Optional[Callable] = None,
+                                              *,
+                                              configuration: Optional[AgenticAIConfiguration] = None,
+                                              metrics: list[GenAIMetric] = []
+                                              ) -> dict:
+        """
+        An evaluation decorator for computing tool call parameter accuracy metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_tool_call_parameter_accuracy, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [ToolCallParameterAccuracyMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(ToolCallParameterAccuracyMetric,))
+
+                metric_inputs = [
+                    EvaluatorFields.INPUT_FIELDS
+                ]
+                metric_outputs = [
+                    EvaluatorFields.TOOL_CALLS_FIELD, EvaluatorFields.OUTPUT_FIELDS]
+
+                if isinstance(configuration.tools, list) and all(callable(item) for item in configuration.tools):
+                    configuration.tools = ToolCallMetricProvider.get_tools_list_schema(
+                        configuration.tools)
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating tool call parameter accuracy metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/tool_call_parameter_accuracy/tool_call_parameter_accuracy_metric.py

@@ -0,0 +1,151 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.llm_judge import LLMJudge
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.tool_call_metric_provider import \
+    ToolCallMetricProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.validation_util import validate_tool_calls
+
+TOOL_CALL_PARAMETER_ACCURACY = "tool_call_parameter_accuracy"
+
+
+class ToolCallParameterAccuracyMetric(GenAIMetric):
+    """
+    ToolCallParameterAccuracyMetric assesses whether ALL parameter values 
+    in a function call are directly supported by conversation history 
+    or API specifications. Identifies hallucinated values, missing information, 
+    format errors, and contradictory values.
+
+    The ToolCallParameterAccuracyMetric will be computed using llm_as_judge.
+
+    Examples:
+        1. Create ToolCallParameterAccuracyMetric by passing the basic configuration.
+            .. code-block:: python
+
+                config = GenAIConfiguration(tools = [get_weather,fetch_stock_price])
+                evaluator = MetricsEvaluator(configuration=config)
+                df = pd.read_csv("")
+                llm_judge = LLMJudge(
+                        model=WxAIFoundationModel(
+                            model_id="meta-llama/llama-3-3-70b-instruct",
+                            project_id=os.getenv("WATSONX_PROJECT_ID"),
+                        )
+                    )
+                metrics = [ToolCallParameterAccuracyMetric(llm_judge=llm_judge)]
+                result = evaluator.evaluate(data=df, metrics=metrics)
+
+        2. Create ToolCallParameterAccuracyMetric by passing custom tool calls field in configuration.
+            .. code-block:: python
+
+                config = GenAIConfiguration(tools = [get_weather,fetch_stock_price],
+                                            tool_calls_field="tools_used")
+                evaluator = MetricsEvaluator(configuration=config)
+                df = pd.read_csv("")
+                llm_judge = LLMJudge(
+                        model=WxAIFoundationModel(
+                            model_id="meta-llama/llama-3-3-70b-instruct",
+                            project_id=os.getenv("WATSONX_PROJECT_ID"),
+                        )
+                    )
+                metrics = [ToolCallParameterAccuracyMetric(llm_judge=llm_judge)]
+                result = evaluator.evaluate(data=df, metrics=metrics)
+
+        3. Create ToolCallParameterAccuracyMetric with a custom threshold.
+            .. code-block:: python
+
+                llm_judge = LLMJudge(
+                        model=WxAIFoundationModel(
+                            model_id="meta-llama/llama-3-3-70b-instruct",
+                            project_id=os.getenv("WATSONX_PROJECT_ID"),
+                        )
+                    )
+                threshold  = MetricThreshold(type="upper_limit", value=0.8)
+                metric = ToolCallParameterAccuracyMetric(llm_judge=llm_judge, threshold=threshold)
+    """
+
+    name: Annotated[Literal["tool_call_parameter_accuracy"], Field(title="Metric Name",
+                                                                   description="The name of metric.",
+                                                                   default=TOOL_CALL_PARAMETER_ACCURACY)]
+    display_name: Annotated[Literal["Tool Call Parameter Accuracy"], Field(title="Display Name",
+                                                                           description="The tool call parameter accuracy metric display name.",
+                                                                           default="Tool Call Parameter Accuracy", frozen=True)]
+    tasks: Annotated[list[TaskType], Field(title="Task Type",
+                                           description="The generative task type.",
+                                           default=[TaskType.RAG])]
+    group: Annotated[MetricGroup, Field(
+        default=MetricGroup.TOOL_CALL_QUALITY, frozen=True)]
+
+    llm_judge: Annotated[LLMJudge | None, Field(
+        description="The LLM judge used to compute the metric.", default=None)]
+
+    method: Annotated[Literal["llm_as_judge"], Field(title="Computation Method",
+                                                           description="The method used to compute the metric.",
+                                                           default="llm_as_judge")]
+    thresholds: Annotated[list[MetricThreshold], Field(title="Metric threshold",
+                                                       description="Value that defines the violation limit for the metric",
+                                                       default=[MetricThreshold(
+                                                           type="lower_limit", value=0.8)]
+                                                       )]
+    metric_mapping_name: Annotated[Literal["general_hallucination_check"], Field(title="Metric Mapping Name",
+                                                                                 description="The mapping name of metric with llmevalkit.",
+                                                                                 default="general_hallucination_check")]
+
+    async def evaluate_async(self, data: pd.DataFrame | dict,
+                             configuration: GenAIConfiguration | AgenticAIConfiguration,
+                             **kwargs) -> AggregateMetricResult:
+        """
+        Evaluate the data for ToolCallParameterAccuracyMetric
+        Args:
+            data (pd.DataFrame | dict): Data to be evaluated
+            configuration (GenAIConfiguration | AgenticAIConfiguration): Metrics configuration
+            **kwargs: Additional keyword arguments
+
+        Returns:
+            AggregateMetricResult: The computed metrics
+        """
+        data_cols = data.columns.to_list()
+        validate_tool_calls(data_cols, configuration)
+
+        tool_call_provider = ToolCallMetricProvider(
+            configuration=configuration, metric=self)
+        metric_config = {
+            "general_metrics": [self.metric_mapping_name],
+            "function_metrics": None,
+            "parameter_metrics": None,
+            "transform_enabled": False
+        }
+        metric_result = await tool_call_provider.compute_metrics(
+            data, syntactic_only=False, metric_result_mapping_name="general",  **metric_config)
+
+        return metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration | AgenticAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/tool_call_relevance/tool_call_relevance_decorator.py

@@ -0,0 +1,71 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+from functools import partial
+from typing import Callable, Optional
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.tool_call_relevance.tool_call_relevance_metric import \
+    ToolCallRelevanceMetric
+from ibm_watsonx_gov.providers.tool_call_metric_provider import \
+    ToolCallMetricProvider
+from ibm_watsonx_gov.utils.python_utils import parse_functions_to_openai_schema
+from wrapt import decorator
+
+
+class ToolCallRelevanceDecorator(BaseMetricDecorator):
+    def evaluate_tool_call_relevance(self,
+                                     func: Optional[Callable] = None,
+                                     *,
+                                     configuration: Optional[AgenticAIConfiguration] = None,
+                                     metrics: list[GenAIMetric] = [
+                                         ToolCallRelevanceMetric()
+                                     ]
+                                     ) -> dict:
+        """
+        An evaluation decorator for computing tool call relevance metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_tool_call_relevance, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [ToolCallRelevanceMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(ToolCallRelevanceMetric,))
+
+                metric_inputs = [
+                    EvaluatorFields.INPUT_FIELDS
+                ]
+                metric_outputs = [
+                    EvaluatorFields.TOOL_CALLS_FIELD, EvaluatorFields.OUTPUT_FIELDS]
+
+                if isinstance(configuration.tools, list) and all(callable(item) for item in configuration.tools):
+                    configuration.tools = ToolCallMetricProvider.get_tools_list_schema(
+                        configuration.tools)
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating tool call relevance metric on {func.__name__},") from ex
+
+        return wrapper(func)