ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/metrics/cost/cost_metric.py

@@ -0,0 +1,155 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import ast
+import re
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.utils import calculate_cost
+from ibm_watsonx_gov.utils.validation_util import validate_field
+
+COST = "cost"
+COST_DISPLAY_NAME = "Cost"
+
+
+class CostResult(RecordMetricResult):
+    name: str = COST
+    display_name: str = COST_DISPLAY_NAME
+
+
+class CostMetric(GenAIMetric):
+    """
+    Defines the Cost metric class.
+
+    The Cost metric keep track of LLM usage cost for provided token count.
+
+    Examples:
+        1. Create Cost metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = CostMetric()
+                result = MetricsEvaluator().evaluate(data={"model": "...", "prompt_tokens": "...", "completion_tokens": "..."},
+                                                    metrics=[metric])
+    """
+    name: Annotated[Literal["cost"],
+                    Field(title="name",
+                          description="The cost metric name.",
+                          default=COST, frozen=True)]
+    display_name: Annotated[Literal["Cost"],
+                            Field(title="Display Name",
+                                  description="The cost metric display name.",
+                                  default=COST_DISPLAY_NAME, frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.USAGE, frozen=True)]
+
+    def evaluate(
+            self,
+            data: pd.DataFrame,
+            configuration: GenAIConfiguration | AgenticAIConfiguration,
+            **kwargs,
+    ) -> list[AggregateMetricResult]:
+        from ibm_watsonx_gov.utils.aggregation_util import get_summaries
+
+        validate_field("model_usage_detail_fields", configuration)
+        meta_data = self._extract_required_dtls(data, configuration)
+        record_ids = data[configuration.record_id_field].to_list()
+        costs = self._evaluate(meta_data)
+        record_level_metrics = [
+            CostResult(record_id=record_id, value=cost,
+                       group=MetricGroup.USAGE.value)
+            for cost, record_id in zip(costs, record_ids)
+        ]
+        summary = get_summaries(costs)
+        aggregate_metric_scores = AggregateMetricResult(
+            name=self.name,
+            display_name=self.display_name,
+            group=self.group,
+            min=summary.get("min"),
+            max=summary.get("max"),
+            mean=summary.get("mean"),
+            value=summary.get("mean"),
+            total_records=len(record_level_metrics),
+            record_level_metrics=record_level_metrics,
+        )
+
+        return aggregate_metric_scores
+
+    def _extract_required_dtls(self, data: pd.DataFrame, configuration: GenAIConfiguration | AgenticAIConfiguration):
+        """
+        Convert the dataframe to a consumable structure for calculate_cost function.
+        Structure:
+            [
+                {
+                    "model": "<model_name>",
+                    "total_prompt_tokens": <token_count>,           # optional
+                    "total_completion_tokens": <token_count>        # optional
+                },
+                ...
+            ]
+        """
+        patterns = configuration.model_usage_detail_fields
+        all_matching_cols = set()
+
+        for pattern in patterns:
+            regex = re.compile(pattern)
+            for col in data.columns:
+                if regex.fullmatch(col):
+                    all_matching_cols.add(col)
+
+        # Early exit if no matches
+        if not all_matching_cols:
+            return [[] for _ in range(len(data))]
+
+        consolidated_data = []
+        for _, row in data.iterrows():
+            cost_meta_data = []
+
+            for col in all_matching_cols:
+                model_data = getattr(row, col, None)
+                if isinstance(model_data, str):
+                    try:
+                        model_data = ast.literal_eval(model_data)
+                    except (ValueError, SyntaxError):
+                        model_data = None  # not a valid dict string
+
+                if not isinstance(model_data, dict):
+                    continue
+
+                model = model_data.get("model")
+                if not model or pd.isna(model):
+                    continue
+
+                cost_meta_data.append(model_data)
+
+            consolidated_data.append(cost_meta_data)
+
+        return consolidated_data
+
+    def _evaluate(self, usage_data: list[list[dict]]) -> list:
+        """
+        Compute cost from associated properties
+        """
+        costs = []
+        for data in usage_data:
+            costs.append(calculate_cost(data))
+        return costs

ibm_watsonx_gov/metrics/duration/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/metrics/duration/duration_decorator.py

@@ -0,0 +1,59 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.duration.duration_metric import \
+    DurationMetric
+
+
+class DurationDecorator(BaseMetricDecorator):
+    def evaluate_duration(self,
+                          func: Optional[Callable] = None,
+                          *,
+                          configuration: Optional[AgenticAIConfiguration] = None,
+                          metrics: list[GenAIMetric] = []
+                          ) -> dict:
+        """
+        An evaluation decorator for computing duration metric.
+        """
+        if func is None:
+            return partial(self.evaluate_duration, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [DurationMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(DurationMetric,))
+
+                metric_inputs = [EvaluatorFields.START_TIME_FIELD,
+                                 EvaluatorFields.END_TIME_FIELD]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating duration metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/duration/duration_metric.py

@@ -0,0 +1,111 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.utils.validation_util import validate_start_time, validate_end_time
+
+DURATION = "duration"
+DURATION_DISPLAY_NAME = "Duration"
+LATENCY = "latency"
+LATENCY_DISPLAY_NAME = "Latency"
+
+
+class DurationMetric(GenAIMetric):
+    """
+    Defines the Duration metric class.
+
+    The Duration metric measures how long it take to process the task.
+
+    Examples:
+        1. Create Duration metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = DurationMetric()
+                result = MetricsEvaluator().evaluate(data={"start_time": "...", "end_time": "..."},
+                                                    metrics=[metric])
+    """
+    name: Annotated[Literal["duration", "latency"],
+                    Field(title="name",
+                          description="The duration metric name.",
+                          default=DURATION, frozen=True)]
+    display_name: Annotated[Literal["Duration", "Latency"],
+                            Field(title="Display Name",
+                                  description="The duration metric display name.",
+                                  default=DURATION_DISPLAY_NAME, frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.PERFORMANCE, frozen=True)]
+
+    def evaluate(
+            self,
+            data: pd.DataFrame,
+            configuration: GenAIConfiguration | AgenticAIConfiguration,
+            **kwargs,
+    ) -> list[AggregateMetricResult]:
+        from ibm_watsonx_gov.utils.aggregation_util import get_summaries
+
+        data_cols = data.columns.to_list()
+        validate_start_time(data_cols, configuration)
+        validate_end_time(data_cols, configuration)
+        record_ids = data[configuration.record_id_field].to_list()
+        if self.applies_to == "message":
+            metric_name = self.name
+            metric_display_name = self.display_name
+        else:  # node
+            metric_name = LATENCY
+            metric_display_name = LATENCY_DISPLAY_NAME
+
+        durations = self._evaluate(data, configuration)
+        record_level_metrics = [
+            RecordMetricResult(record_id=record_id,
+                               value=duration, name=metric_name, display_name=metric_display_name, group=MetricGroup.PERFORMANCE.value)
+            for duration, record_id in zip(durations, record_ids)
+        ]
+        summary = get_summaries(durations)
+        aggregate_metric_scores = AggregateMetricResult(
+            name=self.name,
+            display_name=self.display_name,
+            group=self.group,
+            min=summary.get("min"),
+            max=summary.get("max"),
+            mean=summary.get("mean"),
+            value=summary.get("mean"),
+            total_records=len(record_level_metrics),
+            record_level_metrics=record_level_metrics,
+        )
+
+        return aggregate_metric_scores
+
+    def _evaluate(self, data: pd.DataFrame, config: GenAIConfiguration | AgenticAIConfiguration) -> list:
+        """
+        Compute durations from start_time and end_time columns.
+        """
+        # Convert to numeric, handling errors
+        start_time = pd.to_numeric(
+            data[config.start_time_field], errors="coerce")
+        end_time = pd.to_numeric(
+            data[config.end_time_field], errors="coerce")
+
+        # Calculate duration in seconds (convert from nanoseconds to seconds)
+        durations = ((end_time - start_time) / 1e9).tolist()
+        return durations

ibm_watsonx_gov/metrics/evasiveness/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/metrics/evasiveness/evasiveness_decorator.py

@@ -0,0 +1,61 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.evasiveness.evasiveness_metric import \
+    EvasivenessMetric
+
+
+class EvasivenessDecorator(BaseMetricDecorator):
+
+    def evaluate_evasiveness(self,
+                             func: Optional[Callable] = None,
+                             *,
+                             configuration: Optional[AgenticAIConfiguration] = None,
+                             metrics: list[GenAIMetric] = []
+                             ) -> dict:
+        """
+        An evaluation decorator for computing evasiveness on an agentic node via granite guardian.
+        """
+        if func is None:
+            return partial(self.evaluate_evasiveness, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [EvasivenessMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(EvasivenessMetric))
+
+                metric_inputs = [EvaluatorFields.INPUT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=[])
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating evasiveness on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/evasiveness/evasiveness_metric.py

@@ -0,0 +1,103 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.detectors_provider import DetectorsProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.validation_util import validate_input
+
+EVASIVENESS = "evasiveness"
+
+
+class EvasivenessMetric(GenAIMetric):
+    """
+    Defines the Evasiveness metric class.
+
+    The Evasiveness metric measures the risk of avoiding to engage without providing sufficient reason.
+    It is computed using the granite guardian model.
+
+    Examples:
+        1. Create Evasiveness metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = EvasivenessMetric()
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", metrics=[metric])
+
+        2. Create Evasiveness metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = EvasivenessMetric(threshold=threshold)
+    """
+    name: Annotated[Literal["evasiveness"],
+                    Field(title="Name",
+                          description="The evasiveness metric name.",
+                          default=EVASIVENESS, frozen=True)]
+    display_name: Annotated[Literal["Evasiveness"],
+                            Field(title="Display Name",
+                                  description="The evasiveness metric display name.",
+                                  default="Evasiveness", frozen=True)]
+    method: Annotated[Literal["granite_guardian"],
+                      Field(title="Method",
+                            description="The method used to compute evasiveness metric.",
+                            default="granite_guardian")]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="upper_limit", value=0.5)])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.CONTENT_SAFETY, frozen=True)]
+
+    async def evaluate_async(
+        self,
+        data: pd.DataFrame,
+        configuration: GenAIConfiguration,
+        **kwargs
+    ) -> list[AggregateMetricResult]:
+
+        validate_input(data.columns.to_list(), configuration)
+        kwargs["detector_params"] = {"risk_name": EVASIVENESS}
+        provider = DetectorsProvider(configuration=configuration,
+                                     metric_name=self.name,
+                                     metric_display_name=self.display_name,
+                                     metric_method=self.method,
+                                     metric_group=self.group,
+                                     thresholds=self.thresholds,
+                                     **kwargs)
+        aggregated_metric_result = await provider.evaluate_async(data=data)
+        return aggregated_metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/faithfulness/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/metrics/faithfulness/faithfulness_decorator.py

@@ -0,0 +1,65 @@
+
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.faithfulness.faithfulness_metric import \
+    FaithfulnessMetric
+
+
+class FaithfulnessDecorator(BaseMetricDecorator):
+    def evaluate_faithfulness(self,
+                              func: Optional[Callable] = None,
+                              *,
+                              configuration: Optional[AgenticAIConfiguration] = None,
+                              metrics: list[GenAIMetric] = []
+                              ) -> dict:
+        """
+        An evaluation decorator for computing faithfulness metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_faithfulness, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [FaithfulnessMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(FaithfulnessMetric,))
+
+                metric_inputs = [
+                    EvaluatorFields.INPUT_FIELDS,
+                    EvaluatorFields.CONTEXT_FIELDS
+                ]
+                metric_outputs = [EvaluatorFields.OUTPUT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating faithfulness metric on {func.__name__},") from ex
+
+        return wrapper(func)