ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/metrics/unethical_behavior/unethical_behavior_metric.py

@@ -0,0 +1,103 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.detectors_provider import DetectorsProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.validation_util import validate_input
+
+UNETHICAL_BEHAVIOR = "unethical_behavior"
+
+
+class UnethicalBehaviorMetric(GenAIMetric):
+    """
+    Defines the Unethical Behavior metric class.
+
+    The Unethical Behavior metric measures the risk of content with actions that violate moral or professional standards, focusing on exploitation for personal gain or disregard for others' well-being. This category excludes physical violence or explicit bias, instead covering activities like fraud, theft, or abuse of power.
+    It is computed using the granite guardian model.
+
+    Examples:
+        1. Create Unethical Behavior metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = UnethicalBehaviorMetric()
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", metrics=[metric])
+
+        2. Create Unethical Behavior metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = UnethicalBehaviorMetric(threshold=threshold)
+    """
+    name: Annotated[Literal["unethical_behavior"],
+                    Field(title="Name",
+                          description="The unethical behavior metric name.",
+                          default=UNETHICAL_BEHAVIOR, frozen=True)]
+    display_name: Annotated[Literal["Unethical Behavior"],
+                            Field(title="Display Name",
+                                  description="The unethical behavior metric display name.",
+                                  default="Unethical Behavior", frozen=True)]
+    method: Annotated[Literal["granite_guardian"],
+                      Field(title="Method",
+                            description="The method used to compute harm metric.",
+                            default="granite_guardian")]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="upper_limit", value=0.5)])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.CONTENT_SAFETY, frozen=True)]
+
+    async def evaluate_async(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs
+    ) -> list[AggregateMetricResult]:
+
+        validate_input(data.columns.to_list(), configuration)
+        kwargs["detector_params"] = {"risk_name": UNETHICAL_BEHAVIOR}
+        provider = DetectorsProvider(configuration=configuration,
+                                     metric_name=self.name,
+                                     metric_display_name=self.display_name,
+                                     metric_method=self.method,
+                                     metric_group=self.group,
+                                     thresholds=self.thresholds,
+                                     **kwargs)
+        aggregated_metric_result = await provider.evaluate_async(data=data)
+        return aggregated_metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_decorator.py

@@ -0,0 +1,66 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.unsuccessful_requests.unsuccessful_requests_metric import \
+    UnsuccessfulRequestsMetric
+
+
+class UnsuccessfulRequestsDecorator(BaseMetricDecorator):
+    def evaluate_unsuccessful_requests(self,
+                                       func: Optional[Callable] = None,
+                                       *,
+                                       configuration: Optional[AgenticAIConfiguration] = None,
+                                       metrics: list[GenAIMetric] = []
+                                       ) -> dict:
+        """
+        An evaluation decorator for computing unsuccessful requests metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_unsuccessful_requests, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [UnsuccessfulRequestsMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(UnsuccessfulRequestsMetric,))
+
+                metric_inputs = [
+                    EvaluatorFields.INPUT_FIELDS,
+                    EvaluatorFields.CONTEXT_FIELDS
+                ]
+                metric_references = [EvaluatorFields.REFERENCE_FIELDS]
+                metric_outputs = [EvaluatorFields.OUTPUT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs,
+                                                      metric_references=metric_references)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating unsuccessful requests metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/unsuccessful_requests/unsuccessful_requests_metric.py

@@ -0,0 +1,128 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config import AgenticAIConfiguration, GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.utils.validation_util import validate_output
+
+UNSUCCESSFUL_REQUESTS = "unsuccessful_requests"
+UNSUCCESSFUL_REQUESTS_DISPLAY_NAME = "Unsuccessful Requests"
+
+
+class UnsuccessfulRequestsResult(RecordMetricResult):
+    name: str = UNSUCCESSFUL_REQUESTS
+    display_name: str = UNSUCCESSFUL_REQUESTS_DISPLAY_NAME
+    group: MetricGroup = MetricGroup.ANSWER_QUALITY
+
+
+class UnsuccessfulRequestsMetric(GenAIMetric):
+    """
+    Defines the Unsuccessful Requests metric class.
+
+    The Unsuccessful Requests metric measures whether the model answered the request successfully or not by comparing the generated text against the list of unsuccessful phrases.
+
+    Examples:
+        1. Create Unsuccessful Requests metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = UnsuccessfulRequestsMetric()
+                result = MetricsEvaluator().evaluate(data={"generated_text": "...", metrics=[metric])
+
+        2. Create Unsuccessful Requests metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="upper_limit", value=0.2)
+                metric = UnsuccessfulRequestsMetric(threshold=threshold)
+    """
+    name: Annotated[Literal["unsuccessful_requests"],
+                    Field(title="Name",
+                          description="The unsuccessful requests metric name.",
+                          default=UNSUCCESSFUL_REQUESTS, frozen=True)]
+    display_name: Annotated[Literal["Unsuccessful Requests"],
+                            Field(title="Display Name",
+                                  description="The unsuccessful requests metric display name.",
+                                  default=UNSUCCESSFUL_REQUESTS_DISPLAY_NAME, frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=[TaskType.RAG, TaskType.QA])]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="upper_limit", value=0.1)])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.ANSWER_QUALITY, frozen=True)]
+    unsuccessful_phrases: Annotated[list[str],
+                                    Field(title="Unsuccessful phrases",
+                                          description="List of phrases to identify unsuccessful responses",
+                                          examples=[
+                                              ["i do not know", "i am not sure"]],
+                                          default=["i don't know", "i do not know", "i'm not sure",
+                                                   "i am not sure", "i'm unsure", "i am unsure",
+                                                   "i'm uncertain", "i am uncertain", "i'm not certain",
+                                                   "i am not certain", "i can't fulfill", "i cannot fulfill"],
+                                          )]
+
+    def evaluate(
+            self,
+            data: pd.DataFrame,
+            configuration: GenAIConfiguration | AgenticAIConfiguration,
+            **kwargs
+    ) -> AggregateMetricResult:
+        record_level_metrics = []
+        scores = []
+
+        validate_output(data.columns.to_list(), configuration)
+        for prediction_field in configuration.output_fields:
+            for prediction, record_id in zip(data[prediction_field], data[configuration.record_id_field]):
+                value = 0
+                for phrase in self.unsuccessful_phrases:
+                    if phrase.lower() in prediction.lower():
+                        value = 1
+                        break
+                scores.append(value)
+                record_level_metrics.append(
+                    UnsuccessfulRequestsResult(
+                        method="",
+                        provider="",
+                        record_id=record_id,
+                        value=value,
+                        thresholds=self.thresholds
+                    )
+                )
+
+        mean = sum(scores) / len(scores)
+        aggregate_metric_score = AggregateMetricResult(
+            name=self.name,
+            display_name=self.display_name,
+            method="",
+            provider="",
+            min=min(scores),
+            max=max(scores),
+            mean=mean,
+            value=mean,
+            total_records=len(record_level_metrics),
+            group=self.group,
+            record_level_metrics=record_level_metrics,
+            thresholds=self.thresholds
+        )
+
+        return aggregate_metric_score

ibm_watsonx_gov/metrics/user_id/user_id_metric.py

@@ -0,0 +1,111 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal, Optional
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import (MetricGroup, MetricValueType,
+                                            TaskType)
+from ibm_watsonx_gov.entities.evaluation_result import (AggregateMetricResult,
+                                                        RecordMetricResult)
+from ibm_watsonx_gov.entities.metric import GenAIMetric, Mapping, MappingItem
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+
+USER_ID = "user_id"
+
+
+class UserIdMetric(GenAIMetric):
+    """
+    Defines the User Id metric class.
+    The User Id metric identifies user identifiers from trace data or tabular data 
+    and aggregates them to determine the count of distinct users
+    Examples:
+        1. Create UserId metric with default parameters and compute using metrics AgenticEvaluator.
+            .. code-block:: python
+
+                agent_app = AgenticApp(name="Rag agent",
+                       metrics_configuration=MetricsConfiguration(metrics=[
+                           UserIdMetric()]))
+
+                evaluator = AgenticEvaluator(agentic_app=agent_app)
+                evaluator.start_run()
+                result = rag_app.invoke({"input_text": "What is concept drift?", "ground_truth": "Concept drift occurs when the statistical properties of the target variable change over time, causing a machine learning model’s predictions to become less accurate."})
+                evaluator.end_run()
+    """
+    name: Annotated[Literal["user_id"],
+                    Field(title="Name",
+                          description="The user_id metric name.",
+                          default=USER_ID, frozen=True)]
+    display_name: Annotated[Literal["User Id"],
+                            Field(title="Display Name",
+                                  description="The user_id metric display name.",
+                                  default="User Id", frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.MESSAGE_COMPLETION, frozen=True)]
+    mapping: Annotated[Optional[Mapping],
+                       Field(title="Mapping",
+                             description="The data mapping details for the metric which are used to read the values needed to compute the metric.",
+                             default_factory=lambda: Mapping(items=[MappingItem(name="user_id",
+                                                                                type="user_id",
+                                                                                span_name="LangGraph.workflow",
+                                                                                attribute_name="user.id",
+                                                                                json_path=None)])
+                             )]
+
+    async def evaluate_async(
+            self,
+            data: pd.DataFrame | dict,
+            configuration: GenAIConfiguration,
+            **kwargs
+    ) -> list[AggregateMetricResult]:
+
+        record_level_metrics: list[RecordMetricResult] = []
+
+        for _, row in data.iterrows():
+            if configuration.user_id_field not in row or not row[configuration.user_id_field]:
+                continue
+
+            record_level_metrics.append(
+                RecordMetricResult(
+                    name=self.name,
+                    display_name=self.display_name,
+                    method=self.method,
+                    label=row.get(configuration.user_id_field),
+                    value=None,
+                    group=self.group,
+                    record_id=row.get(configuration.record_id_field),
+                    value_type=MetricValueType.CATEGORICAL.value)
+            )
+
+        aggregated_metric_result = AggregateMetricResult.create(
+            record_level_metrics)
+        return aggregated_metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )