ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/metrics/pii/pii_decorator.py

@@ -0,0 +1,59 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.pii.pii_metric import PIIMetric
+
+
+class PIIDecorator(BaseMetricDecorator):
+
+    def evaluate_pii(self,
+                     func: Optional[Callable] = None,
+                     *,
+                     configuration: Optional[AgenticAIConfiguration] = None,
+                     metrics: list[GenAIMetric] = [],
+                     ) -> dict:
+        """
+        An evaluation decorator for computing pii metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_pii, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [PIIMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(PIIMetric))
+
+                metric_inputs = [EvaluatorFields.INPUT_FIELDS]
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=[])
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating pii metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/pii/pii_metric.py

@@ -0,0 +1,96 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.detectors_provider import DetectorsProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+
+PII = "pii"
+
+
+class PIIMetric(GenAIMetric):
+    """
+    Defines the PII metric class.
+
+    The PII metric measures  measures if your model input or output data contains any personally identifiable information in the input.
+    It is computed using the Watson Natural Language Processing entity extraction model.
+
+    Examples:
+        1. Create PII metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = PIIMetric()
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", metrics=[metric])
+
+        2. Create PII metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = PIIMetric(threshold=threshold)
+    """
+    name: Annotated[Literal["pii"],
+                    Field(title="Name",
+                          description="The pii metric name.",
+                          default=PII, frozen=True)]
+    display_name: Annotated[Literal["PII"],
+                            Field(title="Display Name",
+                                  description="The pii metric display name.",
+                                  default="PII", frozen=True)]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="upper_limit", value=0.1)])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.CONTENT_SAFETY, frozen=True)]
+
+    async def evaluate_async(
+            self,
+            data: pd.DataFrame | dict,
+            configuration: GenAIConfiguration,
+            **kwargs
+    ) -> list[AggregateMetricResult]:
+
+        provider = DetectorsProvider(configuration=configuration,
+                                     metric_name=self.name,
+                                     metric_display_name=self.display_name,
+                                     metric_method=self.method,
+                                     metric_group=self.group,
+                                     thresholds=self.thresholds,
+                                     **kwargs)
+        aggregated_metric_result = await provider.evaluate_async(data=data)
+        return aggregated_metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/profanity/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/metrics/profanity/profanity_decorator.py

@@ -0,0 +1,60 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.profanity.profanity_metric import ProfanityMetric
+
+
+class ProfanityDecorator(BaseMetricDecorator):
+
+    def evaluate_profanity(self,
+                           func: Optional[Callable] = None,
+                           *,
+                           configuration: Optional[AgenticAIConfiguration] = None,
+                           metrics: list[GenAIMetric] = []
+                           ) -> dict:
+        """
+        An evaluation decorator for computing profanity on an agentic node via granite guardian.
+        """
+        if func is None:
+            return partial(self.evaluate_profanity, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [ProfanityMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(ProfanityMetric))
+
+                metric_inputs = [EvaluatorFields.INPUT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=[])
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating profanity on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/profanity/profanity_metric.py

@@ -0,0 +1,103 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.detectors_provider import DetectorsProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.validation_util import validate_input
+
+PROFANITY = "profanity"
+
+
+class ProfanityMetric(GenAIMetric):
+    """
+    Defines the Profanity metric class.
+
+    The Profanity metric measures the risk of use of language considered offensive or socially unacceptable in a given context. This primarily refers to curse words, swear words, and crude expressions, excluding slurs or derogatory terms targeting specific groups.
+    It is computed using the granite guardian model.
+
+    Examples:
+        1. Create Profanity metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = ProfanityMetric()
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", metrics=[metric])
+
+        2. Create Profanity metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = ProfanityMetric(threshold=threshold)
+    """
+    name: Annotated[Literal["profanity"],
+                    Field(title="Name",
+                          description="The profanity metric name.",
+                          default=PROFANITY, frozen=True)]
+    display_name: Annotated[Literal["Profanity"],
+                            Field(title="Display Name",
+                                  description="The profanity metric display name.",
+                                  default="Profanity", frozen=True)]
+    method: Annotated[Literal["granite_guardian"],
+                      Field(title="Method",
+                            description="The method used to compute harm metric.",
+                            default="granite_guardian")]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="upper_limit", value=0.5)])]
+    group: Annotated[MetricGroup,
+                     Field(title="Group",
+                           description="The metric group.",
+                           default=MetricGroup.CONTENT_SAFETY, frozen=True)]
+
+    async def evaluate_async(
+        self,
+        data: pd.DataFrame,
+        configuration: GenAIConfiguration,
+        **kwargs
+    ) -> list[AggregateMetricResult]:
+
+        validate_input(data.columns.to_list(), configuration)
+        kwargs["detector_params"] = {"risk_name": PROFANITY}
+        provider = DetectorsProvider(configuration=configuration,
+                                     metric_name=self.name,
+                                     metric_display_name=self.display_name,
+                                     metric_method=self.method,
+                                     metric_group=self.group,
+                                     thresholds=self.thresholds,
+                                     **kwargs)
+        aggregated_metric_result = await provider.evaluate_async(data=data)
+        return aggregated_metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/prompt_safety_risk/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_decorator.py

@@ -0,0 +1,57 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics import PromptSafetyRiskMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+
+
+class PromptSafetyRiskDecorator(BaseMetricDecorator):
+
+    def evaluate_prompt_safety_risk(self,
+                                    func: Optional[Callable] = None,
+                                    *,
+                                    configuration: Optional[AgenticAIConfiguration] = None,
+                                    metrics: list[GenAIMetric]
+                                    ) -> dict:
+        """
+        An evaluation decorator for computing prompt safety risk metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_prompt_safety_risk, configuration=configuration, metrics=metrics)
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(PromptSafetyRiskMetric,))
+
+                metric_inputs = [EvaluatorFields.INPUT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=[])
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating prompt safety risk metric on {func.__name__},") from ex
+
+        return wrapper(func)

ibm_watsonx_gov/metrics/prompt_safety_risk/prompt_safety_risk_metric.py

@@ -0,0 +1,128 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+import pandas as pd
+from pydantic import Field
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.evaluation_result import AggregateMetricResult
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.providers.detectors_provider import DetectorsProvider
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.validation_util import validate_input
+
+PROMPT_SAFETY_RISK = "prompt_safety_risk"
+
+prompt_safety_methods = ["two_level_detection", "granite_guardian"]
+HARM = "harm"
+
+
+class PromptSafetyRiskMetric(GenAIMetric):
+    """
+    Defines the PromptSafetyRisk metric class.
+
+    The PromptSafetyRisk metric evaluates how likely an AI is to respond with harmful, unsafe, or inappropriate content.
+
+    Note : system_prompt is mandatory 
+
+    Examples:
+        1. Create PromptSafetyRisk metric with default parameters and compute using metrics evaluator.
+            .. code-block:: python
+
+                metric = PromptSafetyRiskMetric(system_prompt="...")
+                result = MetricsEvaluator().evaluate(data={"input_text": "...", metrics=[metric])
+
+        2. Create PromptSafetyRisk metric with a custom threshold.
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = PromptSafetyRiskMetric(threshold=threshold, system_prompt="...")
+
+        3. Create PromptSafetyRisk metric with "granite_guardian" method".
+            .. code-block:: python
+
+                threshold  = MetricThreshold(type="lower_limit", value=0.5)
+                metric = PromptSafetyRiskMetric(threshold=threshold, method="granite_guardian")
+    """
+    name: Annotated[Literal["prompt_safety_risk"],
+                    Field(title="Name",
+                          description="The prompt safety risk metric name.",
+                          default=PROMPT_SAFETY_RISK, frozen=True)]
+    display_name: Annotated[Literal["Prompt Safety Risk"],
+                            Field(title="Display Name",
+                                  description="The prompt safety risk metric display name.",
+                                  default="Prompt Safety Risk", frozen=True)]
+    method: Annotated[
+        Literal["two_level_detection", "granite_guardian"],
+        Field(title="Method",
+              description=f"The method used to compute the prompt safety risk metric.",
+              default="two_level_detection")]
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(title="Thresholds",
+                                description="The metric thresholds.",
+                                default=[MetricThreshold(type="upper_limit", value=0.5)])]
+    tasks: Annotated[list[TaskType],
+                     Field(title="Tasks",
+                           description="The list of supported tasks.",
+                           default=TaskType.values(), frozen=True)]
+    # TODO uncomment when the metric is pushed to prod
+    # group: Annotated[MetricGroup, Field(title="Group",
+    #                                     description="The metric group.",
+    #                                     default=MetricGroup.CONTENT_SAFETY, frozen=True)]
+    system_prompt: Annotated[str, Field(title="System Prompt",
+                                        default=None,
+                                        description=f"The AI model system prompt which contains instructions to define its overall behavior. Required only when the computation method is set to 'two_level_detection'.")]
+
+    async def evaluate_async(
+            self,
+            data: pd.DataFrame | dict,
+            configuration: GenAIConfiguration,
+            **kwargs
+    ) -> list[AggregateMetricResult]:
+        if self.method == "two_level_detection":
+            if not self.system_prompt:
+                raise AssertionError(
+                    f"The system_prompt field is required while using the 'two_level_detection' method for computation but was missing from the input.")
+        if self.method not in prompt_safety_methods:
+            raise ValueError(
+                f"The provided method '{self.method}' for computing '{self.name}' metric is not supported.")
+
+        validate_input(data.columns.to_list(), configuration)
+        # Set system_prompt as part of the detector parameters
+        if self.method == "granite_guardian":
+            kwargs["detector_params"] = {"risk_name": HARM}
+        elif self.method == "two_level_detection":
+            kwargs["detector_params"] = {"system_prompt": self.system_prompt}
+        provider = DetectorsProvider(configuration=configuration,
+                                     metric_name=self.name,
+                                     metric_display_name=self.display_name,
+                                     metric_method=self.method,
+                                     metric_group=MetricGroup.CONTENT_SAFETY,
+                                     thresholds=self.thresholds,
+                                     **kwargs)
+        aggregated_metric_result = await provider.evaluate_async(data=data)
+        return aggregated_metric_result
+
+    def evaluate(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: GenAIConfiguration,
+        **kwargs,
+    ):
+        # If ran in sync mode, block until it is done
+        return run_in_event_loop(
+            self.evaluate_async,
+            data=data,
+            configuration=configuration,
+            **kwargs,
+        )

ibm_watsonx_gov/metrics/reciprocal_rank/reciprocal_rank_decorator.py

@@ -0,0 +1,62 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+from functools import partial
+from typing import Callable, Optional
+
+from wrapt import decorator
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+from ibm_watsonx_gov.entities.metric import GenAIMetric
+from ibm_watsonx_gov.metrics.base_metric_decorator import BaseMetricDecorator
+from ibm_watsonx_gov.metrics.context_relevance.context_relevance_metric import \
+    ContextRelevanceMetric
+from ibm_watsonx_gov.metrics.reciprocal_rank.reciprocal_rank_metric import \
+    ReciprocalRankMetric
+
+
+class ReciprocalRankDecorator(BaseMetricDecorator):
+    def evaluate_reciprocal_rank(self,
+                                 func: Optional[Callable] = None,
+                                 *,
+                                 configuration: Optional[AgenticAIConfiguration] = None,
+                                 metrics: list[GenAIMetric] = []
+                                 ) -> dict:
+        """
+        An evaluation decorator for computing reciprocal rank metric on an agentic node.
+        """
+        if func is None:
+            return partial(self.evaluate_reciprocal_rank, configuration=configuration, metrics=metrics)
+
+        if not metrics:
+            metrics = [ReciprocalRankMetric()]
+
+        @decorator
+        def wrapper(func, instance, args, kwargs):
+
+            try:
+                self.validate(func=func, metrics=metrics,
+                              valid_metric_types=(ReciprocalRankMetric, ContextRelevanceMetric))
+
+                metric_inputs = [EvaluatorFields.INPUT_FIELDS]
+                metric_outputs = [EvaluatorFields.CONTEXT_FIELDS]
+
+                original_result = self.compute_helper(func=func, args=args, kwargs=kwargs,
+                                                      configuration=configuration,
+                                                      metrics=metrics,
+                                                      metric_inputs=metric_inputs,
+                                                      metric_outputs=metric_outputs)
+
+                return original_result
+            except Exception as ex:
+                raise Exception(
+                    f"There was an error while evaluating reciprocal rank metric on {func.__name__},") from ex
+
+        return wrapper(func)