ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/utils/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/utils/aggregation_util.py

@@ -0,0 +1,346 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import copy
+from collections import Counter, defaultdict
+from typing import List
+
+import numpy as np
+import pandas as pd
+from ibm_watsonx_gov.entities.agentic_app import Node
+from ibm_watsonx_gov.entities.agentic_evaluation_result import \
+    AgenticEvaluationResult
+from ibm_watsonx_gov.entities.enums import MetricGroup, MetricValueType
+from ibm_watsonx_gov.entities.evaluation_result import (
+    AgentMetricResult, AggregateAgentMetricResult, RecordMetricResult)
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+from ibm_watsonx_gov.metrics.llm_validation.llm_validation_metric import \
+    LLMValidationMetric
+from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
+
+logger = GovSDKLogger.get_logger(__name__)
+
+
+def get_aggregated_thresholds(metric_results: List[AgentMetricResult]) -> List[MetricThreshold]:
+    """
+    Aggregates thresholds from a list of AgentMetricResult objects.
+
+    This function takes a list of AgentMetricResult objects and returns a list of MetricThreshold objects.
+    It aggregates thresholds if all AgentMetricResult objects have identical sets of thresholds.
+    If the list is empty, it returns an empty list. If there's only one AgentMetricResult, it returns its thresholds.
+
+    Parameters:
+    metric_results (List[AgentMetricResult]): A list of AgentMetricResult objects.
+
+    Returns:
+    List[MetricThreshold]: A list of MetricThreshold objects, either aggregated or an empty list if thresholds do not match.
+    """
+
+    if not metric_results:
+        return []
+
+    if len(metric_results) == 1:
+        return metric_results[0].thresholds
+
+    first_thresholds = set(metric_results[0].thresholds)
+    for metric_result in metric_results[1:]:
+        if first_thresholds != set(metric_result.thresholds):
+            logger.warning(
+                f"Did not get matching thresholds for {metric_results[0].name} metric. Not aggregating.")
+            return []
+
+    return metric_results[0].thresholds
+
+
+def __get_aggregation_result(metric_results: List[AgentMetricResult]) -> AggregateAgentMetricResult | None:
+    values, labels = [], []
+
+    for r in metric_results:
+        if r.value is not None:
+            values.append(r.value)
+        if r.label is not None:
+            labels.append(r.label)
+
+    value, mean, min_val, max_val, labels_count, percentiles = None, None, None, None, None, None
+    if values:
+        mean = sum(values) / len(values)
+        min_val = min(values)
+        max_val = max(values)
+        value = mean
+        if len(values) > 1:
+            # Calculate all percentiles in a single call
+            percentile_values = np.percentile(
+                values, [25, 50, 75, 90, 95, 99])
+            percentiles = {
+                "25": percentile_values[0],
+                "50": percentile_values[1],
+                "75": percentile_values[2],
+                "90": percentile_values[3],
+                "95": percentile_values[4],
+                "99": percentile_values[5]
+            }
+
+    if labels:
+        labels_count = dict(Counter(labels))
+
+    combined_thresholds = get_aggregated_thresholds(
+        metric_results=metric_results)
+    first_metric_result = metric_results[0]
+    return AggregateAgentMetricResult(name=first_metric_result.name,
+                                      value_type=first_metric_result.value_type,
+                                      display_name=first_metric_result.display_name,
+                                      thresholds=combined_thresholds,
+                                      method=first_metric_result.method,
+                                      provider=first_metric_result.provider,
+                                      node_name=first_metric_result.node_name,
+                                      applies_to=first_metric_result.applies_to,
+                                      group=first_metric_result.group,
+                                      value=mean,
+                                      min=min_val,
+                                      max=max_val,
+                                      count=len(metric_results),
+                                      percentiles=percentiles,
+                                      individual_results=metric_results)
+
+
+def __compute_aggregated_metrics_results(metrics_result: List[AgentMetricResult],
+                                         nodes: List[Node],
+                                         include_individual_results: bool = True) -> List[AggregateAgentMetricResult]:
+
+    nodes_result_group, message_result_group, conversation_result_map = __get_grouped_metrics_result(
+        metrics_result)
+
+    aggregated_results = []
+    aggregated_results.extend(__get_aggregated_node_metrics(
+        include_individual_results, nodes, nodes_result_group))
+    aggregated_results.extend(
+        __get_aggregated_metrics(message_result_group))
+    aggregated_results.extend(
+        __get_aggregated_metrics(conversation_result_map))
+
+    return aggregated_results
+
+
+def __get_aggregated_metrics(message_results):
+    aggregated_results = []
+    # Aggregate message or conversation level metrics
+    for values in list(message_results.values()):
+        aggregated_result = __get_aggregation_result(
+            values)
+        if aggregated_result:
+            aggregated_results.append(aggregated_result)
+    return aggregated_results
+
+
+def __get_grouped_metrics_result(metrics_result):
+    """
+    Group the metrics results based on node and message.
+    """
+    nodes_result_map, message_result_map = {}, {}
+    conversation_result_map = defaultdict(list)
+    conversation_metrics = defaultdict(lambda: defaultdict(float))
+    for mr in metrics_result:
+        key = mr.name+"_"+mr.method if mr.method else mr.name
+        if mr.applies_to == "node":
+            if mr.node_name in nodes_result_map:
+                if key in nodes_result_map[mr.node_name]:
+                    nodes_result_map[mr.node_name][key].append(mr)
+                else:
+                    nodes_result_map[mr.node_name][key] = [mr]
+            else:
+                nodes_result_map[mr.node_name] = {
+                    key: [mr]
+                }
+        elif mr.applies_to == "message":
+            if key in message_result_map:
+                message_result_map[key].append(mr)
+            else:
+                message_result_map[key] = [mr]
+            if key in ("duration", "cost", "input_token_count", "output_token_count"):
+                conversation_metrics[mr.conversation_id][key] += mr.value
+    for conversation_id, metric_value in conversation_metrics.items():
+        for metric, value in metric_value.items():
+            conversation_result_map[metric].append(AgentMetricResult(name=metric,
+                                                                     value=value,
+                                                                     display_name=metric,
+                                                                     group=MetricGroup.PERFORMANCE.value if metric == "duration" else MetricGroup.USAGE.value,
+                                                                     message_id=None,
+                                                                     applies_to="conversation",
+                                                                     conversation_id=conversation_id))
+
+    return nodes_result_map, message_result_map, dict(conversation_result_map)
+
+
+def __get_aggregated_node_metrics(include_individual_results, nodes, nodes_results):
+    aggregated_results = []
+
+    # Create node metrics dict for easy access to metrics
+    node_to_metrics = {}
+    for n in nodes:
+        mts = {}
+        for mc in n.metrics_configurations:
+            for m in mc.metrics:
+                mts[m.id] = m
+        node_to_metrics[n.name] = mts
+
+    # Aggregate node level metrics
+    for node, node_metrics in nodes_results.items():
+        for metric_key, values in node_metrics.items():
+            aggregated_result = None
+            metric_obj = node_to_metrics.get(node, {}).get(metric_key)
+
+            if isinstance(metric_obj, LLMValidationMetric):
+                # convert metrics result from AgentMetricResult to RecordMetricResult used by the metric
+                aggregated_result = __get_llm_validation_metric_aggregation_result(
+                    include_individual_results, values, metric_obj)
+            else:
+                aggregated_result = __get_aggregation_result(
+                    values)
+            if aggregated_result:
+                aggregated_results.append(aggregated_result)
+    return aggregated_results
+
+
+def __get_llm_validation_metric_aggregation_result(include_individual_results, values, metric_obj):
+    record_level_metrics = [RecordMetricResult(
+        **v.__dict__, record_id=v.message_id) for v in values]
+    aggregated_result = metric_obj.get_aggregated_results_from_individual_results(
+        record_level_metrics)
+
+    # convert updated record results to AgentMetricResult
+    updated_record_level_metrics = aggregated_result.record_level_metrics
+    agent_individual_results = []
+    for record_result, agent_result in zip(updated_record_level_metrics, values):
+        args = {**agent_result.__dict__,
+                **record_result.__dict__}
+        agent_individual_results.append(
+            AgentMetricResult(**args))
+
+    if aggregated_result:
+        # convert AggregateMetricResult to AggregateAgentMetricResult
+        mv = values[0]
+
+        # Calculate percentiles if we have enough data points
+        percentiles = None
+
+        if len(agent_individual_results) > 1:
+            # Extract values for percentile calculation
+            valid_values = [
+                r.value for r in agent_individual_results if r.value is not None]
+            if valid_values and all(isinstance(v, (int, float)) for v in valid_values):
+                # Calculate all percentiles in a single call
+                percentile_values = np.percentile(
+                    valid_values, [25, 50, 75, 90, 95, 99])
+
+                percentiles = {
+                    "25": percentile_values[0],
+                    "50": percentile_values[1],
+                    "75": percentile_values[2],
+                    "90": percentile_values[3],
+                    "95": percentile_values[4],
+                    "99": percentile_values[5]
+                }
+
+        aggregated_result = AggregateAgentMetricResult(
+            name=mv.name,
+            method=mv.method,
+            provider=mv.provider,
+            node_name=mv.node_name,
+            applies_to=mv.applies_to,
+            group=mv.group,
+            value=aggregated_result.mean,
+            min=aggregated_result.min,
+            max=aggregated_result.max,
+            count=aggregated_result.total_records,
+            percentiles=percentiles,
+            individual_results=copy.deepcopy(
+                agent_individual_results) if include_individual_results else [],
+            additional_info=copy.deepcopy(
+                aggregated_result.additional_info)
+        )
+    return aggregated_result
+
+
+def get_agentic_evaluation_result(metrics_result: list[AgentMetricResult], nodes: list[Node] = []) -> AgenticEvaluationResult:
+    aggregated_metrics_results = __compute_aggregated_metrics_results(
+        metrics_result, nodes)
+    metrics_result = []
+    for amr in aggregated_metrics_results:
+        metrics_result.extend(amr.individual_results)
+
+    return AgenticEvaluationResult(metrics_results=metrics_result,
+                                   aggregated_metrics_results=aggregated_metrics_results)
+
+
+def get_summaries(individual_metric_values: list):
+    """
+    Calculates statistical summaries for a list of numeric metric values.
+
+    Args:
+        individual_metric_values (list): A list of numeric values representing individual
+                                       metrics. May contain None values which will be filtered out.
+
+    Returns:
+        dict: A dictionary containing the following statistical summaries:
+            - "metric_value" (float): Mean of the values (same as "mean")
+            - "mean" (float): Arithmetic mean of the values
+            - "min" (float): Minimum value in the dataset
+            - "max" (float): Maximum value in the dataset
+            - "std" (float): Standard deviation of the values
+            - "percentiles" (dict): Dictionary containing percentile values with keys:
+                - "25": 25th percentile of the values
+                - "50": 50th percentile (median) of the values
+                - "75": 75th percentile of the values
+                - "90": 90th percentile of the values
+                - "95": 95th percentile of the values
+                - "99": 99th percentile of the values
+
+            If input is empty or contains only None values, returns:
+            {"metric_value": 0, "mean": 0, "min": 0, "max": 0, "std": 0,
+             "percentiles": {"25": 0, "50": 0, "75": 0, "90": 0, "95": 0, "99": 0}}
+    """
+    individual_metric_values = [
+        ele for ele in individual_metric_values if ele is not None]
+
+    if individual_metric_values is None or len(individual_metric_values) == 0:
+        return {
+            "metric_value": 0,
+            "mean": 0,
+            "min": 0,
+            "max": 0,
+            "std": 0,
+            "percentiles": {
+                "25": 0,
+                "50": 0,
+                "75": 0,
+                "90": 0,
+                "95": 0,
+                "99": 0
+            }
+        }
+    else:
+        # Calculate all percentiles in a single call
+        percentile_values = np.percentile(
+            individual_metric_values, [25, 50, 75, 90, 95, 99])
+
+        return {
+            "metric_value": np.mean(individual_metric_values).item(),
+            "mean": np.mean(individual_metric_values).item(),
+            "min": np.min(individual_metric_values).item(),
+            "max": np.max(individual_metric_values).item(),
+            "std": np.std(individual_metric_values).item(),
+            "percentiles": {
+                "25": percentile_values[0].item(),
+                "50": percentile_values[1].item(),
+                "75": percentile_values[2].item(),
+                "90": percentile_values[3].item(),
+                "95": percentile_values[4].item(),
+                "99": percentile_values[5].item()
+            }
+        }

ibm_watsonx_gov/utils/async_util.py

@@ -0,0 +1,62 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+
+import asyncio
+from typing import Any, Awaitable, Iterable
+
+
+def run_in_event_loop(task, *args, **kwargs):
+    """Run the given async task in an event loop, safely handling loop reuse."""
+    try:
+        event_loop = asyncio.get_running_loop()
+        # Use existing event loop and wait for the task to be executed.
+        import nest_asyncio
+        nest_asyncio.apply()
+        return event_loop.run_until_complete(task(*args, **kwargs))
+    except RuntimeError:
+        # No running loop, create one and close it when done
+        event_loop = asyncio.new_event_loop()
+        try:
+            return event_loop.run_until_complete(task(*args, **kwargs))
+        finally:
+            event_loop.close()
+
+
+async def gather_with_concurrency(
+    coros: Iterable[Awaitable],
+    return_exceptions: bool = False,
+    max_concurrency: int = 10,
+) -> Any:
+    semaphore = asyncio.Semaphore(max_concurrency)
+
+    async def safe_coroutine_fn(fn):
+        async with semaphore:
+            return await fn
+
+    tasks = [asyncio.create_task(safe_coroutine_fn(fn)) for fn in coros]
+    try:
+        return await asyncio.gather(*tasks, return_exceptions=return_exceptions)
+    except Exception as ex:
+        for task in tasks:
+            task.cancel()
+        raise ex
+
+
+def start_event_loop_run_func(func, data):
+    """
+    Create a wrapper function to start the event loop in the thread as unitxt LiteLLMInference fails without it.
+    """
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    try:
+        return func(data)
+    finally:
+        loop.close()
+        asyncio.set_event_loop(None)

ibm_watsonx_gov/utils/authenticator.py

@@ -0,0 +1,144 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import json
+
+from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
+from ibm_watsonx_gov.utils.rest_util import RestUtil
+
+
+class Authenticator:
+    """
+    Helper class to authenticate with IBM Cloud and CPD
+    """
+
+    def __init__(self, credentials: dict, use_cpd: bool, use_ssl: bool) -> None:
+        """
+        Initialize the authenticator object
+
+        Args:
+            credentials (dict): A dictionary containing the necessary credentials for authentication.
+            use_cpd (bool): A boolean indicating whether to authenticate with CPD or IBM Cloud.
+        """
+        self.__credentials: dict[str, str] = credentials
+        self.__use_cpd: bool = use_cpd
+        self.__iam_token: str = None
+        self.__use_ssl: bool = use_ssl
+        self.logger = GovSDKLogger.get_logger(__name__)
+
+    def authenticate(self) -> str:
+        """
+        Function to complete the authentication flow with either IBM Cloud or CPD based
+        on the configuration. This will set self.__iam_token and return the token to the user
+        """
+        self.logger.info("Authenticating the client")
+        if self.__use_cpd:
+            self.logger.info("Authenticating the client with CPD")
+            self.__iam_token = self.__get_iam_token_cpd()
+        else:
+            self.logger.info("Authenticating the client with ibm cloud")
+            self.__iam_token = self.__get_iam_token_cloud()
+
+        self.logger.info("Client authenticated successfully")
+        return self.__iam_token
+
+    def get_iam_token(self) -> str:
+        """
+        This function retrieves an IAM token from the instance variables. If the token does not exist, it raises an exception.
+
+        Returns:
+            str: IAM token
+        """
+        if not self.__iam_token:
+            message = "Not authenticated yet."
+            self.logger.error(message)
+            raise Exception(message)
+        return self.__iam_token
+
+    def __get_iam_token_cloud(self) -> None:
+        """
+        Method to authenticate the client with ibm cloud.
+        """
+        self.logger.info("Authenticating using cloud credentials")
+
+        headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+        }
+
+        data = {
+            "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+            "response_type": "cloud_iam",
+            "apikey": self.__credentials["apikey"],
+        }
+
+        try:
+            response = RestUtil.request_with_retry().post(
+                url=f"{self.__credentials['iam_url']}/identity/token",
+                data=data,
+                headers=headers,
+                allow_redirects=True,
+                verify=self.__use_ssl,
+            )
+            response.raise_for_status()
+        except Exception as e:
+            message = f"Failed to authenticate. {e}"
+            self.logger.error(message)
+            raise Exception(message)
+
+        try:
+            json_response = response.json()
+            return json_response["access_token"]
+        except Exception as e:
+            message = f"Failed to parse authentication response. {e}"
+            self.logger.error(message)
+            raise Exception(message)
+
+    def __get_iam_token_cpd(self) -> None:
+        """
+        Method to authenticate the client with CPD.
+        """
+        self.logger.info("Authenticating using cpd credentials")
+
+        headers = {
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }
+
+        data = {
+            "username": self.__credentials["username"],
+        }
+
+        # Check if the authentication is done using password or api key and it to the payload
+        if "password" in self.__credentials.keys():
+            data["password"] = self.__credentials["password"]
+        elif "api_key" in self.__credentials.keys():
+            data["api_key"] = self.__credentials["api_key"]
+
+        try:
+            response = RestUtil.request_with_retry().post(
+                url=f"{self.__credentials['url']}/icp4d-api/v1/authorize",
+                data=json.dumps(data).encode("utf-8"),
+                headers=headers,
+                allow_redirects=True,
+                verify=self.__use_ssl,
+            )
+            response.raise_for_status()
+        except Exception as e:
+            message = f"Failed to authenticate. {e}"
+            self.logger.error(message)
+            raise Exception(message)
+
+        try:
+            json_response = response.json()
+            return json_response["token"]
+        except Exception as e:
+            message = f"Failed to parse authentication response. {e}"
+            self.logger.error(message)
+            raise Exception(message)

ibm_watsonx_gov/utils/constants.py

@@ -0,0 +1,15 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+
+from typing import Literal
+
+OOTB_TYPE = Literal["ootb"]
+CUSTOM_TYPE = Literal["custom"]
+METRIC_VALUE_TYPES = Literal["numeric", "categorical"]

ibm_watsonx_gov/utils/errors.py

@@ -0,0 +1,40 @@
+
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import sys
+from http import HTTPStatus
+
+from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
+
+logger = GovSDKLogger.get_logger(__name__)
+
+
+class ClientError(Exception):
+    def __init__(self, code, message, reason=None):
+        self.code = code
+        self.message = message
+        self.reason = reason
+        logger.debug(str(self.code) + ": " +
+                     str(self.message) + ('\nReason: ' + str(self.reason) if sys.exc_info()[0] is not None else ''))
+
+    def __str__(self):
+        return str(self.code) + ": " + str(self.message) + ('\nReason: ' + str(self.reason)
+                                                            if sys.exc_info()[0] is not None else '')
+
+
+class AuthorizationError(ClientError, ValueError):
+    def __init__(self, code, message, reason=None):
+        ClientError.__init__(self, code=code, message=message, reason=reason)
+
+
+class UnsupportedOperationError(ClientError, ValueError):
+    def __init__(self, message, reason=None):
+        ClientError.__init__(
+            self, code=HTTPStatus.NOT_IMPLEMENTED, message=message, reason=reason)

ibm_watsonx_gov/utils/gov_sdk_logger.py

@@ -0,0 +1,39 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import logging
+
+
+class GovSDKLogger:
+    """
+    Logger for watsonx governance SDK
+    """
+    DEFAULT_LOG_LEVEL = logging.WARN
+
+    @staticmethod
+    def get_logger(name):
+        """
+        Function the return a logger object.
+        Params:
+        name (str): The name of the logger.
+        Returns:
+        logging.Logger: A logger object
+        """
+        logger = logging.getLogger(name)
+        logger.propagate = False
+        if not logger.hasHandlers():
+            logger.setLevel(GovSDKLogger.DEFAULT_LOG_LEVEL)
+            logger.propagate = False
+            handler = logging.StreamHandler()
+            logger.addHandler(handler)
+            formatter = logging.Formatter(
+                "[%(asctime)s]-[%(name)s]-[ %(levelname)s ]-[Line %(lineno)d] ~~> %(message)s"
+            )
+            handler.setFormatter(formatter)
+        return logger