ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/ai_experiments/utils/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/ai_experiments/utils/ai_experiment_utils.py

@@ -0,0 +1,139 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from collections import defaultdict
+from typing import List
+
+import pandas as pd
+
+from ibm_watsonx_gov.entities.ai_experiment import Node
+
+AI_SERVICE_QUALITY = "ai_service_quality"
+CUSTOM_METRICS = "custom_metrics"
+
+
+class AIExperimentUtils:
+    """ Class for AI experiment related utility methods """
+
+    @classmethod
+    def construct_result_attachment_payload(cls, evaluation_results: dict, nodes: List[Node] = None):
+        """
+        Constructs payload for result attachment from experiment run's result
+        Args:
+            - evaluation_results: The dict of metrics from evaluators and custom.
+
+        Returns: The payload for result attachment and total number of records processed
+        """
+        attachment_payload = {}
+        node_name_to_id_map = {}
+        if nodes is not None:
+            node_name_to_id_map = {node.name: node.id for node in nodes}
+
+        if not evaluation_results:
+            raise ValueError("Evaluation result is empty or missing.")
+
+        total_records = 0
+        agent_evaluation_result = {
+            "evaluation_result": {}
+        }
+        node_level_results = []
+        node_name_to_node_object = {}
+
+        # Separating agent level and node level metrics using the applies_to field in the result.
+        for monitor_name, metric_results in evaluation_results.items():
+            # If result for some monitor is empty, skipping it
+            if not metric_results:
+                continue
+            agent_level_metrics = []
+            node_level_metrics = defaultdict(list)
+
+            for metric_result in metric_results:
+                result_applies_to = metric_result.get(
+                    "applies_to", "conversation")
+                metric_id = metric_result.get("name", "")
+                metric_result["id"] = metric_id
+                metric_result["name"] = metric_id.capitalize().replace(
+                    "_", " ")
+                # If thresholds exist, add them to the run result
+                thresholds = metric_result.pop("thresholds", [])
+                if thresholds:
+                    for threshold in thresholds:
+                        metric_result[threshold.get(
+                            "type")] = threshold.get("value")
+
+                if result_applies_to == "node":
+                    node_name = metric_result.pop("node_name", "")
+                    if node_name and result_applies_to == "node":
+                        node_level_metrics[node_name].append(metric_result)
+
+                # agentic_result_components ["conversation", "message"]
+                else:
+                    if total_records == 0:
+                        total_records = metric_result.get("count")
+                    metric_id = f"{metric_id}_for_{result_applies_to}" if monitor_name != CUSTOM_METRICS else metric_id
+                    metric_result["id"] = metric_id
+                    metric_result["name"] = metric_id.capitalize().replace(
+                        "_", " ")
+                    agent_level_metrics.append(metric_result)
+
+            # Constructing the agent level metrics result
+            agent_evaluation_result["evaluation_result"][monitor_name] = {
+                "metric_groups": cls.get_result_with_metric_groups(agent_level_metrics)
+            }
+
+            # Constructing the node level metrics result
+            for node_name, metrics in node_level_metrics.items():
+                metric_groups_result = cls.get_result_with_metric_groups(
+                    metrics)
+
+                if node_name not in node_name_to_node_object:
+                    node_obj = {
+                        "type": "tool",
+                        "id": node_name_to_id_map.get(node_name, node_name),
+                        "name": node_name,
+                        "evaluation_result": {}
+                    }
+
+                    node_obj["evaluation_result"][monitor_name] = {
+                        "metric_groups": metric_groups_result
+                    }
+                    node_name_to_node_object[node_name] = node_obj
+                    node_level_results.append(node_obj)
+
+                else:
+                    node_obj = node_name_to_node_object[node_name]
+                    node_obj["evaluation_result"][monitor_name] = {
+                        "metric_groups": metric_groups_result
+                    }
+
+        attachment_payload = {
+            "ai_application": agent_evaluation_result,
+            "nodes": node_level_results
+        }
+        return attachment_payload, total_records
+
+    @classmethod
+    def get_result_with_metric_groups(cls, metrics: list) -> list:
+        """
+        Organises the result based on metric groups
+        Args:
+            - metrics: The list of metrics
+
+        Returns: The list containing metric groups and corresponding metrics for each group
+        """
+        metric_groups_map = defaultdict(list)
+        for metric in metrics:
+            metric_group = metric.pop(
+                "group", "Other metrics").capitalize().replace("_", " ")
+            metric_groups_map[metric_group].append(metric)
+
+        metric_groups_result = [
+            {"name": group_name, "metrics": group_metrics} for group_name, group_metrics in metric_groups_map.items()]
+
+        return metric_groups_result

ibm_watsonx_gov/clients/api_client.py

@@ -0,0 +1,99 @@
+
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from ibm_cloud_sdk_core.authenticators import (BearerTokenAuthenticator,
+                                               CloudPakForDataAuthenticator,
+                                               IAMAuthenticator)
+from ibm_watson_openscale import APIClient as WOSClient
+from ibm_watsonx_gov.clients.wx_ai_client import WXAIClient
+from ibm_watsonx_gov.entities.credentials import Credentials
+from ibm_watsonx_gov.entities.enums import Region
+from ibm_watsonx_gov.utils.async_util import run_in_event_loop
+from ibm_watsonx_gov.utils.segment_batch_manager import SegmentBatchManager
+from ibm_watsonx_gov.utils.url_mapping import (WATSONX_REGION_URLS,
+                                               WOS_URL_MAPPING)
+
+
+class APIClient():
+    """
+    The IBM watsonx.governance sdk client. It is required to access the watsonx.governance APIs.
+    """
+
+    def __init__(self, credentials: Credentials | None = None):
+        self.credentials = credentials
+        self.is_cpd = False
+        if self.credentials.token:
+            authenticator = BearerTokenAuthenticator(
+                bearer_token=self.credentials.token)
+            if self.credentials.version:
+                self.is_cpd = True
+        elif self.credentials.version:
+            authenticator = CloudPakForDataAuthenticator(url=self.credentials.url,
+                                                         username=self.credentials.username,
+                                                         password=self.credentials.password,
+                                                         apikey=self.credentials.api_key,
+                                                         disable_ssl_verification=self.credentials.disable_ssl
+                                                         )
+            self.is_cpd = True
+        elif self.credentials.region == Region.AP_SOUTH.value:
+            from ibm_cloud_sdk_core.authenticators import MCSPV2Authenticator
+            iam_url = WATSONX_REGION_URLS.get(
+                self.credentials.region).iam_url
+            authenticator = MCSPV2Authenticator(apikey=self.credentials.api_key, url=iam_url,
+                                                scope_id=self.credentials.scope_id, scope_collection_type=self.credentials.scope_collection_type)
+        else:
+            iam_url = WATSONX_REGION_URLS.get(
+                self.credentials.region).iam_url
+            authenticator = IAMAuthenticator(apikey=self.credentials.api_key,
+                                             url=iam_url,
+                                             disable_ssl_verification=self.credentials.disable_ssl)
+            if self.credentials.region in [Region.US_GOV_EAST1.value, "govcloudpreprod"]:
+                authenticator.token_manager.OPERATION_PATH = "/api/rest/mcsp/apikeys/token"
+                authenticator.token_manager.token_name = "token"
+
+        try:
+            self.wos_client = WOSClient(
+                authenticator=authenticator,
+                service_url=self.credentials.url,
+                service_instance_id=self.credentials.service_instance_id,
+            )
+        except Exception as e:
+            if "Multiple service instance id exists" in str(e) or "You are not authorized to access AI OpenScale instance" in str(e):
+                raise e
+            else:
+                if self.is_cpd:
+                    dai_url = self.credentials.url
+                else:
+                    dai_url = WATSONX_REGION_URLS.get(
+                        self.credentials.region).dai_url
+                self.wos_client = WXAIClient(service_url=dai_url,
+                                             authenticator=authenticator,
+                                             disable_ssl_verification=self.credentials.disable_ssl,
+                                             is_cpd=self.is_cpd)
+                self.wos_client.validate_user()
+
+        # Adding segment event
+        segment_manager = SegmentBatchManager(api_client=self)
+        run_in_event_loop(segment_manager.track_event, {
+                          "objectType": "API Client Initialization"})
+
+    @property
+    def credentials(self):
+        return self._credentials
+
+    @credentials.setter
+    def credentials(self, credentials):
+        """
+        Setter for credentials object. If not provided, it will create a credentials object from environment variables.
+        """
+        if not credentials:
+            self._credentials = Credentials.create_from_env()
+        else:
+            self._credentials = credentials

ibm_watsonx_gov/clients/segment_client.py

@@ -0,0 +1,46 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import aiohttp
+from ibm_watsonx_gov.utils.gov_sdk_logger import GovSDKLogger
+from ibm_watsonx_gov.utils.python_utils import get_authenticator_token
+
+logger = GovSDKLogger.get_logger(__name__)
+
+
+class SegmentClient():
+
+    def __init__(self, api_client):
+        self.api_client = api_client
+
+    async def trigger_segment_endpoint(self, segment_data):
+        try:
+            segment_publish_url = "{0}/v2/segment/events".format(
+                self.api_client.service_url)
+
+            iam_headers = {
+                "Content-Type": "application/json",
+                "accept": "application/json",
+                "Authorization": f"Bearer {get_authenticator_token(self.api_client.authenticator)}"
+            }
+            async with aiohttp.ClientSession() as session:
+                async with session.post(
+                    url=segment_publish_url,
+                    headers=iam_headers,
+                    json=segment_data,
+                    ssl=False
+                ) as response:
+                    text = await response.text()
+                    logger.info(
+                        f"Segment response: {response.status}, Body: {text}"
+                    )
+                    return response.status == 202
+        except Exception as ex:
+            logger.error(f"Failed to send segment events. Details: {ex}")
+            return False

ibm_watsonx_gov/clients/wx_ai_client.py

@@ -0,0 +1,87 @@
+
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import json
+
+import jwt
+import requests
+from ibm_cloud_sdk_core.authenticators import (BearerTokenAuthenticator,
+                                               CloudPakForDataAuthenticator,
+                                               IAMAuthenticator)
+from jwt.exceptions import DecodeError
+from requests.exceptions import HTTPError
+
+from ibm_watsonx_gov.utils.authenticator import Authenticator
+
+
+class WXAIClient():
+    """
+    Client class to validate user entitlements
+    """
+
+    def __init__(self, service_url: str, authenticator: CloudPakForDataAuthenticator | IAMAuthenticator | BearerTokenAuthenticator, disable_ssl_verification: bool, is_cpd: bool):
+        self.service_url = service_url
+        self.authenticator = authenticator
+        self.disable_ssl_verification = disable_ssl_verification
+        self.is_cpd = is_cpd
+
+    def validate_user(self):
+        entitlements_url = f"{self.service_url}/v2/entitlements"
+        # create a token using the authenticator
+        if isinstance(self.authenticator, BearerTokenAuthenticator):
+            bearer_token = self.authenticator.bearer_token
+        elif self.is_cpd and not isinstance(self.authenticator, BearerTokenAuthenticator):
+            creds = {
+                "url": self.service_url,
+                "username": self.authenticator.token_manager.username
+            }
+            if self.authenticator.token_manager.password:
+                creds.update(
+                    {"password": self.authenticator.token_manager.password})
+            if self.authenticator.token_manager.apikey:
+                creds.update(
+                    {"api_key": self.authenticator.token_manager.apikey})
+
+            auth = Authenticator(
+                credentials=creds,
+                use_cpd=self.is_cpd,
+                use_ssl=(not self.disable_ssl_verification),
+            )
+            bearer_token = auth.authenticate()
+        else:
+            bearer_token = self.authenticator.token_manager.get_token()
+
+        if not self.is_cpd:
+            # decode the token and get bss
+            try:
+                decoded_token = jwt.decode(bearer_token, options={
+                                           "verify_signature": False})
+                bss_account_id = decoded_token["account"]["bss"]
+            except jwt.InvalidTokenError as e:
+                raise DecodeError(
+                    f"Failed while decoding the token. Token is invalid: {e}")
+
+            entitlements_url = f"{entitlements_url}?bss_account_id={bss_account_id}"
+
+        # call entitlements api and check for data_science_experience
+        response = requests.get(entitlements_url, headers={
+                                "Authorization": f"Bearer {bearer_token}"}, verify=not self.disable_ssl_verification)
+
+        if response.status_code == 200:
+            response_json = json.loads(response.text)
+            dsx = response_json.get("entitlements", {}).get(
+                "data_science_experience")
+            if not dsx:
+                raise ValueError("User is not authorized.")
+            else:
+                return True
+        else:
+            raise HTTPError(
+                f"Request failed with status code: {response.status_code}")

ibm_watsonx_gov/config/__init__.py

@@ -0,0 +1,14 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from ..entities.credentials import (Credentials, WxAICredentials,
+                                    WxGovConsoleCredentials)
+from .agentic_ai_configuration import AgenticAIConfiguration
+from .gen_ai_configuration import GenAIConfiguration
+from .model_risk_configuration import ModelRiskConfiguration

ibm_watsonx_gov/config/agentic_ai_configuration.py

@@ -0,0 +1,225 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import os
+from typing import Annotated, Optional
+
+from pydantic import BaseModel, Field, model_validator
+from typing_extensions import Self
+
+from ibm_watsonx_gov.config.gen_ai_configuration import GenAIConfiguration
+from ibm_watsonx_gov.entities.enums import EvaluatorFields
+
+
+class OTLPCollectorConfiguration(BaseModel):
+    """
+    Defines the OTLPCollectorConfiguration class.
+    It contains the configuration settings for the OpenTelemetry Protocol collector.
+
+    Examples:
+        1. Create OTLPCollectorConfiguration with default parameters
+            .. code-block:: python
+
+                oltp_config = OTLPCollectorConfiguration()
+
+        1. Create OTLPCollectorConfiguration by providing server endpoint details.
+            .. code-block:: python
+
+                oltp_config = OTLPCollectorConfiguration(app_name="app",
+                                                         endpoint="https://hostname/ml/v1/traces",
+                                                         timeout=10,
+                                                         headers={"Authorization": "Bearer token"})
+    """
+    app_name: Annotated[str | None,
+                        Field(title="App Name",
+                              description="Application name for tracing.")]
+    endpoint: Annotated[str | None,
+                        Field(title="OTLP Endpoint",
+                              description="The OTLP collector endpoint URL for sending trace data. Default value is 'http://localhost:4318/v1/traces'",
+                              default="http://localhost:4318/v1/traces")]
+    insecure: Annotated[bool | None,
+                        Field(title="Insecure Connection",
+                              description="Whether to disable TLS for the exporter (i.e., use an insecure connection). Default is False.",
+                              default=False)]
+    is_grpc: Annotated[bool | None,
+                       Field(title="Use gRPC",
+                             description="If True, use gRPC for exporting traces instead of HTTP. Default is False.",
+                             default=False)]
+    timeout: Annotated[int | None,
+                       Field(title="Timeout",
+                             description="Timeout in milliseconds for sending telemetry data to the collector. Default is 100ms.",
+                             default=100)]
+    headers: Annotated[dict[str, str] | None,
+                       Field(title="Headers",
+                             description="Headers needed to call the server.",
+                             default_factory=dict)]
+
+
+class TracingConfiguration(BaseModel):
+    """
+    Defines the tracing configuration class. 
+    Tracing configuration is required if the the evaluations are needed to be tracked in an experiment or if the agentic application traces should be sent to a Open Telemetry Collector.
+    One of project_id or space_id is required.
+    If the otlp_collector_config is provided, the traces are logged to Open Telemetry Collector, otherwise the traces are logged to file on disk.
+    If its required to log the traces to both collector and local file, provide the otlp_collector_config and set the flag log_traces_to_file to True.
+
+    Examples:
+        1. Create Tracing configuration to track the results in an experiment
+            .. code-block:: python
+
+                tracing_config = TracingConfiguration(project_id="...")
+                agentic_evaluator = AgenticEvaluator(tracing_configuration=tracing_config)
+                agentic_evaluator.track_experiment(name="my_experiment")
+                ...
+
+        2. Create Tracing configuration to send traces to collector
+            .. code-block:: python
+
+                oltp_collector_config = OTLPCollectorConfiguration(endpoint="http://hostname:4318/v1/traces")
+                tracing_config = TracingConfiguration(space_id="...",
+                                                      resource_attributes={
+                                                            "wx-deployment-id": deployment_id,
+                                                            "wx-instance-id": "wml-instance-id1",
+                                                            "wx-ai-service-id": "ai-service-id1"},
+                                                       otlp_collector_config=oltp_collector_config)
+                agentic_evaluator = AgenticEvaluator(tracing_configuration=tracing_config)
+                ...
+    """
+    project_id: Annotated[str | None,
+                          Field(title="Project ID",
+                                description="The project id.",
+                                default=None)]
+    space_id: Annotated[str | None,
+                        Field(title="Space ID",
+                              description="The space id.",
+                              default=None)]
+    resource_attributes: Annotated[dict[str, str] | None,
+                                   Field(title="Resource Attributes",
+                                         description="The resource attributes set in all the spans.",
+                                         default_factory=dict)]
+    otlp_collector_config: Annotated[OTLPCollectorConfiguration | None,
+                                     Field(title="OTLP Collector Config",
+                                           description="OTLP Collector configuration.",
+                                           default=None)]
+    log_traces_to_file: Annotated[bool | None,
+                                  Field(title="Log Traces to file",
+                                        description="The flag to enable logging of traces to a file. If set to True, the traces are logged to a file. Use the flag when its needed to log the traces to file and to be sent to the server simultaneously.",
+                                        default=False)]
+
+    @model_validator(mode="after")
+    def validate_fields(self) -> Self:
+        if not (self.project_id or self.space_id):
+            raise ValueError(
+                "The project or space id is required. Please provide one of them and retry.")
+
+        return self
+
+    @classmethod
+    def create_from_env(cls):
+        # TODO get the default value and validate the tracing url env variable
+        tracing_url = os.getenv("WATSONX_TRACING_URL") or ""
+        project_id = os.getenv("PROJECT_ID")
+        space_id = os.getenv("SPACE_ID")
+
+        return TracingConfiguration(
+            tracing_url=tracing_url,
+            project_id=project_id,
+            space_id=space_id
+        )
+
+    @property
+    def enable_server_traces(self) -> bool:
+        # Check if otlp_collector_config field was set
+        return self.otlp_collector_config is not None
+
+    @property
+    def enable_local_traces(self) -> bool:
+        # if otlp_collector_config is not set, enable local traces
+        return self.log_traces_to_file or self.otlp_collector_config is None
+
+
+class AgenticAIConfiguration(GenAIConfiguration):
+    """
+    Defines the AgenticAIConfiguration class.
+
+    The configuration interface for Agentic AI tools and applications.
+    This is used to specify the fields mapping details in the data and other configuration parameters needed for evaluation.
+
+    Examples:
+        1. Create configuration with default parameters
+            .. code-block:: python
+
+                configuration = AgenticAIConfiguration()
+
+        2. Create configuration with parameters
+            .. code-block:: python
+
+                configuration = AgenticAIConfiguration(input_fields=["input"], 
+                                                       output_fields=["output"])
+
+        2. Create configuration with dict parameters
+            .. code-block:: python
+
+                config = {"input_fields": ["input"],
+                          "output_fields": ["output"],
+                          "context_fields": ["contexts"],
+                          "reference_fields": ["reference"]}
+                configuration = AgenticAIConfiguration(**config)
+    """
+    message_id_field: Annotated[Optional[str], Field(title="Message id field",
+                                                     description="The message identifier field name. Default value is 'message_id'.",
+                                                     examples=[
+                                                         "message_id"],
+                                                     default="message_id")]
+
+    conversation_id_field: Annotated[Optional[str], Field(title="Conversation id field",
+                                                          description="The conversation identifier field name. Default value is 'conversation_id'.",
+                                                          examples=[
+                                                              "conversation_id"],
+                                                          default="conversation_id")]
+
+    @classmethod
+    def create_configuration(cls, *, app_config: Optional[Self],
+                             method_config: Optional[Self],
+                             defaults: list[EvaluatorFields],
+                             add_record_fields: bool = True) -> Self:
+        """
+        Creates a configuration object based on the provided parameters.
+
+        Args:
+            app_config(Optional[Self]): The application configuration.
+            method_config(Optional[Self]): The method configuration.
+            defaults(list[EvaluatorFields]): The default fields to include in the configuration.
+            add_record_fields(bool, optional): Whether to add record fields to the configuration. Defaults to True.
+
+        Returns:
+            Self: The created configuration object.
+        """
+
+        if method_config is not None:
+            return method_config
+
+        if app_config is not None:
+            return app_config
+
+        mapping = EvaluatorFields.get_default_fields_mapping()
+        config = {field.value: mapping[field] for field in defaults}
+
+        if not add_record_fields:
+            return cls(**config)
+
+        system_fields = [EvaluatorFields.RECORD_ID_FIELD,
+                         EvaluatorFields.RECORD_TIMESTAMP_FIELD,
+                         EvaluatorFields.MESSAGE_ID_FIELD,
+                         EvaluatorFields.CONVERSATION_ID_FIELD]
+        for field in system_fields:
+            if field not in defaults:
+                config[field.value] = EvaluatorFields.get_default_fields_mapping()[
+                    field]
+        return cls(**config)