ibm-watsonx-gov 1.3.3__cp313-cp313-win_amd64.whl

ibm_watsonx_gov/entities/metric.py

@@ -0,0 +1,211 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import asyncio
+import functools
+from abc import abstractmethod
+from typing import (TYPE_CHECKING, Annotated, List, Literal, Optional, Self,
+                    Union)
+
+import pandas as pd
+from pydantic import (BaseModel, Field, computed_field, field_serializer,
+                      field_validator, model_validator)
+
+from ibm_watsonx_gov.entities.base_classes import BaseMetric
+from ibm_watsonx_gov.entities.enums import MetricGroup, TaskType
+from ibm_watsonx_gov.entities.metric_threshold import MetricThreshold
+
+if TYPE_CHECKING:
+    from ibm_watsonx_gov.config import (AgenticAIConfiguration,
+                                        GenAIConfiguration)
+    from ibm_watsonx_gov.entities.evaluation_result import (
+        AggregateMetricResult, RecordMetricResult)
+
+
+class MappingItem(BaseModel):
+    """
+    The mapping details to be used for reading the values from the data.
+    """
+    name: Annotated[str,
+                    Field(title="Name",
+                          description="The name of the item.",
+                          examples=["input_text", "generated_text", "context", "ground_truth"])]
+    type_: Annotated[Literal["input", "output", "reference", "context", "tool_call", "start_time", "end_time", "input_token_count", "output_token_count", "model_usage_details", "status", "user_id", "target_component", "available_tools"],
+                     Field(title="Type",
+                           description="The type of the item.",
+                           examples=["input"],
+                           alias="type",
+                           serialization_alias="type")]
+    column_name: Annotated[Optional[str],
+                           Field(title="Column Name",
+                                 description="The column name in the tabular data to be used for reading the field value. Applicable for tabular source.", default=None)]
+    span_name: Annotated[Optional[str],
+                         Field(title="Span Name",
+                               description="The span name in the trace data to be used for reading the field value. Applicable for trace source.", default=None)]
+    attribute_name: Annotated[Optional[str],
+                              Field(title="Attribute Name",
+                                    description="The attribute name in the trace to be used for reading the field value. Applicable for trace source.", default=None)]
+    json_path: Annotated[Optional[str],
+                         Field(title="Json Path",
+                               description="The json path to be used for reading the field value from the attribute value. Applicable for trace source. If not provided, the span attribute value is read as the field value.", default=None)]
+    lookup_child_spans: Annotated[Optional[bool],
+                                  Field(title="Look up child spans",
+                                        description="The flag to indicate if all the child spans should be searched for the attribute value. Applicable for trace source.",
+                                        default=False)]
+
+
+class Mapping(BaseModel):
+    """
+    Defines the field mapping details to be used for computing a metric.
+    """
+    source: Annotated[Literal["trace", "tabular"],
+                      Field(title="Source",
+                            description="The source type of the data. Use trace if the data should be read from span in trace. Use tabular if the data is passed as a dataframe.",
+                            default="trace",
+                            examples=["trace", "tabular"])]
+    items: Annotated[list[MappingItem],
+                     Field(title="Mapping Items",
+                           description="The list of mapping items for the field. They are used to read the data from trace or tabular data for computing the metric.")]
+
+
+class TargetComponent(BaseModel):
+    type: Literal["string", "mapping"] = Field(
+        description="How the component is referenced. By `string` for directly providing the node names or by `mapping` for reading it from span attributes",
+        examples=["string", "mapping"]
+    )
+    value: Union[str, MappingItem] = Field(
+        description="The component’s value, either a node name represented as a string, or a MappingItem containing span and attribute details."
+    )
+
+    @field_validator("value")
+    def validate_value_based_on_type(cls, v, info):
+        type = info.data.get("type")
+        if type == "string" and not isinstance(v, str):
+            raise ValueError(
+                "Value must be a string when type is 'string'")
+        if type == "mapping" and not isinstance(v, MappingItem):
+            raise ValueError(
+                "Value must be a MappingItem when type is 'mapping'")
+        return v
+
+
+class GenAIMetric(BaseMetric):
+    """Defines the Generative AI metric interface"""
+    thresholds: Annotated[list[MetricThreshold],
+                          Field(description="The list of thresholds", default=[])]
+    tasks: Annotated[list[TaskType], Field(
+        description="The task types this metric is associated with.", frozen=True, default=[])]
+    group: Annotated[MetricGroup | None, Field(
+        description="The metric group this metric belongs to.", frozen=True, default=None)]
+    is_reference_free: Annotated[bool, Field(
+        description="Decides whether this metric needs a reference for computation", frozen=True, default=True)]
+    method: Annotated[
+        str | None,
+        Field(description="The method used to compute the metric.",
+              default=None)]
+    metric_dependencies: Annotated[list["GenAIMetric"], Field(
+        description="Metrics that needs to be evaluated first", default=[])]
+    applies_to: Annotated[Optional[str],
+                          Field(title="Applies to",
+                                description="The tag to indicate for which the metric is applied to. Used for agentic application metric computation.",
+                                examples=["message",
+                                          "conversation", "sub_agent"],
+                                default="message")]
+    target_component: Annotated[Optional[TargetComponent],
+                                Field(
+        title="Target Component",
+        description="The specific application component (node) where this metric is computed. Used for agentic application metric computation.",
+        examples=["Retrieval Node", "Context Node", "Generation Node"],
+        default=None
+    )]
+    mapping: Annotated[Optional[Mapping],
+                       Field(title="Mapping",
+                             description="The data mapping details for the metric which are used to read the values needed to compute the metric.",
+                             default=None,
+                             examples=Mapping(items=[MappingItem(name="input_text",
+                                                                 type="input",
+                                                                 span_name="LangGraph.workflow",
+                                                                 attribute_name="traceloop.entity.input",
+                                                                 json_path="$.inputs.input_text"),
+                                                     MappingItem(name="generated_text",
+                                                                 type="output",
+                                                                 span_name="LangGraph.workflow",
+                                                                 attribute_name="traceloop.entity.output",
+                                                                 json_path="$.outputs.generated_text")])
+                             )]
+
+    @field_serializer("metric_dependencies", when_used="json")
+    def metric_dependencies_serializer(self, metric_dependencies: list["GenAIMetric"]):
+        return [metric.model_dump(mode="json") for metric in metric_dependencies]
+
+    @computed_field(return_type=str)
+    @property
+    def id(self):
+        if self._id is None:
+            self._id = self.name + (f"_{self.method}" if self.method else "")
+        return self._id
+
+    @model_validator(mode="after")
+    def validate(self) -> Self:
+        if not self.display_name:
+            words = self.name.split('_')
+            self.display_name = ' '.join(word.capitalize() for word in words)
+
+        return self
+
+    @abstractmethod
+    def evaluate(self, data: pd.DataFrame | dict,
+                 configuration: "GenAIConfiguration | AgenticAIConfiguration",
+                 **kwargs) -> "AggregateMetricResult":
+        raise NotImplementedError
+
+    async def evaluate_async(
+        self,
+        data: pd.DataFrame | dict,
+        configuration: "GenAIConfiguration | AgenticAIConfiguration",
+        **kwargs,
+    ) -> "AggregateMetricResult":
+        loop = asyncio.get_event_loop()
+        # If called as async, run it in a separate thread
+        return await loop.run_in_executor(
+            None,
+            functools.partial(
+                self.evaluate,
+                data=data,
+                configuration=configuration,
+                **kwargs,
+            )
+        )
+
+    def info(self):
+        pass
+
+    def get_aggregated_results_from_individual_results(self, record_results: List["RecordMetricResult"]):
+        from ibm_watsonx_gov.entities.evaluation_result import \
+            AggregateMetricResult
+
+        values = [record.value for record in record_results]
+        record_result = record_results[0]
+        mean = sum(values) / len(values)
+        return AggregateMetricResult(
+            name=record_result.name,
+            method=record_result.method,
+            provider=record_result.provider,
+            group=record_result.group,
+            value=mean,
+            total_records=len(record_results),
+            record_level_metrics=record_results,
+            min=min(values),
+            max=max(values),
+            mean=mean,
+        )
+
+
+class PredictiveAIMetric(BaseMetric):
+    pass

ibm_watsonx_gov/entities/metric_threshold.py

@@ -0,0 +1,36 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Literal
+
+from pydantic import BaseModel, Field
+
+
+class MetricThreshold(BaseModel):
+    """
+    The class that defines the threshold for a metric.
+    """
+    type: Annotated[Literal["lower_limit", "upper_limit"], Field(
+        description="Threshold type. One of 'lower_limit', 'upper_limit'")]
+    value: Annotated[float, Field(
+        title="Threshold value", description="The value of metric threshold", default=0)]
+
+    def __eq__(self, other):
+        """Check if two MetricThreshold objects are equal."""
+        if not isinstance(other, MetricThreshold):
+            return False
+        return self.type == other.type and self.value == other.value
+
+    def __ne__(self, other):
+        """Check if two MetricThreshold objects are not equal."""
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        """Make the object hashable so it can be used in sets and as dict keys."""
+        return hash((self.type, self.value))

ibm_watsonx_gov/entities/model_provider.py

@@ -0,0 +1,329 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated
+
+from ibm_watsonx_gov.entities.credentials import (AWSBedrockCredentials,
+                                                  AzureOpenAICredentials,
+                                                  GoogleAIStudioCredentials,
+                                                  OpenAICredentials,
+                                                  PortKeyCredentials,
+                                                  RITSCredentials,
+                                                  VertexAICredentials,
+                                                  WxAICredentials,
+                                                  WxoAIGatewayCredentials)
+from ibm_watsonx_gov.entities.enums import ModelProviderType
+from pydantic import BaseModel, Field, model_validator
+from typing_extensions import Self
+
+
+class ModelProvider(BaseModel):
+    type: Annotated[
+        ModelProviderType, Field(
+            description="The type of model provider.")
+    ]
+
+
+class WxAIModelProvider(ModelProvider):
+    """
+    This class represents a model provider configuration for IBM watsonx.ai. It includes the provider type and
+    credentials required to authenticate and interact with the watsonx.ai platform. If credentials are not explicitly
+    provided, it attempts to load them from environment variables.
+
+    Examples:
+        1. Create provider using credentials object:
+            .. code-block:: python
+
+                credentials = WxAICredentials(
+                    url="https://us-south.ml.cloud.ibm.com",
+                    api_key="your-api-key"
+                )
+                provider = WxAIModelProvider(credentials=credentials)
+
+        2. Create provider using environment variables:
+            .. code-block:: python
+
+                import os
+
+                os.environ['WATSONX_URL'] = "https://us-south.ml.cloud.ibm.com"
+                os.environ['WATSONX_APIKEY'] = "your_api_key"
+
+                provider = WxAIModelProvider()
+    """
+
+    type: Annotated[
+        ModelProviderType,
+        Field(
+            description="The type of model provider.",
+            default=ModelProviderType.IBM_WATSONX_AI,
+            frozen=True
+        )
+    ]
+    credentials: Annotated[
+        WxAICredentials | None,
+        Field(
+            default=None,
+            description="The credentials used to authenticate with watsonx.ai. If not provided, they will be loaded from environment variables."
+        )
+    ]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> Self:
+        if self.credentials is None:
+            try:
+                self.credentials = WxAICredentials.create_from_env()
+            except ValueError:
+                self.credentials = None
+        return self
+
+
+class OpenAIModelProvider(ModelProvider):
+    type: Annotated[ModelProviderType,
+                    Field(description="The type of model provider.",
+                          default=ModelProviderType.OPENAI, frozen=True)]
+    credentials: Annotated[OpenAICredentials | None, Field(
+        description="OpenAI credentials. This can also be set by using `OPENAI_API_KEY` environment variable.", default=None)]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> Self:
+        if self.credentials is None:
+            self.credentials = OpenAICredentials.create_from_env()
+        return self
+
+
+class PortKeyModelProvider(ModelProvider):
+    type: Annotated[ModelProviderType,
+                    Field(description="The type of model provider.",
+                          default=ModelProviderType.PORTKEY, frozen=True)]
+    credentials: Annotated[PortKeyCredentials | None, Field(
+        description="PortKey credentials.", default=None)]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> Self:
+        if self.credentials is None:
+            self.credentials = PortKeyCredentials.create_from_env()
+        return self
+
+
+class AzureOpenAIModelProvider(ModelProvider):
+    type: Annotated[ModelProviderType,
+                    Field(description="The type of model provider.",
+                          default=ModelProviderType.AZURE_OPENAI, frozen=True)]
+    credentials: Annotated[AzureOpenAICredentials | None, Field(
+        description="Azure OpenAI credentials.", default=None
+    )]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> Self:
+        if self.credentials is None:
+            self.credentials = AzureOpenAICredentials.create_from_env()
+        return self
+
+
+class RITSModelProvider(ModelProvider):
+    type: Annotated[ModelProviderType,
+                    Field(description="The type of model provider.",
+                          default=ModelProviderType.RITS, frozen=True)]
+    credentials: Annotated[RITSCredentials | None, Field(
+        description="RITS credentials.", default=None
+    )]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> Self:
+        if self.credentials is None:
+            self.credentials = RITSCredentials.create_from_env()
+        return self
+
+
+class VertexAIModelProvider(ModelProvider):
+    """
+    Represents a model provider using Vertex AI.
+
+    Examples:
+        1. Create provider using credentials object:
+            .. code-block:: python
+
+                provider = VertexAIModelProvider(
+                    credentials=VertexAICredentials(
+                        credentials_path="path/to/key.json",
+                        project_id="your-project",
+                        location="us-central1" 
+                    )
+                )
+
+        2. Create provider using environment variables:
+            .. code-block:: python
+
+                os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = "/path/to/service_account.json"
+                os.environ['GOOGLE_CLOUD_PROJECT'] = "your-project"
+                os.environ['GOOGLE_CLOUD_LOCATION'] = "us-central1" # This is optional field, by default us-central1 location is selected
+
+                provider = VertexAIModelProvider()
+    """
+
+    type: Annotated[ModelProviderType, Field(
+        description="The type of model provider.",
+        default=ModelProviderType.VERTEX_AI,
+        frozen=True
+    )]
+    credentials: Annotated[VertexAICredentials | None, Field(
+        description="Vertex AI credentials.", default=None
+    )]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> "VertexAIModelProvider":
+        if self.credentials is None:
+            self.credentials = VertexAICredentials.create_from_env()
+        return self
+
+
+class GoogleAIStudioModelProvider(ModelProvider):
+    """
+    Represents a model provider using Google AI Studio.
+
+    Examples:
+        1. Create provider using credentials object:
+            .. code-block:: python
+
+                provider = GoogleAIStudioModelProvider(
+                    credentials=GoogleAIStudioCredentials(api_key="api-key")
+                )
+
+        2. Create provider using environment variables:
+            .. code-block:: python
+
+                os.environ['GOOGLE_API_KEY'] = "your_api_key"
+
+                provider = GoogleAIStudioModelProvider()
+    """
+
+    type: Annotated[ModelProviderType, Field(
+        description="The type of model provider.",
+        default=ModelProviderType.GOOGLE_AI_STUDIO,
+        frozen=True
+    )]
+    credentials: Annotated[GoogleAIStudioCredentials | None, Field(
+        description="Google AI Studio credentials.", default=None
+    )]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> "GoogleAIStudioModelProvider":
+        if self.credentials is None:
+            self.credentials = GoogleAIStudioCredentials.create_from_env()
+        return self
+
+
+class AWSBedrockModelProvider(ModelProvider):
+    """
+    Represents a model provider using Amazon Bedrock.
+
+    Examples:
+        1. Create provider using credentials object:
+            .. code-block:: python
+
+                provider = AWSBedrockModelProvider(
+                    credentials=AWSBedrockCredentials(
+                        aws_access_key_id="your-access-key-id",
+                        aws_secret_access_key="your-secret-access-key",
+                        aws_region_name="us-east-1",
+                        aws_session_token="optional-session-token"
+                    )
+                )
+
+        2. Create provider using environment variables:
+            .. code-block:: python
+
+                os.environ['AWS_ACCESS_KEY_ID'] = "your-access-key-id"
+                os.environ['AWS_SECRET_ACCESS_KEY'] = "your-secret-access-key"
+                os.environ['AWS_SESSION_TOKEN'] = "optional-session-token"  # Optional
+                os.environ['AWS_DEFAULT_REGION'] = "us-east-1"
+                provider = AWSBedrockModelProvider()
+    """
+
+    type: Annotated[ModelProviderType, Field(
+        description="The type of model provider.",
+        default=ModelProviderType.AWS_BEDROCK,
+        frozen=True
+    )]
+    credentials: Annotated[AWSBedrockCredentials | None, Field(
+        description="AWS Bedrock credentials.", default=None
+    )]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> "AWSBedrockModelProvider":
+        if self.credentials is None:
+            self.credentials = AWSBedrockCredentials.create_from_env()
+        return self
+
+
+class CustomModelProvider(ModelProvider):
+    """
+    Defines the CustomModelProvider class.
+
+    This class represents a custom model provider, typically used when integrating with non-standard or user-defined
+    model backends. It sets the provider type to `CUSTOM` by default.
+
+    Examples:
+        1. Create a custom model provider:
+            .. code-block:: python
+
+                provider = CustomModelProvider()
+
+        2. Use with a custom foundation model:
+            .. code-block:: python
+
+                custom_model = CustomFoundationModel(
+                    scoring_fn=my_scoring_function,
+                    provider=CustomModelProvider()
+                )
+
+    Attributes:
+        type (ModelProviderType): The type of model provider. Always set to `ModelProviderType.CUSTOM`.
+    """
+    type: Annotated[ModelProviderType, Field(
+        description="The type of model provider.", default=ModelProviderType.CUSTOM)]
+
+
+class WxoAIGatewayModelProvider(ModelProvider):
+    """
+    This class represents a model provider configuration for WXO AI Gateway Interface. It includes the provider type and
+    credentials required to authenticate and interact with the WXO AI Gateway. If credentials are not explicitly
+    provided, it attempts to load them from environment variables.
+
+    Examples:
+        1. Create provider using credentials object:
+            .. code-block:: python
+
+                credentials = WxoAIGatewayCredentials(
+                    url="wxo-gateway-url",
+                    api_key="your-api-key"
+                )
+                provider = WxoAIGatewayModelProvider(credentials=credentials)
+
+        2. Create provider using environment variables:
+            .. code-block:: python
+
+                import os
+
+                os.environ['WXO_AI_GATEWAY_URL'] = "wxo-gateway-url"
+                os.environ['WATSONX_APIKEY'] = "your_api_key"
+
+                provider = WxoAIGatewayModelProvider()
+    """
+    type: Annotated[ModelProviderType,
+                    Field(description="The type of model provider.",
+                          default=ModelProviderType.WXO_AI_GATEWAY, frozen=True)]
+    credentials: Annotated[WxoAIGatewayCredentials | None, Field(
+        description="WXO AI Gateway credentials.", default=None)]
+
+    @model_validator(mode="after")
+    def create_credentials_from_env(self) -> Self:
+        if self.credentials is None:
+            self.credentials = WxoAIGatewayCredentials.create_from_env()
+        return self

ibm_watsonx_gov/entities/model_risk_result.py

@@ -0,0 +1,43 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import List
+
+import pandas as pd
+from pydantic import BaseModel
+
+
+class RiskMetric(BaseModel):
+    name: str
+    value: float | str | List[float]
+
+
+class Benchmark(BaseModel):
+    name: str
+    metrics: list[RiskMetric]
+
+    def get_metric_df(self) -> pd.DataFrame:
+        return pd.DataFrame(self.model_dump()["metrics"])
+
+
+class Risk(BaseModel):
+    name: str
+    benchmarks: list[Benchmark]
+
+
+class ModelRiskResult(BaseModel):
+    risks: list[Risk]
+    output_file_path: str | None = None
+
+    def to_json(self, **kwargs):
+        """
+        Transform the model risk result to a json.
+        The kwargs are passed to the model_dump_json method of pydantic model. All the arguments supported by pydantic model_dump_json can be passed.
+        """
+        return self.model_dump_json(**kwargs)

ibm_watsonx_gov/entities/monitor.py

@@ -0,0 +1,71 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# OCO Source Materials
+# 5900-A3Q, 5737-H76
+# Copyright IBM Corp. 2025
+# The source code for this program is not published or other-wise divested of its trade
+# secrets, irrespective of what has been deposited with the U.S.Copyright Office.
+# ----------------------------------------------------------------------------------------------------
+
+
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
+from typing_extensions import Annotated
+
+from ibm_watsonx_gov.entities.metric import MetricThreshold
+
+
+class MonitorThreshold(MetricThreshold):
+    metric_id: Annotated[str, Field(
+        description="Metric id", examples=["faithfulness"])]
+
+
+class BaseMonitor(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True)
+    monitor_name: Annotated[str, Field(description="Monitor name", examples=[
+                                       "generative_ai_quality", "drift_v2"])]
+    thresholds: Annotated[list[MonitorThreshold] | None, Field(
+        default=None, description="List of metric thresholds")]
+    parameters: Annotated[dict[str, Any] | None, Field(
+        default=None, description="Monitor parameters")]
+
+
+class GenerativeAIQualityMonitor(BaseMonitor):
+    monitor_name: str = "generative_ai_quality"
+    parameters: dict[str, Any] = {
+        "metrics_configuration": {
+            "rouge_score": {},
+            "exact_match": {},
+            "bleu": {},
+            "unsuccessful_requests": {},
+            "hap_input_score": {},
+            "hap_score": {},
+            "pii": {},
+            "pii_input": {},
+        },
+    }
+
+
+class DriftV2Monitor(BaseMonitor):
+    monitor_name: str = "drift_v2"
+
+
+class QualityMonitor(BaseMonitor):
+    monitor_name: str = "quality"
+    parameters: dict[str, Any] = {
+        "min_feedback_data_size": 10,
+    }
+
+
+class FairnessMonitor(BaseMonitor):
+    monitor_name: str = "fairness"
+
+
+class DriftMonitor(BaseMonitor):
+    monitor_name: str = "drift"
+
+
+class ModelHealthMonitor(BaseMonitor):
+    monitor_name: str = "model_health"