ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/config/gen_ai_configuration.py

@@ -0,0 +1,129 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Callable, Dict, Optional, Union
+
+from pydantic import Field, model_validator
+from typing_extensions import Self
+
+from ibm_watsonx_gov.entities.base_classes import BaseConfiguration
+from ibm_watsonx_gov.entities.enums import TaskType
+from ibm_watsonx_gov.entities.llm_judge import LLMJudge
+from ibm_watsonx_gov.entities.locale import Locale
+
+
+class GenAIConfiguration(BaseConfiguration):
+    """
+    Defines the GenAIConfiguration class.
+
+    This is used to specify the fields mapping details in the data and other configuration parameters needed for evaluation.
+
+    Examples:
+        1. Create configuration with default parameters
+            .. code-block:: python
+
+                configuration = GenAIConfiguration()
+
+        2. Create configuration with parameters
+            .. code-block:: python
+
+                configuration = GenAIConfiguration(input_fields=["input"], 
+                                                   output_fields=["output"])
+
+        2. Create configuration with dict parameters
+            .. code-block:: python
+
+                config = {"input_fields": ["input"],
+                          "output_fields": ["output"],
+                          "context_fields": ["contexts"],
+                          "reference_fields": ["reference"]}
+                configuration = GenAIConfiguration(**config)    
+    """
+    task_type: Annotated[TaskType | None, Field(title="Task Type",
+                                                description="The generative task type. Default value is None.",
+                                                default=None,
+                                                examples=[TaskType.RAG])]
+    input_fields: Annotated[list[str], Field(title="Input Fields",
+                                             description="The list of model input fields in the data. Default value is ['input_text'].",
+                                             examples=[
+                                                 ["question"]],
+                                             default=["input_text"])]
+    context_fields: Annotated[list[str], Field(title="Context Fields",
+                                               description="The list of context fields in the input fields. Default value is ['context'].",
+                                               default=["context"],
+                                               examples=[["context1", "context2"]])]
+    output_fields: Annotated[list[str], Field(title="Output Fields",
+                                              description="The list of model output fields in the data. Default value is ['generated_text'].",
+                                              default=["generated_text"],
+                                              examples=[["output"]])]
+    reference_fields: Annotated[list[str], Field(title="Reference Fields",
+                                                 description="The list of reference fields in the data. Default value is ['ground_truth'].",
+                                                 default=["ground_truth"],
+                                                 examples=[["reference"]])]
+    locale: Annotated[Locale | None, Field(title="Locale",
+                                           description="The language locale of the input, output and reference fields in the data.",
+                                           default=None)]
+    tools: Annotated[Union[list[Callable], list[Dict]], Field(title="Tools",
+                                                              description="The list of tools used by the LLM.",
+                                                              default=[],
+                                                              examples=[["function1", "function2"]])]
+    tool_calls_field: Annotated[Optional[str], Field(title="Tool Calls Field",
+                                                     description="The tool calls field in the input fields. Default value is 'tool_calls'.",
+                                                     default="tool_calls",
+                                                     examples=["tool_calls"])]
+    available_tools_field: Annotated[Optional[str], Field(title="Available Tools Field",
+                                                          description="The tool inventory field in the data. Default value is 'available_tools'.",
+                                                          default="available_tools",
+                                                          examples=["available_tools"])]
+
+    llm_judge: Annotated[LLMJudge | None, Field(title="LLM Judge",
+                                                description="LLM as Judge Model details.",
+                                                default=None)]
+    prompt_field: Annotated[Optional[str], Field(title="Model Prompt Field",
+                                                 description="The prompt field in the input fields. Default value is 'model_prompt'.",
+                                                 default="model_prompt",
+                                                 examples=["model_prompt"])]
+    start_time_field: Annotated[Optional[str], Field(title="Span Start Time Field ",
+                                                     description="The start time field in span attributes.",
+                                                     default=None,
+                                                     examples=["start_time"])]
+    end_time_field: Annotated[Optional[str], Field(title="Span End Time Field",
+                                                         description="The end time field in span attributes.",
+                                                         default=None,
+                                                         examples=["end_time"])]
+    model_usage_detail_fields: Annotated[Optional[list[str]], Field(title="Model Usage Detail Field",
+                                                              description="The model usage detail field in span attributes.This field should provide information on model name, input_token_count and output_token_count",
+                                                              default=[])]
+    input_token_count_fields: Annotated[Optional[list[str]], Field(title="Input Token Count Field",
+                                                                   description="The input token count field in span attributes.",
+                                                                   default=[],
+                                                                   examples=[["prompt_tokens"]])]
+    output_token_count_fields: Annotated[Optional[list[str]], Field(title="Output Token Count Field",
+                                                              description="The output token count field in span attributes.",
+                                                              default=[
+                                                                  "completion_tokens"],
+                                                              examples=[["completion_tokens"]])]
+    status_field: Annotated[Optional[str], Field(title="Span Status Field ",
+                                                 description="The status field in span attributes.",
+                                                 default="status",
+                                                 examples=["status"])]
+    user_id_field: Annotated[Optional[str], Field(title="User Id Field ",
+                                                  description="The user id field in span attributes.",
+                                                  default="user_id",
+                                                  examples=["user_id"])]
+
+    @model_validator(mode="after")
+    def validate_fields(self) -> Self:
+
+        if self.task_type == TaskType.RAG:
+            if not self.input_fields or not self.context_fields:
+                raise ValueError(
+                    "input_fields and context_fields are required for RAG task type.")
+
+        return self

ibm_watsonx_gov/config/model_risk_configuration.py

@@ -0,0 +1,173 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, List, Optional, Tuple
+
+from pydantic import BaseModel, ConfigDict, Field, PositiveInt, field_validator
+
+from ibm_watsonx_gov.entities.credentials import WxGovConsoleCredentials
+from ibm_watsonx_gov.entities.foundation_model import FoundationModel
+
+
+class WxGovConsoleConfiguration(BaseModel):
+    """
+    Defines the WxGovConsoleConfiguration class.
+
+    This configuration is used to integrate with the watsonx Governance Console for storing model risk evaluation results.
+    It includes the model identifier and the credentials required for authentication.
+
+    Examples:
+        1. Create configuration with explicit credentials:
+            .. code-block:: python
+
+                credentials = WxGovConsoleCredentials(
+                    url="https://governance-console.example.com",
+                    username="admin",
+                    password="securepassword",
+                    api_key="optional-api-key"
+                )
+                configuration = WxGovConsoleConfiguration(
+                    model_id="model-12345",
+                    credentials=credentials
+                )
+    """
+    model_id: Annotated[
+        str,
+        Field(
+            description="The watsonx Governance Console identifier of the model to store the model risk result."
+        ),
+    ]
+    credentials: Annotated[
+        WxGovConsoleCredentials,
+        Field(
+            description="The watsonx Governance Console credentials."
+        ),
+    ]
+    model_config = ConfigDict(protected_namespaces=())
+
+
+class ModelRiskConfiguration(BaseModel):
+    """
+    Defines the ModelRiskConfiguration class.
+
+    This configuration class encapsulates all parameters required to perform model risk evaluation,
+    including model metadata, evaluation scope, thresholds, and output/reporting preferences.
+
+    Examples:
+        1. Create a basic configuration:
+            .. code-block:: python
+
+                model_details = WxAIFoundationModel(
+                    model_name="mymodel_flan",
+                    model_id="ibm/granite-3-3-8b-instruct",
+                    project_id="project_id")
+
+                model_config = ModelRiskConfiguration(
+                    model_details=model_details,
+                    risk_dimensions=["hallucination"],
+                    max_sample_size=500,
+                    thresholds=(20, 80),
+                    pdf_report_output_path="/reports"                    
+                )
+
+        2. Include watsonx Governance Console integration:
+            .. code-block:: python
+
+                model_details = WxAIFoundationModel(
+                    model_name="mymodel_flan",
+                    model_id="ibm/granite-3-3-8b-instruct",
+                    project_id="project_id")
+
+                wx_gc_credentials = WxGovConsoleCredentials(
+                    url="https://governance.example.com",
+                    username="admin",
+                    password="securepass"
+                    api_key="console API key"
+                )        
+
+                wx_config = WxGovConsoleConfiguration(
+                    model_id="model-abc123",
+                    credentials=wx_gc_credentials
+                )
+
+                model_config = ModelRiskConfiguration(
+                    model_details=model_details,
+                    risk_dimensions=["hallucination"],
+                    max_sample_size=500,
+                    thresholds=(20, 80),
+                    wx_gc_configuration=wx_config,
+                    pdf_report_output_path="/reports"                    
+                )                
+
+    Validators:
+        - `thresholds`: Ensures that the threshold values are between 0 and 100, and that the lower value is less than the upper value.
+    """
+    model_details: Annotated[
+        FoundationModel,
+        Field(
+            title="Foundation Model Details",
+            description="The details of the foundation model being evaluated.",
+        )
+    ]
+    risk_dimensions: Annotated[
+        Optional[List[str]],
+        Field(
+            title="Risk Dimensions",
+            description="A list of risk categories to be evaluated for the model. These could include hallucination, jailbreaking etc.",
+            default=None,
+            examples=[["hallucination", "jailbreaking",
+                       "harmful-code-generation"]]
+        )
+    ]
+    max_sample_size: Annotated[
+        Optional[PositiveInt],
+        Field(
+            title="Maximum Sample Size",
+            description="The maximum number of samples to be used during the evaluation process. Must be a positive integer.",
+            default=None,
+            examples=[50]
+        )
+    ]
+    wx_gc_configuration: Annotated[
+        Optional[WxGovConsoleConfiguration],
+        Field(
+            title="watsonx Governance Console Configuration",
+            description="Optional configuration for storing results in watsonx Governance Console.",
+            default=None,
+        )
+    ]
+    pdf_report_output_path: Annotated[
+        Optional[str],
+        Field(
+            title="PDF Report Output Path",
+            description="The output file path to store the model risk evaluation PDF report.",
+            default=None,
+            examples=["/reports/"]
+        )
+    ]
+    thresholds: Annotated[
+        Optional[Tuple[int, int]],
+        Field(
+            title="Performance Thresholds",
+            description="A tuple representing the percentile-based threshold configuration used for categorizing LLM performance. The first element is the lower percentile threshold, and the second is the upper percentile threshold",
+            default=(25, 75),
+            examples=[(25, 75)]
+        )
+    ]
+    model_config = ConfigDict(protected_namespaces=())
+
+    @field_validator("thresholds")
+    @classmethod
+    def validate_thresholds(cls, v):
+        if v is not None:
+            low, high = v
+            if not (0 <= low < high <= 100):
+                raise ValueError(
+                    "Thresholds must be between 0 and 100, and the first must be less than the second.")
+        return v

ibm_watsonx_gov/config/predictive_ai_configuration.py

@@ -0,0 +1,20 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+
+from ibm_watsonx_gov.entities.base_classes import BaseConfiguration
+from ibm_watsonx_gov.entities.enums import InputDataType, ProblemType
+
+
+class PredictiveAIConfiguration(BaseConfiguration):
+    problem_type: ProblemType
+    input_data_type: InputDataType
+    feature_fields: list[str]
+    categorical_fields: list[str] = []
+    text_fields: list[str] = []

ibm_watsonx_gov/entities/__init__.py

@@ -0,0 +1,8 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------

ibm_watsonx_gov/entities/agentic_app.py

@@ -0,0 +1,209 @@
+
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+
+from json import loads
+from typing import Annotated, Optional
+
+from pydantic import BaseModel, Field, TypeAdapter, field_serializer
+
+from ibm_watsonx_gov.config.agentic_ai_configuration import \
+    AgenticAIConfiguration
+from ibm_watsonx_gov.entities.enums import MetricGroup
+from ibm_watsonx_gov.entities.foundation_model import FoundationModelInfo
+from ibm_watsonx_gov.entities.metric import GenAIMetric, Mapping
+from ibm_watsonx_gov.metrics import METRICS_UNION
+
+
+class MetricsConfiguration(BaseModel):
+    """
+    The class representing the metrics to be computed and the configuration details required for them.
+
+    Examples:
+        1. Create MetricsConfiguration with default agentic ai configuration
+            .. code-block:: python
+
+                metrics_configuration = MetricsConfiguration(metrics=[ContextRelevanceMetric()],
+                                                             metric_groups=[MetricGroup.RETRIEVAL_QUALITY])])
+
+        2. Create MetricsConfiguration by specifying agentic ai configuration
+            .. code-block:: python
+
+                config = {
+                    "input_fields": ["input"],
+                    "context_fields": ["contexts"]
+                }
+                metrics_configuration = MetricsConfiguration(configuration=AgenticAIConfiguration(**config)
+                                                               metrics=[ContextRelevanceMetric()],
+                                                               metric_groups=[MetricGroup.RETRIEVAL_QUALITY])])
+    """
+    configuration: Annotated[AgenticAIConfiguration,
+                             Field(title="Metrics configuration",
+                                   description="The configuration of the metrics to compute. The configuration contains the fields names to be read when computing the metrics.",
+                                   default=AgenticAIConfiguration())]
+    metrics: Annotated[Optional[list[GenAIMetric]],
+                       Field(title="Metrics",
+                             description="The list of metrics to compute.",
+                             default=[])]
+    metric_groups: Annotated[Optional[list[MetricGroup]],
+                             Field(title="Metric Groups",
+                                   description="The list of metric groups to compute.",
+                                   default=[])]
+
+    @classmethod
+    def model_validate(cls, obj, **kwargs):
+        if "metrics" in obj:
+            obj["metrics"] = [TypeAdapter(METRICS_UNION).validate_python(
+                m) for m in obj.get("metrics")]
+        return super().model_validate(obj, **kwargs)
+
+    @field_serializer("metrics", when_used="json")
+    def metrics_serializer(self, metrics: list[GenAIMetric]):
+        return [metric.model_dump(mode="json") for metric in metrics]
+
+
+class Node(BaseModel):
+    """
+    The class representing a node in an agentic application.
+
+    Examples:
+        1. Create Node with metrics configuration and default agentic ai configuration
+            .. code-block:: python
+
+                metrics_configurations = [MetricsConfiguration(metrics=[ContextRelevanceMetric()],
+                                                               metric_groups=[MetricGroup.RETRIEVAL_QUALITY])])]
+                node = Node(name="Retrieval Node",
+                            metrics_configurations=metrics_configurations)
+
+        2. Create Node with metrics configuration and specifying agentic ai configuration
+            .. code-block:: python
+
+                node_config = {"input_fields": ["input"],
+                               "output_fields": ["output"],
+                               "context_fields": ["contexts"],
+                               "reference_fields": ["reference"]}
+                metrics_configurations = [MetricsConfiguration(configuration=AgenticAIConfiguration(**node_config)
+                                                               metrics=[ContextRelevanceMetric()],
+                                                               metric_groups=[MetricGroup.RETRIEVAL_QUALITY])])]
+                node = Node(name="Retrieval Node",
+                            metrics_configurations=metrics_configurations)
+    """
+    name: Annotated[str,
+                    Field(title="Name",
+                          description="The name of the node.")]
+    func_name: Annotated[Optional[str],
+                         Field(title="Node function name",
+                               description="The name of the node function.",
+                               default=None)]
+    metrics_configurations: Annotated[list[MetricsConfiguration],
+                                      Field(title="Metrics configuration",
+                                            description="The list of metrics and their configuration details.",
+                                            default=[])]
+    foundation_models: Annotated[
+        list[FoundationModelInfo],
+        Field(
+            description="The Foundation models invoked by the node",
+            default=[],
+        ),
+    ]
+
+    @classmethod
+    def model_validate(cls, obj, **kwargs):
+        if "metrics_configurations" in obj:
+            obj["metrics_configurations"] = [MetricsConfiguration.model_validate(
+                m) for m in obj.get("metrics_configurations")]
+        return super().model_validate(obj, **kwargs)
+
+
+class AgenticApp(BaseModel):
+    """
+    The configuration class representing an agentic application.
+    An agent is composed of a set of nodes.
+    The metrics to be computed at the agent or message level should be specified in the metrics_configuration and the metrics to be computed for the node level should be specified in the nodes list.
+
+    Examples:
+        1. Create AgenticApp with agent level metrics configuration. 
+            .. code-block:: python
+
+                # Below example provides the agent configuration to compute the AnswerRelevanceMetric and all the metrics in Content Safety group on agent or message level.
+                agentic_app = AgenticApp(name="Agentic App",
+                                    metrics_configuration=MetricsConfiguration(metrics=[AnswerRelevanceMetric()],
+                                                                                metric_groups=[MetricGroup.CONTENT_SAFETY]))
+                agentic_evaluator = AgenticEvaluator(agentic_app=agentic_app)
+                ...
+
+        2. Create AgenticApp with agent and node level metrics configuration and default agentic ai configuration for metrics. 
+            .. code-block:: python
+
+                # Below example provides the node configuration to compute the ContextRelevanceMetric and all the metrics in Retrieval Quality group. 
+                nodes = [Node(name="Retrieval Node",
+                            metrics_configurations=[MetricsConfiguration(metrics=[ContextRelevanceMetric()],
+                                                                         metric_groups=[MetricGroup.RETRIEVAL_QUALITY])])]
+
+                # Below example provides the agent configuration to compute the AnswerRelevanceMetric and all the metrics in Content Safety group on agent or message level.
+                agentic_app = AgenticApp(name="Agentic App",
+                                    metrics_configuration=MetricsConfiguration(metrics=[AnswerRelevanceMetric()],
+                                                                                metric_groups=[MetricGroup.CONTENT_SAFETY]),
+                                    nodes=nodes)
+                agentic_evaluator = AgenticEvaluator(agentic_app=agentic_app)
+                ...
+
+        3. Create AgenticApp with agent and nodel level metrics configuration and with agentic ai configuration for metrics. 
+            .. code-block:: python
+
+                # Below example provides the node configuration to compute the ContextRelevanceMetric and all the metrics in Retrieval Quality group.
+                node_fields_config = {
+                    "input_fields": ["input"],
+                    "context_fields": ["web_context"]
+                }
+                nodes = [Node(name="Retrieval Node",
+                            metrics_configurations=[MetricsConfiguration(configuration=AgenticAIConfiguration(**node_fields_config)
+                                                                         metrics=[ContextRelevanceMetric()],
+                                                                         metric_groups=[MetricGroup.RETRIEVAL_QUALITY])])]
+
+                # Below example provides the agent configuration to compute the AnswerRelevanceMetric and all the metrics in Content Safety group on agent or message level.
+                agent_fields_config = {
+                    "input_fields": ["input"],
+                    "output_fields": ["output"]
+                }
+                agentic_app = AgenticApp(name="Agentic App",
+                                    metrics_configuration=MetricsConfiguration(configuration=AgenticAIConfiguration(**agent_fields_config)
+                                                                               metrics=[AnswerRelevanceMetric()],
+                                                                               metric_groups=[MetricGroup.CONTENT_SAFETY]),
+                                    nodes=nodes)
+                agentic_evaluator = AgenticEvaluator(agentic_app=agentic_app)
+                ...
+    """
+    name: Annotated[str, Field(title="Agentic application name",
+                               description="The name of the agentic application.",
+                               default="Agentic App")]
+    message_io_mapping: Annotated[Optional[Mapping],
+                                  Field(title="Message IO mapping",
+                                        description="The message input and output mapping.",
+                                        default=None)]
+    metrics_configuration: Annotated[Optional[MetricsConfiguration],
+                                     Field(title="Metrics configuration",
+                                           description="The list of metrics to be computed on the agentic application and their configuration details.",
+                                           default=MetricsConfiguration())]
+    nodes: Annotated[Optional[list[Node]],
+                     Field(title="Node details",
+                           description="The nodes details.",
+                           default=[])]
+
+    @classmethod
+    def model_validate_json(cls, json_data, **kwargs):
+        data = loads(json_data)
+        if "metrics_configuration" in data:
+            data["metrics_configuration"] = MetricsConfiguration.model_validate(
+                data.get("metrics_configuration"))
+        if "nodes" in data:
+            data["nodes"] = [Node.model_validate(node)
+                             for node in data.get("nodes", [])]
+        return cls.model_validate(data, **kwargs)