ibm-watsonx-gov 1.3.3__cp313-cp313-macosx_11_0_arm64.whl

ibm_watsonx_gov/entities/agentic_evaluation_result.py

@@ -0,0 +1,185 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+import copy
+import pandas as pd
+
+from typing import Annotated, List, Literal, Optional
+from pydantic import BaseModel, Field
+from ibm_watsonx_gov.entities.agentic_app import Node
+
+from ibm_watsonx_gov.entities.evaluation_result import AgentMetricResult, AggregateAgentMetricResult, MessageData, NodeData, MetricsMappingData
+
+AGENTIC_RESULT_COMPONENTS = ["conversation", "message", "node"]
+
+
+class AgenticEvaluationResult(BaseModel):
+    metrics_results: Annotated[List[AgentMetricResult],
+                               Field(title="Metrics result",
+                                     description="The list of metrics result.")]
+    aggregated_metrics_results: Annotated[List[AggregateAgentMetricResult],
+                                          Field(title="Aggregated metrics result",
+                                                description="The list of aggregated metrics result. The metrics are aggregated for each node in the agent.")]
+    messages_data: Annotated[List[MessageData],
+                             Field(title="Messages",
+                                   description="The list of agent messages data.",
+                                   default=[])]
+    nodes_data: Annotated[List[NodeData],
+                          Field(title="Node messages",
+                                description="The list of nodes data.",
+                                default=[])]
+    metrics_mapping_data: Annotated[List[MetricsMappingData],
+                                    Field(title="Metrics mapping data",
+                                          description="The mapping data used to compute the metric.",
+                                          default=[])]
+    nodes: Annotated[list[Node],
+                     Field(title="Nodes",
+                           description="The list of nodes details",
+                           default=[])]
+    edges: Annotated[list[dict],
+                     Field(title="Nodes",
+                           description="The list of nodes details",
+                           default=[])]
+
+    def get_aggregated_metrics_results(self,
+                                       applies_to: list[str] = AGENTIC_RESULT_COMPONENTS,
+                                       node_name: Optional[str] = None,
+                                       include_individual_results: bool = True,
+                                       format: Literal["json",
+                                                       "object"] = "json",
+                                       **kwargs) -> list[AggregateAgentMetricResult] | list[dict]:
+        """
+        Get the aggregated agentic metrics results based on the specified arguments.
+
+        Args:
+            applies_to (AGENTIC_RESULT_COMPONENTS, optional): The type of component the metric result applies to. Defaults to ["conversation", "message", "node"].
+            node_name (str, optional): The name of the node to get the aggregated results for. Defaults to None.
+            include_individual_results (bool, optional): Whether to return the individual metrics results. Defaults to False.
+            format (Literal["json", "object"], optional): The format of the output. Defaults to "json".
+        Return:
+            returns: list[AggregateAgentMetricResult] | list [dict]
+        """
+
+        aggregated_results = []
+        for amr in self.aggregated_metrics_results:
+            if amr.applies_to in applies_to and (not node_name or amr.node_name == node_name):
+                if format == "json":
+                    if kwargs.get("exclude_unset") is None:
+                        kwargs["exclude_unset"] = True
+                    if kwargs.get("exclude_none") is None:
+                        kwargs["exclude_none"] = True
+                    if include_individual_results:
+                        aggregated_results.append(
+                            amr.model_dump(mode="json", **kwargs))
+                    else:
+                        aggregated_results.append(
+                            amr.model_dump(mode="json", exclude=["individual_results"], **kwargs))
+                else:
+                    aggregated_results.append(copy.deepcopy(amr))
+
+        return aggregated_results
+
+    def get_metrics_results(self,
+                            applies_to: list[str] = AGENTIC_RESULT_COMPONENTS,
+                            node_name: Optional[str] = None,
+                            format: Literal["json", "object"] = "json",
+                            **kwargs) -> list[AgentMetricResult] | list[dict]:
+        """
+        Get the agentic metrics results based on the specified arguments.
+
+        Args:
+            applies_to (AGENTIC_RESULT_COMPONENTS, optional): The type of component the metrics results applies to. Defaults to ["conversation", "message", "node"].
+            node_name (str, optional): The name of the node to get the metrics results for. Defaults to None.
+            format (Literal["json", "object"], optional): The format of the output. Defaults to "json".
+        Return:
+            returns: list[AgentMetricResult] | list [dict]
+        """
+
+        metrics_results = []
+        for amr in self.metrics_results:
+            if amr.applies_to in applies_to and (not node_name or amr.node_name == node_name):
+                if format == "json":
+                    if kwargs.get("exclude_unset") is None:
+                        kwargs["exclude_unset"] = True
+                    if kwargs.get("exclude_none") is None:
+                        kwargs["exclude_none"] = True
+                    metrics_results.append(
+                        amr.model_dump(mode="json", **kwargs))
+                else:
+                    metrics_results.append(copy.deepcopy(amr))
+
+        return metrics_results
+
+    def to_json(self, **kwargs) -> dict:
+        """
+        Get the AgenticEvaluationResult as json
+
+        Returns:
+            dict: The AgenticEvaluationResult
+        """
+
+        if kwargs.get("exclude_unset") is None:
+            kwargs["exclude_unset"] = True
+
+        if kwargs.get("exclude_none") is None:
+            kwargs["exclude_none"] = True
+
+        return self.model_dump(mode="json", **kwargs)
+
+    def to_df(self, input_data: Optional[pd.DataFrame] = None,
+              message_id_field: str = "message_id",  wide_format: bool = True) -> pd.DataFrame:
+        """
+        Get individual metrics dataframe.
+
+        If the input dataframe is provided, it will be merged with the metrics dataframe.
+
+        Args:
+            input_data (Optional[pd.DataFrame], optional): Input data to merge with metrics dataframe. Defaults to None.
+            message_id_field (str, optional): Field to use for merging input data and metrics dataframe. Defaults to "message_id".
+            wide_format (bool): Determines whether to display the results in a pivot table format. Defaults to True
+
+        Returns:
+            pd.DataFrame: Metrics dataframe.
+        """
+
+        def converter(m): return m.model_dump(
+            exclude={"provider"}, exclude_none=True)
+
+        metrics_df = pd.DataFrame(list(map(converter, self.metrics_results)))
+        if input_data is not None:
+            metrics_df = input_data.merge(metrics_df, on=message_id_field)
+
+        # Return the metric result dataframe
+        # if the wide_format is False
+        if not wide_format:
+            return metrics_df
+
+        # Prepare the dataframe for pivot table view
+        def col_name(row):
+            if row["applies_to"] == "node":
+                return f"{row['node_name']}.{row['name']}"
+            if row["applies_to"] == "message":
+                return f"message.{row['name']}"
+            # TODO support other types
+
+        metrics_df["idx"] = metrics_df.apply(col_name, axis=1)
+
+        # Pivot the table
+        metrics_df_wide = metrics_df.pivot_table(
+            index="message_id",
+            columns="idx",
+            values="value"
+        ).reset_index().rename_axis("", axis=1)
+
+        # if input_data is provided add
+        # it to the pivot table
+        if input_data is not None:
+            metrics_df_wide = input_data.merge(
+                metrics_df_wide, on=message_id_field)
+        return metrics_df_wide

ibm_watsonx_gov/entities/ai_evaluation.py

@@ -0,0 +1,290 @@
+# ----------------------------------------------------------------------------------------------------
+# IBM Confidential
+# Licensed Materials - Property of IBM
+# 5737-H76, 5900-A3Q
+# © Copyright IBM Corp. 2025  All Rights Reserved.
+# US Government Users Restricted Rights - Use, duplication or disclosure restricted by
+# GSA ADPSchedule Contract with IBM Corp.
+# ----------------------------------------------------------------------------------------------------
+
+from typing import Annotated, Dict, List
+from pydantic import BaseModel, Field
+
+from ibm_watsonx_gov.entities.ai_experiment import Node
+
+
+class EvaluationAsset(BaseModel):
+    id: Annotated[
+        str,
+        Field(
+            description="The id of the AI Experiment asset",
+            examples=["asset-001"],
+            default="",
+        ),
+    ]
+    container_id: Annotated[
+        str,
+        Field(
+            description="The project id or space id.", examples=["proj-01"], default=""
+        ),
+    ]
+    container_type: Annotated[
+        str,
+        Field(
+            description="The container type of AI Experiment",
+            examples=["project", "spcae"],
+            default="",
+        ),
+    ]
+    name: Annotated[
+        str,
+        Field(
+            description="The name of the AI Experiment.",
+            examples=["AI_experiment_1"],
+            default="",
+        ),
+    ]
+    run_id: Annotated[
+        str,
+        Field(
+            description="The experiment run id of the AI Experiment.",
+            examples=["run-01"],
+            default="",
+        ),
+    ]
+    run_name: Annotated[
+        str,
+        Field(
+            description="The experiment run name of the AI Experiment.",
+            examples=["Test run 1"],
+            default="",
+        ),
+    ]
+    attachment_id: Annotated[
+        str,
+        Field(
+            description="the attachment id for the evaluation result for that experiment run.",
+            examples=["att-01"],
+            default="",
+        ),
+    ]
+    test_data: Annotated[
+        Dict,
+        Field(
+            description="The test data of that experiment run.", examples=[], default={}
+        ),
+    ]
+    nodes: Annotated[
+        List[Node],
+        Field(
+            description="List of the node for that experiment run.",
+            examples=[{"id": "node-001", "name": "Node_1", "type": "tool"}],
+            default="",
+        ),
+    ]
+
+
+class EvaluationConfig(BaseModel):
+    monitors: Annotated[
+        Dict,
+        Field(
+            description="The monitors configuration of for that AI Evaluation.",
+            examples=[
+                {"agentic_ai_quality": {"parameters": {"metrics_configuration": {}}}}
+            ],
+            default={},
+        ),
+    ]
+    evaluation_assets: Annotated[
+        List[EvaluationAsset],
+        Field(
+            description="The evaluation asset details.",
+            examples=[
+                [
+                    {
+                        "id": "d4d6ac43-0bec-47f9-8924-0b74ea1b8ec3",
+                        "container_id": "b76d2ebb-4e05-496e-b377-557d409e8c45",
+                        "container_type": "project",
+                        "name": "AI_Experiment asset",
+                        "run_id": "fa7629e9-e1bb-4779-9198-9a6343dab1ad",
+                        "run_name": "Experiment run 1",
+                        "attachment_id": "ab914f9b-9475-4c10-88d8-480b6c9f4963",
+                        "test_data": {"total_rows": 0},
+                        "nodes": [],
+                    }
+                ]
+            ],
+            default=[],
+        ),
+    ]
+
+
+class AIEvaluationAsset(BaseModel):
+    """
+    The class for AIEvaluationAsset.
+
+    Examples
+    --------
+    Create AIEvaluationAsset instance:
+        .. code-block:: python
+
+            # Define evaluation configuration
+            evaluation_config = EvaluationConfig(
+                monitors={
+                    "agentic_ai_quality": {
+                        "parameters": {
+                            "metrics_configuration": {}
+                        }
+                    }
+                }
+            )
+
+            # Create the evaluation asset
+            ai_evaluation_asset = AIEvaluationAsset(
+                name="AI Evaluation for agent",
+                evaluation_configuration=evaluation_config
+            )
+
+            # Compare two or more AI experiments using the evaluation asset
+            response = ai_experiment_client.compare_ai_experiments(
+                ai_experiment_ids=["experiment_id_1", "experiment_id_2"],
+                ai_evaluation_asset=ai_evaluation_asset
+            )
+    """
+
+    container_id: Annotated[
+        str,
+        Field(
+            description="The project or space id for the AI Evaluation.",
+            examples=["proj--1"],
+            default="",
+        ),
+    ]
+    container_type: Annotated[
+        str,
+        Field(
+            description="The container type for the AI Evaluation.",
+            examples=["project", "space"],
+            default="",
+        ),
+    ]
+    container_name: Annotated[
+        str,
+        Field(
+            description="The name of the project or the space.",
+            examples=["Project_1"],
+            default="",
+        ),
+    ]
+    name: Annotated[
+        str,
+        Field(
+            description="The name of the AI Evaluation asset.",
+            examples=["AI agents evaluation"],
+            default="",
+        ),
+    ]
+    description: Annotated[
+        str,
+        Field(
+            description="The description of the AI Evaluation asset.",
+            examples=["AI agents evaluation"],
+            default="",
+        ),
+    ]
+    asset_type: Annotated[
+        str,
+        Field(
+            description="The asset type of the AI Evaluation.",
+            examples=["ai_evaluation"],
+            default="ai_evaluation",
+        ),
+    ]
+    created_at: Annotated[
+        str,
+        Field(
+            description="The timestamp of creation of AI Evaluation asset.",
+            examples=["2025-04-01T12:00:00Z"],
+            default="",
+        ),
+    ]
+    owner_id: Annotated[
+        str,
+        Field(
+            description="The owner of the AI Evaluation.",
+            examples=["user-123"],
+            default="",
+        ),
+    ]
+    asset_id: Annotated[
+        str,
+        Field(
+            description="The asset id of the AI Evaluation.",
+            examples=["43676d70-1ecc-412e-832f-8762aa899247"],
+            default="",
+        ),
+    ]
+    creator_id: Annotated[
+        str,
+        Field(
+            description="The creator id of the AI Evaluation.",
+            examples=["user-123"],
+            default="",
+        ),
+    ]
+    asset_details: Annotated[
+        Dict,
+        Field(
+            description="The asset details of the AI Evluation asset.",
+            examples=[
+                {
+                    "task_ids": [],
+                    "label_column": "",
+                    "operational_space_id": "development",
+                    "input_data_type": "unstructured_text",
+                    "job_id": "",
+                    "service_instance_id": "",
+                    "evaluation_asset_type": "ai_experiment|prompt",
+                }
+            ],
+            default={},
+        ),
+    ]
+    evaluation_configuration: Annotated[
+        EvaluationConfig,
+        Field(
+            description="The list of the evaluation configuration",
+            examples=[
+                {
+                    "monitors": {
+                        "agentic_ai_quality": {
+                            "parameters": {"metrics_configuration": {}}
+                        }
+                    },
+                    "evaluation_assets": [
+                        {
+                            "id": "d4d6ac43-0bec-47f9-8924-0b74ea1b8ec3",
+                            "container_id": "b76d2ebb-4e05-496e-b377-557d409e8c45",
+                            "container_type": "project",
+                            "name": "AI_Experiment asset for Agent governence",
+                            "run_id": "fa7629e9-e1bb-4779-9198-9a6343dab1ad",
+                            "run_name": "Experiment run 1",
+                            "attachment_id": "ab914f9b-9475-4c10-88d8-480b6c9f4963",
+                            "test_data": {"total_rows": 0},
+                            "nodes": [],
+                        }
+                    ],
+                }
+            ],
+            default=[],
+        ),
+    ]
+    href: Annotated[
+        str, Field(description="The link of the AI Evaluation Asset", default="")
+    ]
+
+    def to_json(self):
+        """
+        Transform the AIEvaluationAsset instance to json
+        """
+        return self.model_dump(mode="json")