ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/metrics/evaluations.py

@@ -0,0 +1,107 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+from wxo_agentic_evaluation.metrics.metrics import Metric
+from wxo_agentic_evaluation.prompt.template_render import LLMaaJTemplateRenderer
+from wxo_agentic_evaluation.service_provider.provider import Provider
+from wxo_agentic_evaluation.type import Message, OrchestrateDataset
+from wxo_agentic_evaluation.utils.messages_parser import ParsedMessages
+
+root_dir: str = os.path.dirname(os.path.dirname(__file__))
+LLMAAJ_PROMPT_PATH = os.path.join(root_dir, "prompt", "llmaaj_prompt.jinja2")
+
+
+class Evaluation(ABC):
+    """Abstract base class for all evaluations."""
+
+    def __init__(self, llm_client: Optional[Provider] = None) -> None:
+        self._llm_client = llm_client
+
+    @property
+    def llm_client(self) -> Any:
+        """Access client, require it if used."""
+        if self._llm_client is None:
+            raise RuntimeError(
+                f"{self.__class__.__name__} requires a client, but none was provided"
+            )
+        return self._llm_client
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Unique name for the evaluator."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def evaluate(
+        self,
+        messages: list[Message],
+        ground_truth: OrchestrateDataset,
+        extracted_context: Dict[str, Any],
+    ) -> Optional[Metric]:
+        """
+        Evaluation method.
+
+        Args:
+            messages: agent and user conversational messages (includes tool calls)
+            ground_truth: ground truth data
+            extracted_context: dictionary containing data derived from the messages
+
+        Returns:
+            Metic
+        """
+        raise NotImplementedError
+
+
+class LLMaaJEvaluation(Evaluation, ABC):
+    """Evaluation metric for LLMaaJ."""
+
+    @property
+    @abstractmethod
+    def llmaaj_instructions(self) -> str:
+        """LLMaaJ instructions for the evaluator."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def format_llm_output(self, string: str) -> int | float | bool | str:
+        """Format the output of the LLMaaJ query."""
+        raise NotImplementedError
+
+    @property
+    def selected_context_keys(self) -> set[str]:
+        """Override to implement context keys to pass to the prompt."""
+        return set()
+
+    def select_context(
+        self, extracted_context: Dict[str, Any]
+    ) -> dict[str, Any]:
+        """Additional context to be added to the prompt."""
+        selected_context = {
+            key: value
+            for key, value in extracted_context.items()
+            if key in self.selected_context_keys
+        }
+
+        return selected_context
+
+    def evaluate(
+        self,
+        messages: list[Message],
+        ground_truth: OrchestrateDataset,
+        extracted_context: Dict[str, Any],
+    ) -> Optional[Metric]:
+        renderer = LLMaaJTemplateRenderer(LLMAAJ_PROMPT_PATH)
+        parsed = ParsedMessages(messages=messages)
+        if parsed.user_input is None or parsed.agent_response is None:
+            return None
+        context = str(self.select_context(extracted_context))
+        prompt = renderer.render(
+            user_input=parsed.user_input,
+            agent_answer=parsed.agent_response,
+            llmaaj_instructions=self.llmaaj_instructions,
+            context=context,
+        )
+        score_str = self.llm_client.query(prompt)
+        value = self.format_llm_output(score_str)
+        return Metric(eval_name=self.name, value=value)

wxo_agentic_evaluation/metrics/journey_success.py

@@ -0,0 +1,137 @@
+import json
+from collections import defaultdict
+
+from langfuse.api.resources.commons.types.score_data_type import ScoreDataType
+
+from wxo_agentic_evaluation.metrics import Evaluation, argument_matching
+from wxo_agentic_evaluation.metrics.metrics import LangfuseMetric
+
+## fix later
+from wxo_agentic_evaluation.otel_parser.parser_types import (
+    Message as OtelMessage,
+)
+
+"""
+    - hyphens are not allowed in python function names, so it is safe to use as a dummy function name
+    - purpose behind `DUMMY_GRAPH_NODE_NAME` is to append
+    a dummy node to the ground truth and the labelled messages to take into account 
+    single, summary step goals.
+"""
+DUMMY_GRAPH_NODE_NAME = "dummy-goal"
+
+
+class JourneySuccessMetric(Evaluation):
+    def __init__(self, llm_client=None):
+        super().__init__(llm_client)
+        self.is_strict = True
+
+    @property
+    def name(self):
+        return "Journey Success"
+
+    def find_terminal_nodes(self, graph: dict[str, list[str]]) -> set[str]:
+        """Finds terminal nodes (nodes with no outgoing edges).
+
+        Args:
+            graph: the input graph
+
+        Returns:
+            a set of the terminal nodes
+        """
+
+        seen_nodes = set()  # track seen nodes
+        non_terminal_nodes = set()  # track nodes with children
+
+        for node in graph:
+            seen_nodes.add(node)
+            if graph[node]:
+                non_terminal_nodes.add(node)
+                for n in graph[node]:
+                    seen_nodes.add(n)
+        return seen_nodes - non_terminal_nodes
+
+    def is_topological_sort(
+        self,
+        graph: dict[str, list[str]],
+        ordering: list[str],
+        is_strict: bool = True,
+    ) -> bool:
+        """Graph traversal to check if every node in `graph` is visited in `ordering` only after all its dependencies are visited.
+
+        Args:
+            graph: the graph representing the ground truth, where keys represent nodes and values represent its dependent nodes
+            ordering: the nodes visited, in order
+
+        Returns:
+            Boolean representing if `ordering` visits all nodes in a valid order based on the dependencies in graph.
+        """
+        # No keyword match or goal details were achieved
+        if not ordering:
+            return False
+
+        if is_strict:
+            # strict matching: only consider most recent tool call
+            position = {node: [i] for i, node in enumerate(ordering)}
+        else:
+            # lenient matching: consider all tool calls (account for all indexes of the node)
+            position = defaultdict(list)
+            for i, node in enumerate(ordering):
+                position[node].append(i)
+
+        terminal_nodes = self.find_terminal_nodes(graph)
+        # adds a dummy node for each terminal node
+        next_idx = (
+            max(val for values in position.values() for val in values) + 1
+        )
+
+        for n in terminal_nodes:
+            graph[n] = [DUMMY_GRAPH_NODE_NAME]
+            graph[DUMMY_GRAPH_NODE_NAME] = []
+            position[DUMMY_GRAPH_NODE_NAME] = [next_idx]
+            next_idx += 1
+
+        for node in graph:
+            for child_nodes in graph[node]:
+                # Current node/children doesn't show up in made calls
+                if node not in position or child_nodes not in position:
+                    return False
+                # Current node doesn't show up before any of its child
+                # all index in current nodes are larger than every child nodes' index
+                if all(
+                    curr >= max(position[child_nodes])
+                    for curr in position[node]
+                ):
+                    return False
+        return True
+
+    def evaluate(
+        self, messages, ground_truth, extracted_context, metadata, **kwargs
+    ):
+        labeled_messages = extracted_context.get("labeled_messages")
+        correct_tool_calls = []
+
+        for message_idx, matching_goal_details in labeled_messages.items():
+            msg_tool_call = messages[message_idx]
+            msg_tool_call = msg_tool_call.tool_calls[0].function
+            for goal_detail in matching_goal_details:
+                args_match = argument_matching(
+                    expected=goal_detail.args,
+                    actual=None if len(msg_tool_call.arguments) == 0 else json.loads(msg_tool_call.arguments),
+                )
+
+                if args_match:
+                    correct_tool_calls.append(goal_detail.name)
+
+        is_topological_sort = self.is_topological_sort(
+            graph=ground_truth.goals,
+            ordering=correct_tool_calls,
+            is_strict=self.is_strict,
+        )
+
+        return LangfuseMetric(
+            eval_name=self.name,
+            comment="",
+            value=is_topological_sort,
+            data_type="NUMERIC",
+            metadata=metadata,
+        )

wxo_agentic_evaluation/metrics/llm_as_judge.py

@@ -44,3 +44,29 @@ class AnswerRelevancy(BaseLLMJudgeMetric):
             "answer_relevancy": self.answer_relevancy,
             "answer_relevancy_score": self.answer_relevancy_score,
         }
+
+
+class AnswerDerailment(BaseLLMJudgeMetric):
+    in_scope: str | float
+    statement: str
+    reason: str
+
+    def table(self):
+        return {
+            "statement": self.statement,
+            "reason": self.reason,
+            "on_topic_score": str(self.in_scope),
+        }
+
+
+class AnswerUnsafeTopic(BaseLLMJudgeMetric):
+    is_safe: str | float
+    statement: str
+    reason: str
+
+    def table(self):
+        return {
+            "statement": self.statement,
+            "reason": self.reason,
+            "safe_topic_score": str(self.is_safe),
+        }

wxo_agentic_evaluation/metrics/metrics.py

@@ -1,8 +1,9 @@
-import math
-from enum import Enum
-from typing import Any, List, Mapping, Optional, Tuple
+from collections import defaultdict
+from enum import Enum, StrEnum
+from typing import Any, Dict, List, Mapping, Optional, Tuple
 
 from pydantic import BaseModel, computed_field
+from pydantic.fields import Field
 
 from wxo_agentic_evaluation.metrics.llm_as_judge import (
     AnswerRelevancy,
@@ -11,12 +12,34 @@ from wxo_agentic_evaluation.metrics.llm_as_judge import (
 from wxo_agentic_evaluation.type import ConversationalConfidenceThresholdScore
 
 
-def average(array):
-    if len(array) == 0:
-        return math.nan
+class DescriptionQuality(StrEnum):
+    GOOD = "GOOD"
+    BAD = "BAD"
+    MISSING = "MISSING"
 
-    else:
-        return sum(array) / len(array)
+
+class DescriptionQualityMetric(BaseModel):
+    tool_name: str = None
+    description_score: float | None = None
+    threshold: float | None = None
+
+    @computed_field
+    @property
+    def is_bad_description(self) -> Optional[bool]:
+        if self.description_score and self.threshold:
+            return self.description_score >= self.threshold
+
+        return None
+
+    @computed_field
+    @property
+    def description_quality(self) -> str:
+        if self.description_score is None:
+            return DescriptionQuality.MISSING
+        elif self.is_bad_description:
+            return DescriptionQuality.BAD
+        else:
+            return DescriptionQuality.GOOD
 
 
 class KnowledgeBaseMetrics(BaseModel):
@@ -72,6 +95,8 @@ class KnowledgeBaseMetricSummary(BaseModel):
     @computed_field(alias="summary")
     @property
     def average(self) -> Mapping[str, Any]:
+        from wxo_agentic_evaluation.utils.utils import average
+
         summary = {}
         for dataset, metric in self.groupby_dataset.items():
             average_metric = {}
@@ -175,6 +200,13 @@ class ToolCallAndRoutingMetrics(BaseModel):
         )
 
 
+class Annotation(BaseModel):
+    recommendation: str
+    details: str
+    quote: str
+    parameter_name: Optional[str]
+
+
 class FailedStaticTestCases(BaseModel):
     metric_name: str
     description: str
@@ -187,6 +219,15 @@ class FailedSemanticTestCases(BaseModel):
     explanation: str
     output: int
     confidence: float
+    annotations: Optional[List[Annotation]] = None
+
+
+class EnhancedAnalyzeMetrics(BaseModel):
+    test_case_name: str
+    tool_names: List[str]
+    parameter_annotations: List[List[FailedSemanticTestCases]] = [[]]
+    tool_annotations: List[List[FailedSemanticTestCases]] = [[]]
+    static_metrics: List[List[FailedStaticTestCases]] = [[]]
 
 
 class ReferenceLessEvalMetrics(BaseModel):
@@ -201,3 +242,230 @@ class ReferenceLessEvalMetrics(BaseModel):
     failed_semantic_tool_calls: Optional[
         List[Tuple[int, List[FailedSemanticTestCases]]]
     ]
+
+
+class Metric(BaseModel):
+    """Generic metric result."""
+
+    eval_name: str = Field(description="name of eval that produce metric")
+    value: int | float | bool | str = Field(description="metric value")
+    metadata: Optional[dict] = Field(
+        default=None,
+        description="metadata that was generated along side the metric. example: llmaaj reason, retrieval score",
+    )
+
+
+class LangfuseMetric(Metric):
+    comment: Optional[str] = ""
+    data_type: Optional[str] = ""
+
+
+class CustomEvalMetrics(BaseModel):
+    dataset_name: str
+    custom_metrics: list[Metric]
+
+
+def create_avg_row(metrics: List[Dict[str, Any]]) -> Dict[str, Any]:
+    """
+    Create an average row from a list of metric dictionaries.
+
+    Args:
+        metrics: List of metric dictionaries
+
+    Returns:
+        Dictionary with averaged metrics
+    """
+    from wxo_agentic_evaluation.utils.utils import safe_divide
+
+    avg_row = {
+        "Dataset": "Summary (Average)",
+        "Runs": 0,
+        "Total Steps": 0,
+        "LLM Steps": 0,
+        "Total Tool Calls": 0,
+        "Tool Call Precision": 0,
+        "Tool Call Recall": 0,
+        "Agent Routing Accuracy": 0,
+        "Text Match": 0,
+        "Journey Success": 0,
+        "Avg Resp Time (sec)": 0,
+    }
+
+    if metrics:
+        for row in metrics:
+            avg_row["Runs"] += row.get("Runs", 0)
+            avg_row["Total Steps"] += row["Total Steps"]
+            avg_row["LLM Steps"] += row["LLM Steps"]
+            avg_row["Total Tool Calls"] += row["Total Tool Calls"]
+            avg_row["Tool Call Precision"] += row["Tool Call Precision"]
+            avg_row["Tool Call Recall"] += row["Tool Call Recall"]
+            avg_row["Agent Routing Accuracy"] += row["Agent Routing Accuracy"]
+            avg_row["Text Match"] += row["Text Match"]
+            avg_row["Journey Success"] += row["Journey Success"]
+            avg_row["Avg Resp Time (sec)"] += row["Avg Resp Time (sec)"]
+
+        n = len(metrics)
+        # Average over datasets
+        avg_row["Runs"] = round(safe_divide(avg_row["Runs"], n), 2)
+        avg_row["Total Steps"] = round(
+            safe_divide(avg_row["Total Steps"], n), 2
+        )
+        avg_row["LLM Steps"] = round(safe_divide(avg_row["LLM Steps"], n), 2)
+        avg_row["Total Tool Calls"] = round(
+            safe_divide(avg_row["Total Tool Calls"], n), 2
+        )
+        avg_row["Tool Call Precision"] = round(
+            safe_divide(avg_row["Tool Call Precision"], n), 2
+        )
+        avg_row["Tool Call Recall"] = round(
+            safe_divide(avg_row["Tool Call Recall"], n), 2
+        )
+        avg_row["Agent Routing Accuracy"] = round(
+            safe_divide(avg_row["Agent Routing Accuracy"], n), 2
+        )
+        avg_row["Text Match"] = round(safe_divide(avg_row["Text Match"], n), 2)
+        avg_row["Journey Success"] = round(
+            safe_divide(avg_row["Journey Success"], n), 2
+        )
+        avg_row["Avg Resp Time (sec)"] = round(
+            safe_divide(avg_row["Avg Resp Time (sec)"], n), 2
+        )
+
+    return avg_row
+
+
+def format_metrics_for_display(
+    tool_call_metrics: list[ToolCallAndRoutingMetrics],
+) -> list[dict[str, Any]]:
+    from wxo_agentic_evaluation.utils.utils import mean, safe_divide, to_pct
+
+    # Group metrics by dataset name
+    grouped = defaultdict(list)
+    for m in tool_call_metrics:
+        grouped[m.dataset_name].append(
+            {
+                "Dataset": m.dataset_name,
+                "Total Steps": m.total_steps,
+                "LLM Steps": m.llm_step,
+                "Total Tool Calls": m.total_tool_calls,
+                "Tool Call Precision": m.tool_call_precision,
+                "Tool Call Recall": m.tool_call_recall,
+                "Agent Routing Accuracy": m.agent_routing_accuracy,
+                "Text Match": m.text_match,
+                "Journey Success": m.is_success,
+                "Avg Resp Time (sec)": m.avg_resp_time,
+            }
+        )
+
+    # Create per-test rows with averages over runs
+    per_test_rows = []
+    for ds, rows in grouped.items():
+        out = {"Dataset": ds}
+
+        # Average numeric columns over runs
+        numeric_keys = [
+            "Total Steps",
+            "LLM Steps",
+            "Total Tool Calls",
+            "Tool Call Precision",
+            "Tool Call Recall",
+            "Agent Routing Accuracy",
+            "Avg Resp Time (sec)",
+        ]
+
+        for k in numeric_keys:
+            out[k] = mean(
+                [r[k] for r in rows if isinstance(r.get(k), (int, float))]
+            )
+
+        # Add total runs per dataset
+        out["Runs"] = round(float(len(rows)), 2)
+
+        # Journey Success -> numeric fraction in [0,1]
+        js_vals = [1 if bool(r.get("Journey Success")) else 0 for r in rows]
+        out["Journey Success"] = round(
+            safe_divide(sum(js_vals), len(js_vals)), 2
+        )
+
+        # Text Match -> numeric fraction in [0,1]
+        tm_hits = 0
+        tm_den = len(rows)
+        for r in rows:
+            val = r.get("Text Match")
+            if str(val).strip() == TextMatchType.text_match.value:
+                tm_hits += 1
+        out["Text Match"] = round(safe_divide(tm_hits, tm_den), 2)
+
+        per_test_rows.append(out)
+
+    # Create overall average row
+    overall_row = create_avg_row(per_test_rows)
+
+    # Format percentages
+    tool_call_metrics_for_display = per_test_rows + [overall_row]
+    for row in tool_call_metrics_for_display:
+        row["Text Match"] = to_pct(row.get("Text Match"), decimals=0)
+        row["Journey Success"] = to_pct(row.get("Journey Success"), decimals=0)
+
+    column_order = [
+        "Dataset",
+        "Runs",
+        "Total Steps",
+        "LLM Steps",
+        "Total Tool Calls",
+        "Tool Call Precision",
+        "Tool Call Recall",
+        "Agent Routing Accuracy",
+        "Text Match",
+        "Journey Success",
+        "Avg Resp Time (sec)",
+    ]
+
+    tool_call_metrics_for_display = [
+        {col: row.get(col, "") for col in column_order}
+        for row in tool_call_metrics_for_display
+    ]
+
+    return tool_call_metrics_for_display
+
+
+def extract_metrics(
+    results: List[
+        Tuple[
+            ToolCallAndRoutingMetrics,
+            KnowledgeBaseMetricSummary,
+            CustomEvalMetrics,
+        ]
+    ],
+) -> tuple[
+    list[ToolCallAndRoutingMetrics],
+    KnowledgeBaseMetricSummary,
+    List[CustomEvalMetrics],
+]:
+    """
+    Aggregate metrics from test results.
+
+    Args:
+        results: List of tuples (metrics, knowledge_base_metrics, custom_metrics)
+
+    Returns:
+        Tuple of (knowledge_base_summary, tool_rows, custom_metrics)
+    """
+
+    tool_call_metrics = [metric[0] for metric in results]
+    knowledge_base_metrics = [metric[1] for metric in results]
+    custom_metrics: List[CustomEvalMetrics] = [metric[2] for metric in results]
+
+    kb_summary = KnowledgeBaseMetricSummary(
+        knowledge_base_metrics=knowledge_base_metrics
+    )
+
+    if len(tool_call_metrics) > 0:
+        # Remove the average row if it exists
+        tool_call_metrics = [
+            row
+            for row in tool_call_metrics
+            if row.dataset_name != "Summary (Average)"
+        ]
+
+    return tool_call_metrics, kb_summary, custom_metrics

wxo_agentic_evaluation/metrics/tool_calling.py

@@ -0,0 +1,93 @@
+import json
+from typing import List, Union
+
+from wxo_agentic_evaluation.metrics import Evaluation, argument_matching
+from wxo_agentic_evaluation.metrics.metrics import (
+    LangfuseMetric,
+    ToolCallAndRoutingMetrics,
+)
+from wxo_agentic_evaluation.type import ContentType
+
+
+class ToolCalling(Evaluation):
+    @property
+    def name(self):
+        return "Tool Calling Metrics"
+
+    def evaluate(
+        self, messages, ground_truth, extracted_context, metadata, **kwargs
+    ) -> Union[LangfuseMetric, List[LangfuseMetric]]:
+        dataset_name = kwargs.get("dataset", "")
+
+        total_tool_calls = 0
+        relevant_tool_calls = 0
+        tool_calls_with_incorrect_parameter = 0
+        correct_tool_calls = set()
+
+        tool_dictionary = (
+            {
+                goal_detail.name: goal_detail
+                for goal_detail in ground_truth.goal_details
+                if goal_detail.type == ContentType.tool_call
+            }
+            if ground_truth.goal_details
+            else {}
+        )
+
+        labeled_messages = extracted_context.get("labeled_messages")
+        total_tool_calls = len(
+            [
+                message
+                for message in messages
+                if message.type == ContentType.tool_call
+            ]
+        )
+        relevant_tool_calls = len(labeled_messages)
+
+        for message_idx, matching_goal_details in labeled_messages.items():
+            msg_tool_call = messages[message_idx]
+            msg_tool_call = msg_tool_call.tool_calls[0].function
+            for goal_detail in matching_goal_details:
+                # TODO flesh out to match ADK EVAL
+                args_match = argument_matching(
+                    expected=goal_detail.args,
+                    actual=None if len(msg_tool_call.arguments) == 0 else json.loads(msg_tool_call.arguments),
+                )
+
+                if args_match:
+                    correct_tool_calls.add(goal_detail.name)
+                else:
+                    tool_calls_with_incorrect_parameter += 1
+
+        # TODO: think about the dataset name
+        # TODO: total_steps
+        tool_call_metrics = ToolCallAndRoutingMetrics(
+            dataset_name=dataset_name,
+            total_tool_calls=total_tool_calls,
+            expected_tool_calls=len(tool_dictionary),
+            correct_tool_calls=len(correct_tool_calls),
+            relevant_tool_calls=relevant_tool_calls,
+            tool_calls_with_incorrect_parameter=tool_calls_with_incorrect_parameter,
+        )
+
+        tool_call_metrics = tool_call_metrics.model_dump()
+
+        metrics = []
+
+        for tool in [
+            "total_tool_calls",
+            "correct_tool_calls",
+            "expected_tool_calls",
+            "tool_calls_with_incorrect_parameter",
+            "tool_call_recall",
+            "tool_call_precision",
+        ]:
+            metric = LangfuseMetric(
+                eval_name=tool,
+                value=tool_call_metrics.get(tool),
+                metadata=metadata,
+                data_type="NUMERIC",
+            )
+            metrics.append(metric)
+
+        return metrics

wxo_agentic_evaluation/otel_parser/__init__.py

@@ -0,0 +1 @@
+from wxo_agentic_evaluation.otel_parser import parser as otel_parser