ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/utils/utils.py

@@ -1,24 +1,179 @@
+import csv
+import glob
+import json
+import math
+import os
+import re
+from collections import defaultdict
+from pathlib import Path
+from typing import Any, Dict, List, Mapping, Optional, Tuple, Union
 from urllib.parse import urlparse
+
+import rich
+import yaml
+from rich import box, print
 from rich.console import Console, Group
-from rich.table import Table
 from rich.panel import Panel
 from rich.rule import Rule
-from rich import box
-from rich import print
-
-from typing import List
+from rich.style import Style
+from rich.table import Table
 
 from wxo_agentic_evaluation.metrics.llm_as_judge import Faithfulness
-from wxo_agentic_evaluation.metrics.metrics import KnowledgeBaseMetricSummary
-from wxo_agentic_evaluation.type import ConversationalConfidenceThresholdScore
+from wxo_agentic_evaluation.metrics.metrics import (
+    KnowledgeBaseMetricSummary,
+    ReferenceLessEvalMetrics,
+    ToolCallAndRoutingMetrics,
+)
+from wxo_agentic_evaluation.type import (
+    ConversationalConfidenceThresholdScore,
+    ExtendedMessage,
+    Message,
+)
 
 console = Console()
 
+RUN_FILE_RE = re.compile(
+    r"^(?P<base>.+)\.run(?P<run>\d+)\.(?P<kind>messages(?:\.analyze)?|metrics)\.json$"
+)
+N_A = "N/A"
+
+# File name constants
+REFERENCE_FILE_NAME = "reference"
+EXPERIMENT_FILE_NAME = "experiment"
+
+
+class AttackResultsTable:
+    def __init__(self, attack_results: dict):
+        self.table = Table(
+            title="Attack Results",
+            box=box.ROUNDED,
+            show_lines=True,
+        )
+        self.table.add_column("Attack Category", style="magenta")
+        self.table.add_column("Count", style="cyan")
+        self.table.add_column("Success Rate", style="green")
+
+        # Extract values
+        n_on_policy = attack_results.get("n_on_policy_attacks", 0)
+        n_off_policy = attack_results.get("n_off_policy_attacks", 0)
+        n_on_policy_successful = attack_results.get("n_on_policy_successful", 0)
+        n_off_policy_successful = attack_results.get(
+            "n_off_policy_successful", 0
+        )
+
+        # Calculate success rates
+        on_policy_rate = (
+            f"{round(100 * safe_divide(n_on_policy_successful, n_on_policy))}%"
+            if n_on_policy
+            else "0%"
+        )
+        off_policy_rate = (
+            f"{round(100 * safe_divide(n_off_policy_successful, n_off_policy))}%"
+            if n_off_policy
+            else "0%"
+        )
+
+        self.table.add_row("On Policy", str(n_on_policy), on_policy_rate)
+        self.table.add_row("Off Policy", str(n_off_policy), off_policy_rate)
+
+    def print(self):
+        console.print(self.table)
+
+
+class TestCaseResources:
+    def __init__(self, output_dir: str):
+        """Todo flesh out for all resources that are saved"""
+        self.output_dir = Path(output_dir)
+
+    @property
+    def get_summary(self):
+        summary = []
+
+        with open(self.output_dir / "summary_metrics.csv", "r") as f:
+            reader = csv.reader(f)
+            header = next(reader)
+            for row in reader:
+                summary.append(dict(zip(header, row)))
+
+        return summary
+
+    def get_analyze_messages(
+        self, test_case_name=None, path=None
+    ) -> Tuple[List[ExtendedMessage], Mapping[str, Any]]:
+        test_messages = []
+
+        if test_case_name:
+            path = os.path.join(
+                self.output_dir,
+                "messages",
+                f"{test_case_name}.messages.analyze.json",
+            )
+
+        if not Path(str(path)).is_file():
+            rich.print(f"[r]No analyze file found at {path}")
+            raise Exception(f"No analyze file found at {path}")
+
+        with open(path, "r", encoding="utf-8") as f:
+            temp = json.load(f)
+            meta = None
+            if temp and isinstance(temp[-1], dict) and "meta" in temp[-1]:
+                meta = temp[-1]["meta"]
+                temp = temp[:-1]
+
+            for entry in temp:
+                msg = ExtendedMessage(**entry)
+                test_messages.append(msg)
+
+        return test_messages, meta
+
+    def get_messages(self, test_case_name=None, path=None) -> List[Message]:
+        test_messages = []
+
+        if test_case_name:
+            path = os.path.join(
+                self.output_dir,
+                "messages",
+                f"{test_case_name}.messages.json",
+            )
+
+        if not Path(str(path)).is_file():
+            rich.print(f"[r]No messages file found at {path}")
+            raise Exception(f"No messages file found at {path}")
+
+        with open(path, "r", encoding="utf-8") as f:
+            temp = json.load(f)
+            for entry in temp:
+                msg = Message(**entry)
+                test_messages.append(msg)
+
+        return test_messages
+
+    def get_test_metrics(
+        self, test_case_name=None, path=None
+    ) -> ToolCallAndRoutingMetrics:
+        if test_case_name:
+            path = os.path.join(
+                self.output_dir,
+                "messages",
+                f"{test_case_name}.metrics.json",
+            )
+
+        if not Path(str(path)).is_file():
+            rich.print(f"[r]No metrics file found at {path}")
+            raise Exception(f"No metrics file found at {path}")
+
+        with open(path, "r", encoding="utf-8") as f:
+            metrics = ToolCallAndRoutingMetrics(**json.load(f))
+
+        return metrics
+
 
 class AgentMetricsTable:
-    def __init__(self, data):
+    def __init__(self, data, title: Optional[str] = None):
+        if title is None:
+            title = "Agent Metrics"
         self.table = Table(
-            title="Agent Metrics",
+            title=title,
             box=box.ROUNDED,
             show_lines=True,
         )
@@ -39,7 +194,9 @@ class AgentMetricsTable:
         console.print(self.table)
 
 
-def create_table(data: List[dict]) -> AgentMetricsTable:
+def create_table(
+    data: List[dict], title: Optional[str] = None
+) -> AgentMetricsTable:
     """
     Generate a Rich table from a list of dictionaries.
     Returns the AgentMetricsTable instance.
@@ -51,14 +208,55 @@ def create_table(data: List[dict]) -> AgentMetricsTable:
         print("create_table() received an empty dataset. No table generated.")
         return None
 
-    return AgentMetricsTable(data)
+    return AgentMetricsTable(data, title=title)
+
+
+def mean(vals: List[float]) -> float:
+    """
+    Calculate the mean of a list of values.
+
+    Args:
+        vals: List of values
+
+    Returns:
+        Mean value
+    """
+    return round(sum(vals) / len(vals), 2) if vals else 0.0
+
+
+def to_pct(value: float | None, decimals: int = 0) -> str:
+    """
+    Convert a value to a percentage string.
+
+    Args:
+        value: Value to convert
+        decimals: Number of decimal places
+
+    Returns:
+        Percentage string
+    """
+    if value is None:
+        return "NA"
+    try:
+        return f"{round(float(value) * 100, decimals)}%"
+    except Exception:
+        return "NA"
+
+
+def average(array) -> float:
+    if len(array) == 0:
+        return math.nan
+
+    else:
+        return sum(array) / len(array)
 
 
 def safe_divide(nom, denom):
     if denom == 0:
         return 0
     else:
-        return nom/denom
+        return nom / denom
+
 
 def is_saas_url(service_url: str) -> bool:
     hostname = urlparse(service_url).hostname
@@ -70,22 +268,132 @@ def is_ibm_cloud_url(service_url: str) -> bool:
     return ".cloud.ibm.com" in hostname
 
 
-def add_line_seperator():
-    console.print(Rule(style="grey42"))
+def add_line_seperator(
+    style_config: Optional[Union[str, Style]] = None,
+    print=True,
+):
+    """
+    Adds a lined seperator provided the style config.
+    `print` is a boolean to indicate if the lined seperator should go to stdout immeadiatly or returned as an object.
+    Set `print` to False, the lined seperator is printed later as part of the pager view for example.
+    """
+
+    if not style_config:
+        style = "grey42"
+    else:
+        style = style_config
+
+    if print:
+        console.print(
+            Rule(
+                style=style,
+            )
+        )
+    else:
+        return Rule(style=style, characters="==")
+
+
+def get_reference_column(base_name: str) -> str:
+    """Generate a column name with the reference suffix."""
+    return f"{base_name}_{REFERENCE_FILE_NAME}"
+
+
+def get_experiment_column(base_name: str) -> str:
+    """Generate a column name with the experiment suffix."""
+    return f"{base_name}_{EXPERIMENT_FILE_NAME}"
+
+
+def get_diff_column(base_name: str) -> str:
+    """Generate a diff column name."""
+    return f"{base_name}_diff"
+
+
+def get_column_value(
+    row: Dict[str, Any], base_name: str, file_type: str
+) -> Any:
+    """Get a value from a column with the appropriate suffix.
+
+    Args:
+        row: The data row
+        base_name: The base column name
+        file_type: Either 'reference' or 'experiment'
+
+    Returns:
+        The value from the column, or None if not found
+    """
+    if file_type.lower() == "reference":
+        key = get_reference_column(base_name)
+    elif file_type.lower() == "experiment":
+        key = get_experiment_column(base_name)
+    else:
+        raise ValueError(f"Invalid file_type: {file_type}")
+
+    return row.get(key)
+
+
+def has_column_in_both(row: Dict[str, Any], base_name: str) -> bool:
+    """Check if a column exists with both reference and experiment suffixes."""
+    return (
+        get_reference_column(base_name) in row
+        and get_experiment_column(base_name) in row
+    )
+
+
+def format_ratio(ratio: Optional[float]) -> str:
+    """Format a ratio as a percentage string."""
+    if ratio is None:
+        return "N/A"
+    return f"{ratio * 100:.1f}%"
+
+
+def read_file(path: str, type: str = "csv") -> List[Dict[str, Any]]:
+    """Read a file and return its contents as a structured object."""
+    if type == "csv":
+        return read_csv_file(path)
+    elif type == "json":
+        # Add JSON reading logic if needed
+        raise NotImplementedError("JSON reading not yet implemented")
+    else:
+        raise ValueError(f"Unsupported file type: {type}")
+
+
+def read_csv_file(file_path: str) -> List[Dict[str, Any]]:
+    """Read a CSV file and return a list of dictionaries."""
+    data = []
+    with open(file_path, "r") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            # Convert numeric values to appropriate types
+            for key, value in row.items():
+                if key == "dataset_name" or key == "text_match":
+                    continue
+                elif key == "is_success":
+                    row[key] = value.lower() == "true"
+                else:
+                    try:
+                        row[key] = float(value)
+                    except ValueError:
+                        pass
+            data.append(row)
+    return data
 
 
 class FaithfulnessTable:
     def __init__(
         self, faithfulness_metrics: List[Faithfulness], tool_call_ids: List[str]
     ):
-        self.table = Table(title="Faithfulness", box=box.ROUNDED, show_lines=True)
+        self.table = Table(
+            title="Faithfulness", box=box.ROUNDED, show_lines=True
+        )
 
         self.table.add_column("Tool Call Id", style="blue")
         self.table.add_column("Faithfulness Score", style="blue3")
         self.table.add_column("Evidence", style="cyan")
         self.table.add_column("Reasoning", style="yellow3")
 
-        for tool_call_id, faithfulness in zip(tool_call_ids, faithfulness_metrics):
+        for tool_call_id, faithfulness in zip(
+            tool_call_ids, faithfulness_metrics
+        ):
             faithfulness = faithfulness.table()
             self.table.add_row(
                 tool_call_id,
@@ -139,7 +447,9 @@ class KnowledgePanel:
         self.confidence_scores = ConversationalSearchTable(
             confidence_scores, tool_call_id
         )
-        self.group = Group(self.faithfulness.table, self.confidence_scores.table)
+        self.group = Group(
+            self.faithfulness.table, self.confidence_scores.table
+        )
 
         # Panel acts as a section
         self.section = Panel(
@@ -183,3 +493,190 @@ class SummaryPanel:
 
     def print(self):
         console.print(self.table)
+
+
+class Tokenizer:
+    PATTERN = r"""
+            \w+(?=n't)|              # Words before n't contractions (e.g., "do" in "don't")
+            n't|                     # n't contractions themselves
+            \w+(?=')|                # Words before apostrophes (e.g., "I" in "I'm")
+            '|                       # Apostrophes as separate tokens
+            \w+|                     # Regular words (letters, numbers, underscores)
+            [^\w\s]                  # Punctuation marks (anything that's not word chars or whitespace)
+        """
+
+    def __init__(self):
+        self.compiled_pattern = re.compile(
+            self.PATTERN, re.VERBOSE | re.IGNORECASE
+        )
+
+    def __call__(self, text: str) -> List[str]:
+        """
+        Tokenizes text by splitting on punctuation and handling contractions.
+
+        Args:
+            text: Input text to tokenize.
+
+        Returns:
+            List of tokenized words (lowercase, no punctuation).
+
+        Examples:
+            - "I'm fine"      -> ['i', 'm', 'fine']
+            - "don't go"      -> ['do', "n't", 'go']
+            - "Hello, world!" -> ['hello', 'world']
+        """
+
+        tokens = self.compiled_pattern.findall(text)
+
+        return self._clean_tokens(tokens)
+
+    def _clean_tokens(self, raw_tokens: List[str]) -> List[str]:
+        """
+        Applies some basic post-processing to tokenized messages.
+
+        Args:
+            raw_tokens: list of tokens extracted from a message.
+        """
+
+        filtered_tokens = [
+            token.lower()
+            for token in raw_tokens
+            if token.strip() and not (len(token) == 1 and not token.isalnum())
+        ]
+
+        return filtered_tokens
+
+
+class ReferencelessEvalPanel:
+    def __init__(self, referenceless_metrics: List[ReferenceLessEvalMetrics]):
+        self.table = Table(
+            title="Quick Evaluation Summary Metrics",
+            box=box.ROUNDED,
+            show_lines=True,
+        )
+
+        self.table.add_column("Dataset", style="yellow", justify="center")
+        self.table.add_column(
+            "Tool Calls", style="deep_sky_blue1", justify="center"
+        )
+        self.table.add_column(
+            "Successful Tool Calls", style="magenta", justify="center"
+        )
+        self.table.add_column(
+            "Tool Calls Failed due to Schema Mismatch",
+            style="deep_sky_blue1",
+            justify="center",
+        )
+        self.table.add_column(
+            "Tool Calls Failed due to Hallucination",
+            style="magenta",
+            justify="center",
+        )
+
+        for metric in referenceless_metrics:
+            self.table.add_row(
+                str(metric.dataset_name),
+                str(metric.number_of_tool_calls),
+                str(metric.number_of_successful_tool_calls),
+                str(metric.number_of_static_failed_tool_calls),
+                str(metric.number_of_semantic_failed_tool_calls),
+            )
+
+    def print(self):
+        console.print(self.table)
+
+
+# Function to load messages from JSON file
+def load_messages(file_path):
+    """TODO: replace in favor of TestCaseResources.get_messages(...)"""
+    with open(file_path, "r") as f:
+        try:
+            message_data = json.load(f)
+            messages = []
+            for msg in message_data:
+                messages.append(Message.model_validate(msg))
+
+            return messages
+
+        except Exception as e:
+            print(file_path)
+            print(e)
+            return None
+
+
+def load_agents_from_disk(agents_path: str):
+    agents_json = glob.glob(os.path.join(agents_path, "*.json"))
+    agents_yaml = glob.glob(os.path.join(agents_path, "*.yaml"))
+
+    agents = []
+
+    for agent_path in agents_json:
+        with open(agent_path, "r") as f:
+            agents.append(json.load(f))
+
+    for agent_path in agents_yaml:
+        with open(agent_path, "r") as f:
+            agents.append(yaml.safe_load(f))
+
+    return agents
+
+
+def list_run_files(messages_dir: str, dataset_base: str, filter_run: int = -1):
+    """
+    Returns: dict[run_id] -> {"analyze": path|None, "metrics": path|None}
+    (We only need analyze+metrics for this feature.)
+
+    `filter_run` only get gets the runs files for that run. If it is -1, then all run files are retrieved
+    For example, if there is `data3.run1.messages.json`, `data3.run2.messages.json`, and filter_run is 2, then,
+    the files related to only the second run are retrieved.
+
+    """
+    runs = defaultdict(
+        lambda: {"analyze": None, "metrics": None, "messages": None}
+    )
+    for fn in os.listdir(messages_dir):
+        m = RUN_FILE_RE.match(fn)
+        if not m or m.group("base") != dataset_base:
+            continue
+        run_id = int(m.group("run"))
+        if filter_run != -1 and run_id != filter_run:
+            continue
+
+        kind = m.group("kind")
+        full = os.path.join(messages_dir, fn)
+        if kind == "messages.analyze":
+            runs[run_id]["analyze"] = full
+        elif kind == "metrics":
+            runs[run_id]["metrics"] = full
+        elif kind == "messages":
+            runs[run_id]["messages"] = full
+    return runs
+
+
+def load_run_metrics(metrics_path: str) -> ToolCallAndRoutingMetrics:
+    """Todo remove in a later PR"""
+    with open(metrics_path, "r", encoding="utf-8") as f:
+        return ToolCallAndRoutingMetrics(**json.load(f))
+
+
+def csv_dump(file_path: Union[str, Path], rows: List[Dict[str, Any]]) -> None:
+    """
+    Write rows to a CSV file.
+
+    Args:
+        file_path: Path to the output CSV file
+        rows: List of dictionaries representing CSV rows
+    """
+    if not rows:
+        return
+
+    # Ensure the parent directory exists
+    if isinstance(file_path, str):
+        file_path = Path(file_path)
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write to CSV
+    with open(file_path, "w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=rows[0].keys())
+        writer.writeheader()
+        writer.writerows(rows)

wxo_agentic_evaluation/wxo_client.py

@@ -0,0 +1,81 @@
+import os
+from typing import Any, Dict, Optional
+
+import requests
+import urllib3
+from urllib3.exceptions import InsecureRequestWarning
+
+from wxo_agentic_evaluation.service_instance import tenant_setup
+
+
+class WXOClient:
+    def __init__(
+        self, service_url, api_key, env: Optional[Dict[str, Any]] = None
+    ):
+        self.service_url = service_url
+        self.api_key = api_key
+
+        ov = os.getenv("WO_SSL_VERIFY")
+        if ov and ov.strip().lower() in ("true", "false"):
+            self._verify_ssl = ov.strip().lower() == "true"
+        else:
+            v, bs = (env.get("verify") if env else None), (
+                env.get("bypass_ssl") if env else None
+            )
+            self._verify_ssl = (
+                False
+                if (
+                    (bs is True)
+                    or (isinstance(bs, str) and bs.strip().lower() == "true")
+                    or (v is None)
+                    or (
+                        isinstance(v, str)
+                        and v.strip().lower() in {"none", "null"}
+                    )
+                )
+                else (v if isinstance(v, bool) else True)
+            )
+
+        if not self._verify_ssl:
+            urllib3.disable_warnings(InsecureRequestWarning)
+
+    def _get_headers(self) -> dict:
+        headers = {}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    def post(self, payload: dict, path: str, stream=False):
+        url = f"{self.service_url}/{path}"
+        return requests.post(
+            url=url,
+            headers=self._get_headers(),
+            json=payload,
+            stream=stream,
+            verify=self._verify_ssl,
+        )
+
+    def get(self, path: str, params: dict = None):
+        url = f"{self.service_url}/{path}"
+        return requests.get(
+            url,
+            params=params,
+            headers=self._get_headers(),
+            verify=self._verify_ssl,
+        )
+
+
+def get_wxo_client(
+    service_url: Optional[str], tenant_name: str, token: Optional[str] = None
+) -> WXOClient:
+
+    token, resolved_url, env = tenant_setup(service_url, tenant_name)
+    service_url = service_url or resolved_url
+
+    if not (service_url and str(service_url).strip()):
+        raise ValueError(
+            f"service_url not provided and not found in config for tenant '{tenant_name}'"
+        )
+
+    wxo_client = WXOClient(service_url=service_url, api_key=token, env=env)
+    return wxo_client