ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/compare_runs/diff.py

@@ -0,0 +1,554 @@
+import csv
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional, Tuple
+
+from wxo_agentic_evaluation.compare_runs.model import EvaluationResult
+from wxo_agentic_evaluation.utils.utils import (
+    EXPERIMENT_FILE_NAME,
+    REFERENCE_FILE_NAME,
+    create_table,
+    get_column_value,
+    get_diff_column,
+    get_experiment_column,
+    get_reference_column,
+    has_column_in_both,
+    safe_divide,
+)
+
+# Status constants
+STATUS_ONLY_IN_REFERENCE = f"Only in {REFERENCE_FILE_NAME.title()}"
+STATUS_ONLY_IN_EXPERIMENT = f"Only in {EXPERIMENT_FILE_NAME.title()}"
+STATUS_IN_BOTH = f"In both {REFERENCE_FILE_NAME} and {EXPERIMENT_FILE_NAME}"
+
+
+@dataclass
+class DiffResults:
+    """Class for comparing two evaluation results."""
+
+    result1: EvaluationResult
+    result2: EvaluationResult
+
+    # Private cache for filtered data - not included in constructor
+    _filtered_data: Dict[str, List] = field(default_factory=dict, init=False)
+
+    def _filter_joined_data(self) -> Dict[str, List[Dict[str, Any]]]:
+        """Filter joined data into categories and cache the results.
+
+        This method caches the results to avoid repeated filtering operations.
+        """
+        # Return cached results if already computed
+        if self._filtered_data:
+            return self._filtered_data
+
+        # Get the joined data
+        joined_data = self.join_datasets()
+
+        # Filter data into categories
+        only_in_reference = []
+        only_in_experiment = []
+        in_both = []
+
+        for row in joined_data:
+            if row["status"] == STATUS_ONLY_IN_REFERENCE:
+                only_in_reference.append(row)
+            elif row["status"] == STATUS_ONLY_IN_EXPERIMENT:
+                only_in_experiment.append(row)
+            elif row["status"] == STATUS_IN_BOTH:
+                in_both.append(row)
+
+        # Cache the filtered data
+        self._filtered_data = {
+            "all": joined_data,
+            "only_in_reference": only_in_reference,
+            "only_in_experiment": only_in_experiment,
+            "in_both": in_both,
+        }
+
+        return self._filtered_data
+
+    def get_overlapping_metrics(
+        self, result: EvaluationResult, other: EvaluationResult
+    ) -> Dict[str, Any]:
+        """Calculate metrics for tests that overlap between two result sets."""
+        overlapping_names = set(result.test_case_results.keys()) & set(
+            other.test_case_results.keys()
+        )
+
+        overlapping_count = len(overlapping_names)
+
+        overlapping_matched = sum(
+            result.test_case_results[name].matches_count()
+            for name in overlapping_names
+            if name in result.test_case_results
+        )
+
+        overlapping_success = sum(
+            1
+            for name in overlapping_names
+            if name in result.test_case_results
+            and result.test_case_results[name].is_success
+        )
+
+        return {
+            "count": overlapping_count,
+            "matched": overlapping_matched,
+            "success": overlapping_success,
+            "matched_ratio": safe_divide(
+                overlapping_matched, overlapping_count
+            ),
+            "success_ratio": safe_divide(
+                overlapping_success, overlapping_count
+            ),
+        }
+
+    def _join_datasets_on_name(
+        self, data1: List[Dict[str, Any]], data2: List[Dict[str, Any]]
+    ) -> List[Dict[str, Any]]:
+        """Join two datasets on dataset_name and compute differences."""
+        # Create dictionaries for quick lookup
+        dict1 = {row["dataset_name"]: row for row in data1}
+        dict2 = {row["dataset_name"]: row for row in data2}
+
+        # Get all unique dataset names
+        all_names = set(dict1.keys()) | set(dict2.keys())
+
+        # Initialize result
+        joined_data = []
+
+        for name in all_names:
+            result_row = {"dataset_name": name}
+
+            # Handle datasets that exist in only one file
+            if name not in dict1:
+                result_row["status"] = STATUS_ONLY_IN_EXPERIMENT
+                result_row.update(
+                    {
+                        get_experiment_column(k): v
+                        for k, v in dict2[name].items()
+                        if k != "dataset_name"
+                    }
+                )
+                joined_data.append(result_row)
+                continue
+
+            if name not in dict2:
+                result_row["status"] = STATUS_ONLY_IN_REFERENCE
+                result_row.update(
+                    {
+                        get_reference_column(k): v
+                        for k, v in dict1[name].items()
+                        if k != "dataset_name"
+                    }
+                )
+                joined_data.append(result_row)
+                continue
+
+            # Dataset exists in both files
+            result_row["status"] = STATUS_IN_BOTH
+
+            # Add values from both files and compute differences
+            row1 = dict1[name]
+            row2 = dict2[name]
+
+            for key in set(row1.keys()) | set(row2.keys()):
+                if key == "dataset_name":
+                    continue
+
+                # Handle keys that exist in only one file
+                if key not in row1:
+                    result_row[get_experiment_column(key)] = row2[key]
+                    result_row[get_diff_column(key)] = (
+                        f"{STATUS_ONLY_IN_EXPERIMENT}: {row2[key]}"
+                    )
+                    continue
+
+                if key not in row2:
+                    result_row[get_reference_column(key)] = row1[key]
+                    result_row[get_diff_column(key)] = (
+                        f"{STATUS_ONLY_IN_REFERENCE}: {row1[key]}"
+                    )
+                    continue
+
+                # Add values from both files
+                result_row[get_reference_column(key)] = row1[key]
+                result_row[get_experiment_column(key)] = row2[key]
+
+                # Compute difference
+                if isinstance(row1[key], (int, float)) and isinstance(
+                    row2[key], (int, float)
+                ):
+                    diff = row2[key] - row1[key]
+                    result_row[get_diff_column(key)] = diff
+                elif isinstance(row1[key], bool) and isinstance(
+                    row2[key], bool
+                ):
+                    result_row[get_diff_column(key)] = (
+                        "No change" if row1[key] == row2[key] else "Changed"
+                    )
+                else:
+                    result_row[get_diff_column(key)] = (
+                        "Same" if row1[key] == row2[key] else "Different"
+                    )
+
+            joined_data.append(result_row)
+        return joined_data
+
+    def join_datasets(self) -> List[Dict[str, Any]]:
+        """Join datasets from both evaluation results and compute differences."""
+        # Convert test case results to the format expected by _join_datasets_on_name
+        data1 = [
+            result.to_dict()
+            for result in self.result1.test_case_results.values()
+        ]
+        data2 = [
+            result.to_dict()
+            for result in self.result2.test_case_results.values()
+        ]
+        return self._join_datasets_on_name(data1, data2)
+
+    def summary_statistics(self) -> List[Dict[str, Any]]:
+        """Generate summary statistics comparing the two evaluation results."""
+        # Calculate overlapping metrics
+        overlap1 = self.get_overlapping_metrics(self.result1, self.result2)
+        overlap2 = self.get_overlapping_metrics(self.result2, self.result1)
+
+        overlapping_tests = overlap1["count"]  # Same for both
+
+        return [
+            {
+                "Metric": "Total Tests",
+                "Reference": self.result1.test_count,
+                "Experiment": self.result2.test_count,
+                "Experiment - Reference": self.result2.test_count
+                - self.result1.test_count,
+            },
+            {
+                "Metric": "Overlapping Tests",
+                "Reference": overlapping_tests,
+                "Experiment": overlapping_tests,
+                "Experiment - Reference": 0,
+            },
+            {
+                "Metric": "Summary Matches",
+                "Reference": overlap1["matched"],
+                "Experiment": overlap2["matched"],
+                "Experiment - Reference": overlap2["matched"]
+                - overlap1["matched"],
+            },
+            {
+                "Metric": "Is Success",
+                "Reference": overlap1["success"],
+                "Experiment": overlap2["success"],
+                "Experiment - Reference": overlap2["success"]
+                - overlap1["success"],
+            },
+            # {"Metric": "Summary Match / Overlapping Tests",
+            #  "Reference": format_ratio(overlap1['matched_ratio']),
+            #  "Experiment": format_ratio(overlap2['matched_ratio']),
+            #  "Difference": f"{(overlap2['matched_ratio'] - overlap1['matched_ratio']) * 100:.1f}%" if overlapping_tests > 0 else "N/A"},
+            # {"Metric": "Is Success / Overlapping Tests",
+            #  "Reference": format_ratio(overlap1['success_ratio']),
+            #  "Experiment": format_ratio(overlap2['success_ratio']),
+            #  "Difference": f"{(overlap2['success_ratio'] - overlap1['success_ratio']) * 100:.1f}%" if overlapping_tests > 0 else "N/A"},
+        ]
+
+    def compute_tabular_diff(
+        self, do_display: bool = True, verbose: bool = True
+    ) -> List[Dict[str, Any]]:
+        """Display the differences in a tabular format with one row per dataset."""
+        # Get filtered data using the caching mechanism
+        filtered_data = self._filter_joined_data()
+        joined_data = filtered_data["all"]
+        in_both = filtered_data["in_both"]
+
+        # Collect all possible column names (excluding dataset_name, status, and non-diff columns)
+        # Only include columns with numeric values
+        all_columns = set()
+        for row in joined_data:
+            for key in row.keys():
+                if key.endswith("_diff") and not key.endswith("_percent_diff"):
+                    # Extract the base column name without the _diff suffix
+                    base_column = key[
+                        :-5
+                    ]  # Still need this for backward compatibility
+                    # Check if the diff value is numeric
+                    if isinstance(row[key], (int, float)):
+                        all_columns.add(base_column)
+
+        # Define preferred column order based on main.py
+        preferred_columns = [
+            "total_steps",
+            "llm_step",
+            "total_tool_calls",
+            "tool_call_precision",
+            "tool_call_recall",
+            "agent_routing_accuracy",
+            "text_match",
+            "summary_matched_count",
+            "is_success",
+            "avg_resp_time",
+        ]
+
+        # Sort columns with preferred columns first, then alphabetically for the rest
+        sorted_columns = [
+            col for col in preferred_columns if col in all_columns
+        ]
+        sorted_columns += sorted(
+            [col for col in all_columns if col not in preferred_columns]
+        )
+
+        # Prepare data for table formatting
+        table_rows = []
+
+        # Add data rows - ONLY for datasets that are in both files
+        for row in in_both:
+            dataset_name = row["dataset_name"]
+            table_row = {"Dataset": dataset_name}
+
+            for col in sorted_columns:
+                diff_key = get_diff_column(col)
+                if diff_key in row:
+                    value = row[diff_key]
+                    # Format the value based on its type
+                    if isinstance(value, float):
+                        table_row[col] = round(value, 1)
+                    else:
+                        table_row[col] = value
+                else:
+                    # If the column doesn't exist for this dataset
+                    table_row[col] = "N/A"
+
+            table_rows.append(table_row)
+
+        # Calculate average values for experiment, reference, and delta
+        if table_rows:
+            # Calculate average differences (as before)
+            summary_row = {"Dataset": "Average Difference"}
+            for col in sorted_columns:
+                values = []
+                for row in table_rows:
+                    val = row.get(col)
+                    if isinstance(val, (int, float)):
+                        values.append(val)
+
+                if values:
+                    avg_value = sum(values) / len(values)
+                    summary_row[col] = str(round(avg_value, 1))
+                else:
+                    summary_row[col] = "N/A"
+
+            table_rows.append(summary_row)
+
+            # Calculate average experiment and reference values
+            experiment_avgs = {}
+            reference_avgs = {}
+
+            for col in sorted_columns:
+                exp_values = []
+                ref_values = []
+
+                for row in in_both:
+                    exp_key = get_experiment_column(col)
+                    ref_key = get_reference_column(col)
+
+                    if exp_key in row and isinstance(
+                        row[exp_key], (int, float)
+                    ):
+                        exp_values.append(row[exp_key])
+
+                    if ref_key in row and isinstance(
+                        row[ref_key], (int, float)
+                    ):
+                        ref_values.append(row[ref_key])
+
+                # Calculate averages
+                if exp_values:
+                    experiment_avgs[col] = round(
+                        sum(exp_values) / len(exp_values), 1
+                    )
+                else:
+                    experiment_avgs[col] = "N/A"
+
+                if ref_values:
+                    reference_avgs[col] = round(
+                        sum(ref_values) / len(ref_values), 1
+                    )
+                else:
+                    reference_avgs[col] = "N/A"
+
+            # Create the new table format with experiment, reference, and delta values
+            avg_table_rows = []
+            for col in sorted_columns:
+                delta_value = summary_row.get(col, "N/A")
+                avg_table_rows.append(
+                    {
+                        "Metric": col,
+                        "Reference": reference_avgs.get(col, "N/A"),
+                        "Experiment": experiment_avgs.get(col, "N/A"),
+                        "Experiment - Reference": delta_value,
+                    }
+                )
+
+            # Create and print the new average values table
+            if do_display:
+                print("\n")
+                avg_table = create_table(avg_table_rows, title="Average Values")
+                avg_table.print()
+                print("\n")
+
+        if verbose:
+            table = create_table(
+                table_rows,
+                title="DIFFERENCES TABLE (Experiment value - Reference value)",
+            )
+            table.print()
+
+        return table_rows
+
+    def _write_diff_table_to_csv(
+        self,
+        table_rows: List[Dict[str, Any]],
+        output_path: str,
+        summary_matched_count1: int,
+        summary_matched_count2: int,
+    ) -> None:
+        """Write the diff table to a CSV file."""
+        if not table_rows:
+            print("No data to write to CSV.")
+            return
+
+        # Ensure we have all column names
+        fieldnames = ["Dataset"]
+        for row in table_rows:
+            for key in row.keys():
+                if key != "Dataset" and key not in fieldnames:
+                    fieldnames.append(key)
+
+        try:
+            # Create a new fieldnames list with "_delta" appended to each column except "Dataset"
+            delta_fieldnames = ["Dataset"]
+            for field in fieldnames:
+                if field != "Dataset":
+                    delta_fieldnames.append(f"{field}_delta")
+
+            with open(output_path, "w", newline="") as csvfile:
+                writer = csv.DictWriter(csvfile, fieldnames=delta_fieldnames)
+                writer.writeheader()
+
+                # Write all rows except the summary row, with renamed columns
+                for row in table_rows:
+                    if row["Dataset"] != "Average Difference":
+                        # Create a new row with renamed columns
+                        new_row = {"Dataset": row["Dataset"]}
+                        for key, value in row.items():
+                            if key != "Dataset":
+                                new_row[f"{key}_delta"] = value
+                        writer.writerow(new_row)
+
+                # Add the summary row with the summary matched counts
+                summary_row = next(
+                    (
+                        row
+                        for row in table_rows
+                        if row["Dataset"] == "Average Difference"
+                    ),
+                    {},
+                )
+                if summary_row:
+                    # Create a new summary row with renamed columns
+                    new_summary_row = {
+                        "Dataset": summary_row.get(
+                            "Dataset", "Average Difference"
+                        )
+                    }
+                    for key, value in summary_row.items():
+                        if key != "Dataset":
+                            new_summary_row[f"{key}_delta"] = value
+
+                    writer.writerow(new_summary_row)
+
+        except Exception as e:
+            print(f"Error writing to CSV: {e}")
+
+    def to_csv(self, path: str) -> None:
+        """Write the diff results to a CSV file."""
+        table_rows = self.compute_tabular_diff(do_display=False, verbose=False)
+
+        self._write_diff_table_to_csv(
+            table_rows,
+            path,
+            self.result1.summary_matched_count,
+            self.result2.summary_matched_count,
+        )
+
+    def display_exclusive_tests(self) -> Tuple[List[str], List[str]]:
+        """Display lists of tests that are exclusive to each file."""
+        joined_data = self.join_datasets()
+
+        # Filter tests only in reference file
+        only_in_reference = [
+            row["dataset_name"]
+            for row in joined_data
+            if row["status"] == STATUS_ONLY_IN_REFERENCE
+        ]
+
+        # Filter tests only in experiment file
+        only_in_experiment = [
+            row["dataset_name"]
+            for row in joined_data
+            if row["status"] == STATUS_ONLY_IN_EXPERIMENT
+        ]
+
+        # Display the results
+        print(f"\nTests {STATUS_ONLY_IN_REFERENCE}:")
+        print(sorted(only_in_reference), "\n")
+
+        print(f"\nTests {STATUS_ONLY_IN_EXPERIMENT}:")
+        print(sorted(only_in_experiment))
+
+        return only_in_reference, only_in_experiment
+
+    def display_differing_summary_matches(self) -> List[Dict[str, str]]:
+        """Display test cases where summary match status differs between reference and experiment."""
+        # Get filtered data using the caching mechanism
+        filtered_data = self._filter_joined_data()
+        in_both = filtered_data["in_both"]
+
+        # Find test cases where summary match status differs
+        differing_cases = []
+        for row in in_both:
+            # Initialize result row with dataset name
+            result_row = {"Dataset": row["dataset_name"]}
+
+            # Check if text_match is available in both files
+            if has_column_in_both(row, "text_match"):
+                ref_match = (
+                    get_column_value(row, "text_match", "reference")
+                    == "Summary Matched"
+                )
+                exp_match = (
+                    get_column_value(row, "text_match", "experiment")
+                    == "Summary Matched"
+                )
+
+                # If the match status differs, add to our result
+                if ref_match != exp_match:
+                    result_row["Reference Summary Match"] = (
+                        "Yes" if ref_match else "No"
+                    )
+                    result_row["Experiment Summary Match"] = (
+                        "Yes" if exp_match else "No"
+                    )
+                    differing_cases.append(result_row)
+
+        # Display the results
+        title = "Differing Summary Matches"
+
+        if differing_cases:
+            print("\nTest cases with differing summary match status:")
+            table = create_table(differing_cases, title=title)
+            table.print()
+        else:
+            print("\nNo test cases with differing summary match status found.")
+
+        return differing_cases