ibm-watsonx-orchestrate-evaluation-framework 1.1.5__py3-none-any.whl → 1.1.6__py3-none-any.whl

wxo_agentic_evaluation/analyze_run.py

@@ -1,153 +1,481 @@
-import csv
 import json
 import os
 import re
+import time
+import traceback
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from threading import Lock
 from pathlib import Path
-from typing import Dict, List, Optional, Set
+from typing import Dict, List, Optional, Set, Tuple
 
+import rich
 from jsonargparse import CLI
-from rich.console import Group
+from rich import box
+from rich.rule import Rule
+from rich.console import Group, Console
 from rich.panel import Panel
+from rich.progress import Progress
 from rich.style import Style
 from rich.table import Table
 from rich.text import Text
 
-from wxo_agentic_evaluation.arg_configs import AnalyzeConfig
+from wxo_agentic_evaluation.arg_configs import AnalyzeConfig, AnalyzeMode
 from wxo_agentic_evaluation.description_quality_checker import (
     DescriptionQualityInspector,
 )
 from wxo_agentic_evaluation.metrics.metrics import (
+    EnhancedAnalyzeMetrics,
     TextMatchType,
     ToolCallAndRoutingMetrics,
+    DescriptionQualityMetric,
+    DescriptionQuality,
 )
+from wxo_agentic_evaluation.referenceless_eval import ReferencelessEvaluation
 from wxo_agentic_evaluation.type import (
     ContentType,
     ExtendedMessage,
     ToolDefinition,
 )
-from wxo_agentic_evaluation.utils.rich_utils import (
-    IncorrectParameterUtils,
-    is_ok,
-    pretty_print,
-    print_done,
-    warn,
-)
-from wxo_agentic_evaluation.utils.utils import (
+from wxo_agentic_evaluation.utils import (
+    ReferencelessEvalParser,
+    TestCaseResources,
+    ToolExtractionOpenAIFormat,
     add_line_seperator,
     list_run_files,
     load_run_metrics,
+    N_A,
+)
+
+MODEL_ID = "meta-llama/llama-3-405b-instruct"
+GATE_TOOL_ENRICHMENTS = (
+    os.getenv("GATE_TOOL_ENRICHMENTS", "true").lower().strip() == "true"
 )
+LOCK = Lock()
+
+
+class AnalyzerBase(ABC):
+    @abstractmethod
+    def analyze(self, config: AnalyzeConfig):
+        pass
+
+    @abstractmethod
+    def render(self):
+        pass
+
+    def _is_failed_tool_call(self, message: ExtendedMessage):
+        if message.reason and message.message.type == ContentType.tool_call:
+            if (
+                reason := message.reason.get("reason")
+            ) and reason != "irrelevant tool call":
+                return True
+
+    def _single_run(
+        self, test_case_name, run_map, test_cases_resource: TestCaseResources
+    ):
+        if not run_map:
+            # Legacy single-run files
+            test_messages, meta = test_cases_resource.get_analyze_messages(
+                test_case_name=test_case_name
+            )
+            metrics: ToolCallAndRoutingMetrics = (
+                test_cases_resource.get_test_metrics(
+                    test_case_name=test_case_name
+                )
+            )
+        else:
+            run_id = next(iter(run_map))
+            paths = run_map[run_id]
+            metrics = test_cases_resource.get_test_metrics(
+                path=paths["metrics"]
+            )
+            test_messages, meta = test_cases_resource.get_analyze_messages(
+                path=paths["analyze"]
+            )
+
+        # --- compute status uniformly (legacy & run1) ---
+        runs_problematic = self._is_failed_test_case(metrics)
+
+        return test_messages, meta, metrics, runs_problematic
+
+    def _is_failed_test_case(self, data) -> bool:
+        """
+        True -> test case failed
+        False -> test success
+        """
+
+        # not ideal if statement
+        # in the future, refactor so this if statement is not needed
+        # this if statement is needed because this function is called in two cases:
+        # 1. if data is an instance ToolCallAndRoutingMetrics
+        # 2. if data is a row in the summary table (dictionary)
+
+        # ideal the SummaryMetrics should be parsed into pydantic class as well
+
+        if isinstance(data, ToolCallAndRoutingMetrics):
+            is_success = data.is_success
+            had_incorrect_param = data.tool_calls_with_incorrect_parameter > 0
+            low_precision = float(data.tool_call_precision) < 1.0
+            low_recall = float(data.tool_call_recall) < 1.0
+        else:
+            is_success = str(data["is_success"]).strip().lower() == "true"
+            had_incorrect_param = (
+                float(data.get("tool_calls_with_incorrect_parameter", 0) or 0)
+                > 0
+            )
+            low_precision = float(data.get("tool_call_precision", 1) or 1) < 1.0
+            low_recall = float(data.get("tool_call_recall", 1) or 1) < 1.0
+
+        return (
+            not is_success or had_incorrect_param or low_precision or low_recall
+        )
+
+    def _get_test_case_with_failed_tools(self, summary) -> List[str]:
+        test_case_with_failed_tools = []
+
+        for entry in summary:
+            test_case_name = entry["dataset_name"]
+
+            if test_case_name.lower().strip() == "summary (average)":
+                continue
+
+            if self._is_failed_test_case(entry):
+                test_case_with_failed_tools.append(entry)
+
+        return test_case_with_failed_tools
 
 
-class Analyzer:
+class DescriptionQualityAnalyzer(AnalyzerBase):
     def __init__(self):
-        self.analysis_cache: Dict[str, List[Text]] = (
-            {}
-        )  # the failing tools cached here won't be re-analyzed.
+        self.analysis_cache: Dict[str, DescriptionQualityMetric] = {}
         # tool_name -> description analysis
+        self.missing_tools = set()
+        self.tools_not_found = set()
 
-    @staticmethod
-    def _generate_style_config():
-        return Style(
-            color="magenta",
-            blink=True,
-            bold=True,
-        )
+    def _get_tools_not_found_in_source(
+        self,
+        tools_to_analyze: List[str],
+        failing_tool_definitions: List[ToolDefinition],
+    ) -> Set[str]:
+
+        return set(tools_to_analyze) - {
+            tool_def.tool_name for tool_def in failing_tool_definitions
+        }
 
-    def _split_cache(
-        self, failing_tools: Set[str]
-    ) -> tuple[List[str], List[Text]]:
+    def _failing_tool_from_messages(self, messages: List[ExtendedMessage]):
+        failed_tool_calls = set()
+        for message in messages:
+            if self._is_failed_tool_call(message):
+                content = json.loads(message.message.content)
+                tool_call_name = content["name"]
+                failed_tool_calls.add(tool_call_name)
+
+        return failed_tool_calls
+
+    def failing_tools(self, data_path):
+        messages_dir = os.path.join(data_path, "messages")
+        test_case_resources = TestCaseResources(data_path)
+        processed_test_cases = set()
+        failed_tool_calls = set()
+
+        for test_case in test_case_resources.get_summary:
+            dataset_name = test_case["dataset_name"]
+            if dataset_name in processed_test_cases:
+                continue
+            processed_test_cases.add(dataset_name)
 
-        tools_to_analyze: List[str] = []
-        cached_lines: List[Text] = []
-        tools_analyzed: List[str] = []
+            run_map = list_run_files(messages_dir, test_case["dataset_name"])
 
-        for tool_name in sorted(failing_tools):
-            cached_analysis = self.analysis_cache.get(tool_name)
-            if cached_analysis:
-                cached_lines.extend(cached_analysis)
-                tools_analyzed.append(tool_name)
-            else:
-                tools_to_analyze.append(tool_name)
+            if not run_map:
+                test_messages, _ = test_case_resources.get_analyze_messages(
+                    test_case_name=dataset_name
+                )
+                failed_tool_calls.update(
+                    self._failing_tool_from_messages(test_messages)
+                )
 
-        if tools_analyzed:
-            pretty_print(
-                content=f"ℹ️ Loading cached analysis since these failing tools: {tools_analyzed} have been analyzed previously.",
-                style="bold cyan",
-            )
+            else:
+                for paths in run_map.values():
+                    test_messages, _ = test_case_resources.get_analyze_messages(
+                        path=paths["analyze"]
+                    )
+                    failed_tool_calls.update(
+                        self._failing_tool_from_messages(test_messages)
+                    )
 
-        return (tools_to_analyze, cached_lines)
+        return failed_tool_calls
 
     def analyze_failing_tool_description_quality(
         self,
         inspector: DescriptionQualityInspector,
         tool_definition_path: str,
         failing_tools: Set[str],
-    ) -> List[Text]:
+    ) -> Tuple[List[DescriptionQualityMetric], List[str]]:
         """
         :param tool_definition_path: Path to the tool definition file.
         :param failing_tools: Set of tool names that failed.
-        :return: List of rich `Text` objects containing feedback for the customer.
+        :return: A tuple where the first item in the tuple is List[DescriptionQualityMetric] for failed tools that were analyzed,
+        the second item in the list is a list of missing tools
         """
 
-        pretty_print(
-            content=f"⚙️ Checking tool description quality for failing tools: {sorted(failing_tools)}",
-            style="bold cyan",
+        failing_tool_definitions: List[ToolDefinition] = (
+            inspector.extract_tool_desc_from_tool_source(
+                Path(tool_definition_path),
+                failing_tools,
+            )
+        )
+
+        if not failing_tool_definitions:
+            """
+            No tool definitions(with '@tool' decorators) for failed tools: '{tools_to_analyze}' found in the file: '{tool_definition_path}'"
+            """
+            with Lock:
+                self.tools_not_found.add(failing_tools)
+
+        missing_tools = self._get_tools_not_found_in_source(
+            failing_tools, failing_tool_definitions
+        )
+        for tool_definition in failing_tool_definitions:
+            tool_analysis = inspector.detect_bad_description(tool_definition)
+            with LOCK:
+                self.analysis_cache[tool_definition.tool_name] = tool_analysis
+                self.missing_tools.update(missing_tools)
+
+        return 1
+
+    def analyze(self, config):
+        failing_tools = self.failing_tools(config.data_path)
+        inspector = DescriptionQualityInspector()
+        tool_definition_path = config.tool_definition_path
+
+        with ThreadPoolExecutor(
+            max_workers=config.num_workers, thread_name_prefix="[Worker]"
+        ) as pool:
+            futures = [
+                pool.submit(
+                    self.analyze_failing_tool_description_quality,
+                    inspector,
+                    tool_definition_path,
+                    [failing_tool],
+                )
+                for failing_tool in failing_tools
+            ]
+
+            if futures:
+                with Progress() as progress:
+                    task = progress.add_task(
+                        f"[purple]Analyzing description quality for {len(futures)} tasks...",
+                        total=len(futures),
+                    )
+                    for future in as_completed(futures):
+                        try:
+                            future.result()
+                        except Exception:
+                            traceback.print_exc()
+                        finally:
+                            progress.update(task, advance=1)
+
+    def render(self):
+        raise NotImplementedError("Not implemented")
+
+
+class Analyzer(AnalyzerBase):
+    def __init__(
+        self,
+        enhanced_metrics: Optional[List[EnhancedAnalyzeMetrics]] = None,
+        description_quality_analyzer: DescriptionQualityAnalyzer = None,
+    ):
+        self.enhanced_metrics = enhanced_metrics
+        self.enhanced_metrics_idx_map = {}
+
+        if self.enhanced_metrics:
+            # do some post-processing on the enhanced metrics
+            # create a mapping between test case name and index
+            if self.enhanced_metrics:
+                for idx, metric in enumerate(self.enhanced_metrics):
+                    self.enhanced_metrics_idx_map[metric.test_case_name] = idx
+
+        self.description_quality_analyzer = description_quality_analyzer
+
+    @staticmethod
+    def _generate_style_config():
+        return Style(
+            color="magenta",
+            blink=True,
+            bold=True,
         )
 
-        analysis_for_display: List[Text] = []
+    def _parse_enhanced_metrics(self, test_case_name) -> Optional[Table]:
+        table = Table(
+            box=box.ROUNDED,
+            show_lines=True,
+        )
 
-        # Step 1: get tools not yet analyzed and cached analysis for tools analyzed previously
-        tools_to_analyze, cached_analysis = self._split_cache(failing_tools)
-        if cached_analysis:
-            analysis_for_display.extend(cached_analysis)
+        columns = [
+            "Tool Name",
+            "Root Cause Analysis",
+            "Docstring Recommendations",
+        ]
 
-        # Step 2: analyze cache misses
-        if tools_to_analyze:
+        rows = []
+
+        if (
+            self.enhanced_metrics
+            and (index := self.enhanced_metrics_idx_map.get(test_case_name))
+            is not None
+        ):
+            enhanced_metric: EnhancedAnalyzeMetrics = self.enhanced_metrics[
+                index
+            ]
+
+            for idx, tool_call in enumerate(enhanced_metric.tool_names):
+                static_root_causes = []
+                parsed_tool_annotations = []
+                param_annotations = defaultdict(list)
+
+                row = [tool_call]
+
+                # if this is true, then there are no semantic metrics
+                static_root_causes = [
+                    Text(item.explanation)
+                    for item in enhanced_metric.static_metrics[idx]
+                ]
+
+                static_root_causes = Text().join(static_root_causes)
+
+                # Parameter Root Cause
+                parameter_annotations = enhanced_metric.parameter_annotations[
+                    idx
+                ]
+                formatted_param_root_cause = [
+                    Text(metric.explanation) for metric in parameter_annotations
+                ]
+                formatted_param_root_cause = Text().join(
+                    formatted_param_root_cause
+                )
 
-            failing_tool_definitions: List[ToolDefinition] = (
-                inspector.extract_tool_desc_from_tool_source(
-                    Path(tool_definition_path),
-                    tools_to_analyze,
+                # Tool Root Cause
+                tool_annotations = enhanced_metric.tool_annotations[idx]
+                formatted_tool_root_cause = [
+                    Text(metric.explanation) for metric in tool_annotations
+                ]
+                formatted_tool_root_cause = Text().join(
+                    formatted_tool_root_cause
                 )
-            )
 
-            if not failing_tool_definitions:
-                analysis_for_display.append(
-                    warn(
-                        message=f"No tool definitions(with '@tool' decorators) for failed tools: '{tools_to_analyze}' found in the file: '{tool_definition_path}'"
+                if formatted_param_root_cause or formatted_tool_root_cause:
+                    root_cause = (
+                        formatted_tool_root_cause
+                        if len(formatted_tool_root_cause)
+                        > len(formatted_param_root_cause)
+                        else formatted_param_root_cause
                     )
+                elif static_root_causes:
+                    root_cause = static_root_causes
+                else:
+                    root_cause = N_A
+
+                row.append(root_cause)
+
+                # Parameter Level Docstring
+                for metric in parameter_annotations:
+                    if annotations := metric.annotations:
+                        for annotation in annotations:
+                            param_annotations[annotation.parameter_name].append(
+                                f"[b][i][cyan]{annotation.quote}[/b][/i][/cyan]"
+                            )
+
+                newline = "\n"
+                param_annotations = [
+                    f"- [b]{param_name}:[/b] {newline.join(doc_string)}"
+                    for param_name, doc_string in param_annotations.items()
+                ]
+                param_annotations = "\n".join(param_annotations)
+
+                # Tool Level Docstring
+                for metric in tool_annotations:
+                    if annotations := metric.annotations:
+                        for annotation in annotations:
+                            parsed_tool_annotations.append(
+                                f"[b][i][cyan]{annotation.quote}[/b][/i][/cyan]"
+                            )
+                parsed_tool_annotations = "\n".join(parsed_tool_annotations)
+                docstring_cell = Table(
+                    show_lines=False, show_header=False, box=None
                 )
-                return analysis_for_display
+                add_divider = False
+
+                # - Gate the Doc String Enrichments.
+                # - Ensure the environment variable is enabled.
+                if GATE_TOOL_ENRICHMENTS and self.description_quality_analyzer:
+                    # check if tool in cache
+                    tool_description_analysis = (
+                        self.description_quality_analyzer.analysis_cache.get(
+                            tool_call
+                        )
+                    )
+                    is_missing_tool = (
+                        tool_call
+                        in self.description_quality_analyzer.missing_tools
+                    )  # tool call not in tool_definition_path
+                    # failed tool call that failed to get extracted from the tool_definition_path because of missing `@tool` decorator
+                    # TODO: figure out if this edge is needed? taken from original Analyze implementation
+                    tool_not_found = (
+                        tool_call
+                        in self.description_quality_analyzer.tools_not_found
+                    )
 
-            missing_tools = self._get_tools_not_found_in_source(
-                tools_to_analyze, failing_tool_definitions
-            )
-            if missing_tools:
-                analysis_for_display.append(
-                    warn(
-                        message=f"Missing tool definitions for failed tools: '{missing_tools}' in the file: '{tool_definition_path}'"
+                    # If the tool_call is in `missing_tools`, don't show the annotations
+                    if is_missing_tool or tool_not_found:
+                        parsed_tool_annotations = []
+                        param_annotations = []
+
+                    if tool_description_analysis is not None:
+                        if (
+                            tool_description_analysis.description_quality
+                            == DescriptionQuality.GOOD
+                        ):
+                            parsed_tool_annotations = []
+                            param_annotations = []
+                    else:
+                        print("cache miss: ", tool_call)
+
+                if not parsed_tool_annotations and not param_annotations:
+                    docstring_cell.add_row(N_A)
+                if parsed_tool_annotations:
+                    docstring_cell.add_row(
+                        "[b]Tool Docstrings", parsed_tool_annotations
+                    )
+                    add_divider = True
+                if param_annotations:
+                    if add_divider:
+                        docstring_cell.add_row(Rule(characters="--"))
+                    docstring_cell.add_row(
+                        "[b]Parameter Docstrings", param_annotations
                     )
-                )
 
-            for tool_definition in failing_tool_definitions:
+                row.append(docstring_cell)
+                rows.append(row)
 
-                tool_analysis = self._analyze_tool_definition(
-                    inspector=inspector,
-                    tool_definition=tool_definition,
-                    tool_definition_path=tool_definition_path,
-                )
+        is_empty = not any(rows)
+        if is_empty:
+            return None
 
-                self.analysis_cache[tool_definition.tool_name] = tool_analysis
-                analysis_for_display.extend(tool_analysis)
+        for idx, column in enumerate(columns):
+            table.add_column(column)
+
+        for row in rows:
+            table.add_row(*row)
 
-        return analysis_for_display
+        return table
 
     def render(
         self,
         data: List[ExtendedMessage],
         tool_definition_path: Optional[str],
         meta: Optional[dict] = None,
+        test_case_name=None,
     ) -> Group:
         """
         Render the conversation history and analysis results.
@@ -158,7 +486,6 @@ class Analyzer:
         conversation_lines = []
         reason_lines = []
         failing_tools = []
-        added_errors_header = False
         added_missed_header = False
 
         for entry in data:
@@ -192,31 +519,11 @@ class Analyzer:
 
             text_line = Text(f"{label}: {content}\n")
             if reason:
-                if not added_errors_header:
-                    reason_lines.append(
-                        Text("\nTool Call Errors:\n", style="bold red")
-                    )
-                    added_errors_header = True
                 text_line.stylize("bold red")
                 reason_text = f"❌ {tool_name}: {json.dumps(reason)}\n\n"
                 reason_lines.append(Text(reason_text, style="red"))
             conversation_lines.append(text_line)
 
-        if failing_tools and tool_definition_path:
-
-            inspector = DescriptionQualityInspector()
-
-            description_quality_inspection_lines = (
-                self.analyze_failing_tool_description_quality(
-                    inspector, tool_definition_path, set(failing_tools)
-                )
-            )
-
-            print_done()
-
-            if description_quality_inspection_lines:
-                reason_lines.extend(description_quality_inspection_lines)
-
         if meta:
             missed = meta.get("missed_tool_calls") or []
             if missed:
@@ -231,18 +538,21 @@ class Analyzer:
         conversation_panel = Panel(
             Text().join(conversation_lines),
             title="Conversation History",
-            border_style="blue",
+            border_style="bold deep_sky_blue2",
         )
         reason_panel = Panel(
             Text().join(reason_lines),
-            title="Analysis Results",
-            border_style="red",
+            box=box.ROUNDED,
+            title=f"[bold red]Tool Call Errors[/bold red]",
+            border_style="bold red",
         )
+        table = self._parse_enhanced_metrics(test_case_name=test_case_name)
+        if table:
+            group = Group(conversation_panel, reason_panel, table)
+        else:
+            group = Group(conversation_panel, reason_panel)
 
-        return Group(
-            conversation_panel,
-            reason_panel,
-        )
+        return group
 
     def analyze(self, config: AnalyzeConfig):
         """
@@ -250,59 +560,15 @@ class Analyzer:
         :param config: AnalyzeConfig object containing user provided paths for analysis.
         """
 
-        def get_summary(summary_file_name: str = "summary_metrics.csv"):
-            summary = []
-
-            path_to_summary_file = os.path.join(
-                config.data_path, summary_file_name
-            )
-
-            with open(path_to_summary_file, "r") as f:
-                reader = csv.reader(f)
-                header = next(reader)
-                for row in reader:
-                    summary.append(dict(zip(header, row)))
-
-            return summary
-
-        def get_test_messages(test_case_name):
-            test_messages = []
-            meta = {}
-
-            test_case_path = os.path.join(
-                config.data_path,
-                "messages",
-                f"{test_case_name}.messages.analyze.json",
-            )
-
-            with open(test_case_path, "r", encoding="utf-8") as f:
-                temp = json.load(f)
-                if temp and isinstance(temp[-1], dict) and "meta" in temp[-1]:
-                    meta = temp[-1]["meta"]
-                    temp = temp[:-1]
-
-                for entry in temp:
-                    msg = ExtendedMessage(**entry)
-                    test_messages.append(msg)
-
-            return test_messages, meta
-
-        def get_metrics(test_case_name):
-            test_metrics_path = os.path.join(
-                config.data_path, "messages", f"{test_case_name}.metrics.json"
-            )
-
-            with open(test_metrics_path, "r", encoding="utf-8") as f:
-                metrics = ToolCallAndRoutingMetrics(**json.load(f))
-
-            return metrics
-
-        summary = get_summary()
+        test_case_resources = TestCaseResources(config.data_path)
+        summary = test_case_resources.get_summary
 
         test_case_with_failed_tools = self._get_test_case_with_failed_tools(
             summary=summary
         )
 
+        output_panels = []
+
         if len(test_case_with_failed_tools) == 0:
             header_table = Table(show_header=False, box=None)
 
@@ -313,7 +579,7 @@ class Analyzer:
                 title="[bold green]📋 Analysis Summary[/bold green]",
             )
 
-            pretty_print(panel)
+            output_panels.append(panel)
 
         messages_dir = os.path.join(config.data_path, "messages")
 
@@ -340,100 +606,17 @@ class Analyzer:
             if dataset_base in processed_parents:
                 continue
 
-            run_map = list_run_files(messages_dir, dataset_base)
+            run_map = list_run_files(messages_dir, dataset_base, config.run)
 
             # ---- SINGLE RUN (legacy or run1 only) ----
             if not run_map or len(run_map) == 1:
-                if not run_map:
-                    # Legacy single-run files
-                    test_messages, meta = get_test_messages(
-                        test_case_name=dataset_base
+                runs_performed = 1
+                test_messages, meta, metrics, runs_problematic = (
+                    self._single_run(
+                        test_case_name=dataset_base,
+                        run_map=run_map,
+                        test_cases_resource=test_case_resources,
                     )
-                    metrics: ToolCallAndRoutingMetrics = get_metrics(
-                        test_case_name=dataset_base
-                    )
-                    runs_performed = 1
-                else:
-                    run_id = next(iter(run_map))
-                    paths = run_map[run_id]
-                    runs_performed = 1
-                    if not paths["metrics"]:
-                        pretty_print(
-                            f"❌ {dataset_base}.run{run_id} — metrics file missing.",
-                            style="bold red",
-                        )
-                        # Count it as analyzed & problematic
-                        processed_parents.add(dataset_base)
-                        ds_table = Table(show_header=False, box=None)
-                        ds_table.add_row("Type: Single-run")
-                        ds_table.add_row("Status: ❌ Problematic")
-                        pretty_print(
-                            Panel(
-                                ds_table,
-                                title=f"📋 Analysis Summary — {dataset_base}",
-                                border_style="green",
-                            )
-                        )
-                        overall_runs_performed += 1
-                        overall_runs_problematic += 1
-                        add_line_seperator(self._generate_style_config())
-                        continue
-
-                    metrics = load_run_metrics(paths["metrics"])
-                    meta = {}
-
-                    if paths["analyze"]:
-                        with open(paths["analyze"], "r", encoding="utf-8") as f:
-                            raw = json.load(f)
-                        if (
-                            raw
-                            and isinstance(raw[-1], dict)
-                            and "meta" in raw[-1]
-                        ):
-                            meta = raw[-1]["meta"]
-                            raw = raw[:-1]
-                        test_messages = [
-                            ExtendedMessage(**entry) for entry in raw
-                        ]
-                    else:
-                        test_messages, meta = [], {}
-
-                # --- compute status uniformly (legacy & run1) ---
-                had_incorrect_param = (
-                    hasattr(metrics, "tool_calls_with_incorrect_parameter")
-                    and float(metrics.tool_calls_with_incorrect_parameter or 0)
-                    > 0
-                )
-                low_precision = (
-                    hasattr(metrics, "tool_call_precision")
-                    and float(
-                        metrics.tool_call_precision
-                        if metrics.tool_call_precision is not None
-                        else 1.0
-                    )
-                    < 1.0
-                )
-                low_recall = (
-                    hasattr(metrics, "tool_call_recall")
-                    and float(
-                        metrics.tool_call_recall
-                        if metrics.tool_call_recall is not None
-                        else 1.0
-                    )
-                    < 1.0
-                )
-                runs_problematic = (
-                    1
-                    if (
-                        (
-                            hasattr(metrics, "is_success")
-                            and not metrics.is_success
-                        )
-                        or had_incorrect_param
-                        or low_precision
-                        or low_recall
-                    )
-                    else 0
                 )
 
                 processed_parents.add(dataset_base)
@@ -445,14 +628,6 @@ class Analyzer:
                     "❌ Problematic" if runs_problematic else "✅ No problems"
                 )
                 ds_table.add_row(f"Status: {status}")
-                pretty_print(
-                    Panel(
-                        ds_table,
-                        title=f"📋 Analysis Summary — {dataset_base}",
-                        border_style="green",
-                    )
-                )
-
                 # Update overall counters/averages
                 overall_runs_performed += runs_performed
                 overall_runs_problematic += runs_problematic
@@ -469,19 +644,43 @@ class Analyzer:
                         1 if bool(metrics.is_success) else 0
                     )
 
-                # Replay details only if problematic
-                if runs_problematic:
-                    pretty_print(
+                header_group = Group(
+                    *[
+                        ds_table,
                         self._create_header_analysis_panel(
                             dataset_base, metrics
+                        ),
+                    ],
+                )
+                border_style = "bold red" if runs_problematic else "bold green"
+                header_panel = Panel(
+                    header_group,
+                    title=f"[b]📋 Analysis Summary — {dataset_base}[/b]",
+                    border_style=border_style,
+                )
+                output_panels.append(header_panel)
+
+                if runs_problematic:
+                    output_panels.append(
+                        self.render(
+                            test_messages,
+                            config.tool_definition_path,
+                            meta,
+                            test_case_name=dataset_base,
                         )
                     )
-                    pretty_print(
-                        self.render(
-                            test_messages, config.tool_definition_path, meta
+                    output_panels.append(
+                        add_line_seperator(
+                            self._generate_style_config(), print=False
+                        )
+                    )
+
+                else:
+                    output_panels.append(
+                        add_line_seperator(
+                            self._generate_style_config(), print=False
                         )
                     )
-                    add_line_seperator(self._generate_style_config())
 
                 continue
 
@@ -555,31 +754,32 @@ class Analyzer:
                         }
                     )
 
-            # Print the dataset panel FIRST with both lines inside
-            ds_table = Table(show_header=False, box=None)
-            ds_table.add_row(f"Type: Multi-run ({runs_performed} runs)")
-            ds_table.add_row(
-                f"Runs with problems: {runs_problematic} / {runs_performed}"
-            )
-            status = (
-                "❌ Problematic" if runs_problematic > 0 else "✅ No problems"
-            )
-            ds_table.add_row(f"Status: {status}")
-            pretty_print(
-                Panel(
-                    ds_table,
-                    title=f"📋 Analysis Summary — {dataset_base}",
-                    border_style="green",
-                )
-            )
-
             # Second pass: now replay only the problematic runs (so summary stays at the top)
             for item in deferred_runs:
-                pretty_print(
-                    self._create_header_analysis_panel(
-                        item["title"], item["metrics"]
+                ds_table = Table(show_header=False, box=None)
+                ds_table.add_row(f"Type: Multi-run ({runs_performed} runs)")
+                ds_table.add_row(
+                    f"Runs with problems: {runs_problematic} / {runs_performed}"
+                )
+                status = (
+                    "❌ Problematic"
+                    if runs_problematic > 0
+                    else "✅ No problems"
+                )
+                ds_table.add_row(f"Status: {status}")
+                header_table = self._create_header_analysis_panel(
+                    item["title"], item["metrics"]
+                )
+
+                group = Group(*[ds_table, header_table])
+                output_panels.append(
+                    Panel(
+                        group,
+                        title=f"📋 Analysis Summary — {dataset_base}",
+                        border_style="green",
                     )
                 )
+
                 if item["analyze_path"]:
                     with open(item["analyze_path"], "r", encoding="utf-8") as f:
                         raw = json.load(f)
@@ -589,12 +789,16 @@ class Analyzer:
                         raw = raw[:-1]
                     test_messages = [ExtendedMessage(**entry) for entry in raw]
 
-                    pretty_print(
+                    output_panels.append(
                         self.render(
                             test_messages, config.tool_definition_path, meta
                         )
                     )
-                add_line_seperator(self._generate_style_config())
+                output_panels.append(
+                    add_line_seperator(
+                        self._generate_style_config(), print=False
+                    )
+                )
 
             # Update overall aggregates
             overall_runs_performed += runs_performed
@@ -624,9 +828,9 @@ class Analyzer:
             )
             overall_lines.append(f"Avg journey success: {js_pct}%")
         else:
-            overall_lines.append("Avg journey success: N/A")
+            overall_lines.append(f"Avg journey success: N/A")
 
-        pretty_print(
+        output_panels.append(
             Panel(
                 Text("\n".join(overall_lines)),
                 title="📋 Overall Summary",
@@ -634,6 +838,11 @@ class Analyzer:
             )
         )
 
+        console = Console()
+        with console.pager(styles=True):
+            for panel in output_panels:
+                console.print(panel, overflow="crop")
+
     def _create_header_analysis_panel(
         self, test_case_name: str, metrics: ToolCallAndRoutingMetrics
     ) -> Panel:
@@ -649,86 +858,338 @@ class Analyzer:
         header_table.add_row(f"Text Match: {metrics.text_match.value}")
         header_table.add_row(f"Journey Success: {metrics.is_success}")
 
-        header_panel = Panel(
-            header_table, title="[bold green]Test Case Summary[/bold green]"
-        )
+        return header_table
 
-        return header_panel
 
-    def _get_test_case_with_failed_tools(self, summary) -> List:
+class AnalyzerEnhanced(AnalyzerBase):
+    PARAMETER_DOCUMENTATION = "PARAMETER_DOCUMENTATION"
+    TOOL_USAGE_EXAMPLES = "TOOL_USAGE_EXAMPLES"
+    TOOL_DOCUMENTATION = "TOOL_DOCUMENTATION"
 
-        test_case_with_failed_tools = []
+    DEFAULT_GENERATION_PARAMS = {
+        "min_new_tokens": 0,
+        "decoding_method": "greedy",
+        "max_new_tokens": 10_000,
+        "random_seed": 42,
+    }
 
-        for entry in summary:
-            test_case_name = entry["dataset_name"]
+    def __init__(self):
+        super().__init__()
 
-            if test_case_name.lower().strip() == "summary (average)":
+    def _deduplicate_tool_call_failures(self, messages: List[ExtendedMessage]):
+        """If there are multiple failures from the same tool, then choose the failure that occurs later in the trajectory
+
+        ex.
+        1. Tool A fails
+        2. Tool A Error response
+        3. Tool A call again which fails
+        4. Tool A error response
+
+        For the analysis, we analyze the second time the tool call fails, with the previous messages serving as context.
+
+        """
+        tool_indices = []
+        seen_tools = set()
+
+        for idx, message in enumerate(reversed(messages)):
+            if self._is_failed_tool_call(message):
+                content = json.loads(message.message.content)
+                tool_call_name = content["name"]
+                if tool_call_name not in seen_tools:
+                    seen_tools.add(tool_call_name)
+                    tool_indices.append(len(messages) - 1 - idx)
+
+        return sorted(tool_indices)
+
+    def process_messages(self, task_name, test_case, tools, messages):
+        eval = ReferencelessEvaluation(
+            api_spec=tools,
+            model_id=MODEL_ID,
+            task_n=task_name,
+            dataset_name=test_case,
+            runtime_pipeline=False,
+            generation_params=AnalyzerEnhanced.DEFAULT_GENERATION_PARAMS,
+        )
+
+        processed_data = [
+            {
+                k: msg.model_dump().get(k)
+                for k in ["role", "content", "type"]
+                if k in msg.model_dump()
+            }
+            for msg in messages
+        ]
+
+        context = processed_data[:-1]
+        tool_call = processed_data[
+            -1
+        ]  # assume that the message is the last tool call
+        tool_call_msg = json.loads(tool_call["content"])
+        call = ReferencelessEvaluation.fmt_tool_call(
+            tool_id=tool_call_msg.get("id", "1"),
+            tool_call_name=tool_call_msg["name"],
+            arguments=json.dumps(tool_call_msg["args"]),
+            context=context,
+        )
+        return test_case, eval.run([call])
+
+    def _extract_semantic_metrics(
+        self, metrics_dictionary, annotation_filters: Optional[List[str]]
+    ):
+        semantic_analysis = []
+        for metric_data in metrics_dictionary.values():
+            raw_response = metric_data.get("raw_response")
+            if raw_response is None:
                 continue
 
-            is_success = str(entry["is_success"]).strip().lower() == "true"
+            is_correct = metric_data.get("is_correct", False)
+            if is_correct:
+                continue
 
-            tip = float(
-                entry.get("tool_calls_with_incorrect_parameter", 0) or 0
+            failed_semantic_test_case = ReferencelessEvalParser.semantic_parser(
+                metric_name=metric_data.get("metric_name"),
+                data=raw_response,
+                annotation_filters=annotation_filters,
             )
-            tcp = float(entry.get("tool_call_precision", 1) or 1)
-            tcr = float(entry.get("tool_call_recall", 1) or 1)
 
-            # Apply the 4 checks
-            if (not is_success) or (tip > 0) or (tcp < 1.0) or (tcr < 1.0):
-                test_case_with_failed_tools.append(entry)
+            semantic_analysis.append(failed_semantic_test_case)
 
-        return test_case_with_failed_tools
+        return semantic_analysis
 
-    def _get_tools_not_found_in_source(
-        self,
-        tools_to_analyze: List[str],
-        failing_tool_definitions: List[ToolDefinition],
-    ) -> Set[str]:
+    def tool_enrichment_view(self, results):
+        enhanced_metrics = []
+        tool_enrichment_metrics = defaultdict(list)
+        for result in results:
+            for test_case, eval_results in result.items():
+                for result in eval_results:
+                    # for metric in result:
+                    failed_static_metrics = []
+                    parameter_annotations = []
+                    tool_annotations = []
 
-        return set(tools_to_analyze) - {
-            tool_def.tool_name for tool_def in failing_tool_definitions
-        }
+                    static_metrics_passed = result.get("static", {}).get(
+                        "final_decision", False
+                    )
+                    tool_call_obj = result.get("inputs", {}).get(
+                        "tool_call", {}
+                    )
 
-    def _analyze_tool_definition(
-        self,
-        inspector: DescriptionQualityInspector,
-        tool_definition: ToolDefinition,
-        tool_definition_path: str,
-    ) -> List[Text]:
+                    if static_metrics_passed:
+                        semantic_metrics = result.get("semantic")
+                        function_selection_metrics = semantic_metrics.get(
+                            "function_selection", {}
+                        ).get("metrics", {})
+                        tool_annotations = self._extract_semantic_metrics(
+                            function_selection_metrics,
+                            [
+                                AnalyzerEnhanced.TOOL_DOCUMENTATION,
+                                AnalyzerEnhanced.TOOL_USAGE_EXAMPLES,
+                            ],
+                        )
 
-        tool_name = tool_definition.tool_name
-        tool_desc = tool_definition.tool_description
+                        general_metrics = semantic_metrics.get(
+                            "general", {}
+                        ).get("metrics", {})
+                        parameter_annotations = self._extract_semantic_metrics(
+                            general_metrics,
+                            [AnalyzerEnhanced.PARAMETER_DOCUMENTATION],
+                        )
+                    else:
+                        static_metrics = result.get("static").get("metrics")
+                        failed_static_metrics = (
+                            ReferencelessEvalParser.static_parser(
+                                static_metrics=static_metrics
+                            )
+                        )
 
-        tool_analysis = []
+                    parsed_metrics = {
+                        "tool_name": tool_call_obj.get("function", {}).get(
+                            "name"
+                        ),
+                        "parameter_annotations": parameter_annotations,
+                        "tool_annotations": tool_annotations,
+                        "static_metrics": failed_static_metrics,
+                    }
+                    tool_enrichment_metrics[test_case].append(parsed_metrics)
+
+        for test_case, metrics in tool_enrichment_metrics.items():
+            failed_tools = [metric["tool_name"] for metric in metrics]
+            parameter_annotations = [
+                metric["parameter_annotations"] for metric in metrics
+            ]
+            tool_annotation = [metric["tool_annotations"] for metric in metrics]
+            static_metrics = [metric["static_metrics"] for metric in metrics]
+
+            # don't add to final metrics array if there were no annotations
+            if (
+                not any(parameter_annotations)
+                and not any(tool_annotation)
+                and not any(static_metrics)
+            ):
+                continue
 
-        # missing description
-        if tool_desc is None:
-            tool_analysis.extend(
-                IncorrectParameterUtils.format_missing_description_message(
-                    tool_name=tool_name,
-                    tool_definition_path=tool_definition_path,
+            enhanced_metrics.append(
+                EnhancedAnalyzeMetrics(
+                    test_case_name=test_case,
+                    parameter_annotations=parameter_annotations,
+                    tool_annotations=tool_annotation,
+                    tool_names=failed_tools,
+                    static_metrics=static_metrics,
                 )
             )
-            return tool_analysis
 
-        # bad description
-        if inspector.detect_bad_description(tool_definition):
-            tool_analysis.extend(
-                IncorrectParameterUtils.format_bad_description_message(
-                    tool_name=tool_name, tool_desc=tool_desc
-                )
+        return enhanced_metrics
+
+    def analyze(
+        self, config: AnalyzeConfig
+    ) -> Optional[List[EnhancedAnalyzeMetrics]]:
+        start = time.time()
+        all_tools = ToolExtractionOpenAIFormat.from_path(
+            config.tool_definition_path
+        )
+        messages_dir = os.path.join(config.data_path, "messages")
+        test_case_resources = TestCaseResources(config.data_path)
+
+        failed_test_cases = {}
+        for test_case in test_case_resources.get_summary:
+            if test_case["dataset_name"] in failed_test_cases:
+                continue
+            run_map = list_run_files(
+                messages_dir, test_case["dataset_name"], config.run
             )
-            return tool_analysis
+            if run_map and config.run == -1:
+                rich.print(
+                    "[red]Enhanced Mode only operates on a single run for a dataset. Since there are multiple runs, set the `--run` flag to the specific run for enhanced analysis."
+                )
+                # run the first run in the config map
+                rich.print(
+                    f"[b]Defaulting to run {next(iter(run_map))} to analyze for {test_case['dataset_name']}"
+                )
+                config.run = next(iter(run_map))
+                run_map = {config.run: run_map.get(config.run)}
 
-        # good description
-        tool_analysis.append(
-            is_ok(
-                message=f"The description for the `{tool_name}` looks sufficient."
+            _, _, _, run_problematic = self._single_run(
+                test_case["dataset_name"], run_map, test_case_resources
             )
+            if run_problematic:
+                if run_files := run_map.get(config.run):
+                    failed_test_cases[test_case["dataset_name"]] = run_files
+
+                else:
+                    # legacy runs without n runs
+                    # tranform the legacy runs into the same data structure from `list_files`
+
+                    messages_path = os.path.join(
+                        test_case_resources.output_dir,
+                        "messages",
+                        f"{test_case['dataset_name']}.messages.json",
+                    )
+
+                    analyze_path = os.path.join(
+                        test_case_resources.output_dir,
+                        "messages",
+                        f"{test_case['dataset_name']}.messages.analyze.json",
+                    )
+
+                    metrics_path = os.path.join(
+                        test_case_resources.output_dir,
+                        "messages",
+                        f"{test_case['dataset_name']}.metrics.json",
+                    )
+
+                    failed_test_cases[test_case["dataset_name"]] = {
+                        "analyze": analyze_path,
+                        "messages": messages_path,
+                        "metrics": metrics_path,
+                    }
+
+        max_workers = config.num_workers
+        rich.print(
+            f"[bold green]INFO:[/bold green] Number of workers set to: {max_workers}"
         )
-        return tool_analysis
+
+        jobs = []
+
+        with ThreadPoolExecutor(
+            max_workers=max_workers, thread_name_prefix="[Worker]"
+        ) as pool:
+            aggregate_results = []
+            for test_case, file_mapping in failed_test_cases.items():
+                analyze_messages, _ = test_case_resources.get_analyze_messages(
+                    path=file_mapping["analyze"]
+                )
+                idx_failed_tool_calls = self._deduplicate_tool_call_failures(
+                    analyze_messages
+                )
+                messages = test_case_resources.get_messages(
+                    path=file_mapping["messages"]
+                )
+
+                for idx in idx_failed_tool_calls:
+                    jobs.append(
+                        {
+                            "task_name": f"{test_case}-0-{idx + 1}",
+                            "test_case": test_case,
+                            "tools": all_tools,
+                            "messages": messages[0 : idx + 1],
+                        }
+                    )
+            jobs = sorted(jobs, key=lambda x: len(x["messages"]))
+            futures = [
+                pool.submit(
+                    self.process_messages,
+                    job["task_name"],
+                    job["test_case"],
+                    job["tools"],
+                    job["messages"],
+                )
+                for job in jobs
+            ]
+
+            if futures:
+                with Progress() as progress:
+                    task = progress.add_task(
+                        f"[purple]Evaluating {len(futures)} tasks...",
+                        total=len(futures),
+                    )
+                    for future in as_completed(futures):
+                        try:
+                            test_case, results = future.result()
+                            aggregate_results.append({test_case: results})
+                        except Exception as e:
+                            rich.print(
+                                f"test case, {test_case} ,fails with {e}"
+                            )
+                            traceback.print_exc()
+                        finally:
+                            progress.update(task, advance=1)
+
+            enhanced_metrics = self.tool_enrichment_view(aggregate_results)
+            end = time.time()
+            rich.print(f"Enhanced Analysis took {end - start} s")
+
+            return enhanced_metrics
+
+    def render(self):
+        raise NotImplementedError("Not implemented")
+
+
+def run(args):
+    d = DescriptionQualityAnalyzer()
+    if args.mode == AnalyzeMode.enhanced:
+        if GATE_TOOL_ENRICHMENTS:
+            d.analyze(args)
+
+        enhanced = AnalyzerEnhanced()
+        enhanced_metrics = enhanced.analyze(config=args)
+        dummy_analyzer = Analyzer(enhanced_metrics, d)
+        dummy_analyzer.analyze(args)
+
+    else:
+        dummy_analyzer = Analyzer()
+        dummy_analyzer.analyze(args)
 
 
 if __name__ == "__main__":
-    dummy_analyzer = Analyzer()
-    dummy_analyzer.analyze(CLI(AnalyzeConfig, as_positional=False))
+    args = CLI(AnalyzeConfig, as_positional=False)
+    run(args)