ibm-watsonx-orchestrate-evaluation-framework 1.0.8__py3-none-any.whl → 1.1.0__py3-none-any.whl

wxo_agentic_evaluation/analyze_run.py

@@ -1,123 +1,384 @@
 import json
 import os
 import csv
-import rich
+from jsonargparse import CLI
+from pathlib import Path
+from typing import List, Dict, Set, Optional
+
 from rich.text import Text
-from rich.panel import Panel
-from rich.layout import Layout
 from rich.table import Table
-from typing import List
-from wxo_agentic_evaluation.type import (
-    ExtendedMessage,
-    ContentType
-)
+from rich.panel import Panel
+from rich.console import Group
+from rich.style import Style
+
+from wxo_agentic_evaluation.type import ExtendedMessage, ContentType, ToolDefinition
 from wxo_agentic_evaluation.metrics.metrics import ToolCallAndRoutingMetrics
 from wxo_agentic_evaluation.arg_configs import AnalyzeConfig
-from jsonargparse import CLI
+from wxo_agentic_evaluation.description_quality_checker import (
+    DescriptionQualityInspector,
+)
+from wxo_agentic_evaluation.utils.rich_utils import (
+    pretty_print,
+    warn,
+    is_ok,
+    print_done,
+    IncorrectParameterUtils,
+)
+from wxo_agentic_evaluation.utils.utils import (
+    add_line_seperator,
+)
 
 
-def render(data: List[ExtendedMessage]):
-    conversation_lines = []
-    reason_lines = []
+class Analyzer:
 
-    for entry in data:
-        msg = entry.message
-        role = msg.role
-        content = msg.content
-        reason = entry.reason
-        tool_name = None
-        if role == "user":
-            label = "👤 User"
-        elif role == "assistant" and msg.type == ContentType.tool_call:
-            if reason:
-                label = "❌ Tool Call"
-                tool_name = json.loads(msg.content)["name"]
+    def __init__(self):
+        self.analysis_cache: Dict[str, List[Text]] = (
+            {}
+        )  # the failing tools cached here won't be re-analyzed.
+        # tool_name -> description analysis
+
+    @staticmethod
+    def _generate_style_config():
+        return Style(
+            color="magenta",
+            blink=True,
+            bold=True,
+        )
+    
+    def _split_cache(self, failing_tools: Set[str]) -> tuple[List[str], List[Text]]:
+
+        tools_to_analyze: List[str] = []
+        cached_lines: List[Text] = []
+        tools_analyzed: List[str] = []
+
+        for tool_name in sorted(failing_tools):
+            cached_analysis = self.analysis_cache.get(tool_name)
+            if cached_analysis:
+                cached_lines.extend(cached_analysis)
+                tools_analyzed.append(tool_name)
             else:
-                label = "✅ Tool Call"
-        elif role == "assistant":
-            label = "🤖 Assistant"
-        else:
-            label = "📦 Unknown"
-
-        text_line = Text(f"{label}: {content}\n")
-        if reason:
-            text_line.stylize("bold red")
-            reason_text = f"❌ {tool_name}: {json.dumps(reason)}\n\n"
-            reason_lines.append(Text(reason_text, style="red"))
-        conversation_lines.append(text_line)
-
-    conversation_panel = Panel(
-        Text().join(conversation_lines),
-        title="Conversation History",
-        border_style="blue",
-    )
-    reason_panel = Panel(
-        Text().join(reason_lines), title="Analysis Results", border_style="red"
-    )
-
-    layout = Layout()
-    layout.split_row(Layout(conversation_panel), Layout(reason_panel))
-
-    return layout
-
-
-def analyze(config: AnalyzeConfig):
-    summary = []
-    with open(os.path.join(config.data_path, "summary_metrics.csv"), "r") as f:
-        reader = csv.reader(f)
-        header = next(reader)
-        for row in reader:
-            summary.append(dict(zip(header, row)))
-
-    test_case_with_failed_tools = []
-    for entry in summary:
-        test_case_name = entry["dataset_name"]
-        if test_case_name.lower().strip() == "summary (average)":
-            continue
-        if not entry["is_success"] or float(entry["tool_calls_with_incorrect_parameter"]) > 0 or float(entry["tool_call_precision"]) < 1.0\
-                or float(entry["tool_call_recall"]) < 1.0:
-            test_case_with_failed_tools.append(entry)
-    if len(test_case_with_failed_tools) == 0:
-        header_table = Table(show_header=False, box=None)
-        header_table.add_row(f"No Tool Call Error found!")
-        header_panel = Panel(
-            header_table, title="[bold green]📋 Analysis Summary[/bold green]"
+                tools_to_analyze.append(tool_name)
+
+        if tools_analyzed:
+            pretty_print(
+                content=f"ℹ️ Loading cached analysis since these failing tools: {tools_analyzed} have been analyzed previously.",
+                style="bold cyan",
+            )
+
+        return (
+            tools_to_analyze, 
+            cached_lines
+            )
+
+
+    def analyze_failing_tool_description_quality(
+        self,
+        inspector: DescriptionQualityInspector,
+        tool_definition_path: str,
+        failing_tools: Set[str],
+    ) -> List[Text]:
+        """
+        :param tool_definition_path: Path to the tool definition file.
+        :param failing_tools: Set of tool names that failed.
+        :return: List of rich `Text` objects containing feedback for the customer.
+        """
+
+        pretty_print(
+            content=f"⚙️ Checking tool description quality for failing tools: {sorted(failing_tools)}",
+            style="bold cyan",
         )
-        rich.print(header_panel)
 
-    for test_case_entry in test_case_with_failed_tools:
-        test_case_name = test_case_entry["dataset_name"]
+        analysis_for_display: List[Text] = []
+
+        # Step 1: get tools not yet analyzed and cached analysis for tools analyzed previously
+        tools_to_analyze, cached_analysis = self._split_cache(failing_tools)
+        if cached_analysis:
+            analysis_for_display.extend(cached_analysis)
+
+        # Step 2: analyze cache misses
+        if tools_to_analyze:
+
+            failing_tool_definitions: List[ToolDefinition] = inspector.extract_tool_desc_from_tool_source(
+                Path(tool_definition_path),
+                tools_to_analyze,
+            )
+
+            if not failing_tool_definitions:
+                analysis_for_display.append(
+                    warn(
+                        message=f"No tool definitions(with '@tool' decorators) for failed tools: '{tools_to_analyze}' found in the file: '{tool_definition_path}'"
+                    )
+                )
+                return analysis_for_display
+        
+            missing_tools = self._get_tools_not_found_in_source(
+                tools_to_analyze, failing_tool_definitions
+            )
+            if missing_tools:
+                analysis_for_display.append(
+                    warn(
+                        message=f"Missing tool definitions for failed tools: '{missing_tools}' in the file: '{tool_definition_path}'"
+                    )
+                )
+
+            for tool_definition in failing_tool_definitions:
+
+                tool_analysis = self._analyze_tool_definition(
+                    inspector=inspector,
+                    tool_definition=tool_definition,
+                    tool_definition_path=tool_definition_path,
+                )
+
+                self.analysis_cache[tool_definition.tool_name] = tool_analysis
+                analysis_for_display.extend(tool_analysis)
+
+        return analysis_for_display
+
+    def render(self, data: List[ExtendedMessage], tool_definition_path: Optional[str]) -> Group:
+        """
+        Render the conversation history and analysis results.
+        :param data: List of ExtendedMessage objects containing the conversation history.
+        :param tool_definition_path: Path to the tool definition file.
+        :return: A rich Group object containing the conversation and analysis results.
+        """
+        conversation_lines = []
+        reason_lines = []
+        failing_tools = []
+
+        for entry in data:
+            msg = entry.message
+            role = msg.role
+            content = msg.content
+            reason = entry.reason
+            tool_name = None
+            if msg.type == ContentType.tool_call or msg.type == ContentType.tool_response:
+                tool_name = json.loads(msg.content)["name"]
 
-        test_case_path = os.path.join(
-            config.data_path, "messages", f"{test_case_name}.messages.analyze.json"
+            if role == "user":
+                label = "👤 User"
+            elif role == "assistant" and msg.type == ContentType.tool_call:
+                if reason:
+                    label = "❌ Tool Call"
+                      
+                    if reason.get("reason") == "incorrect parameter":
+                        failing_tools.append(
+                            tool_name
+                        )  # create a list of failing tools for description quality analysis.
+                else:
+                    label = "✅ Tool Call"
+            elif role == "assistant":
+                label = "🤖 Assistant"
+            else:
+                label = "📦 Unknown"
+
+            text_line = Text(f"{label}: {content}\n")
+            if reason:
+                text_line.stylize("bold red")
+                reason_text = f"❌ {tool_name}: {json.dumps(reason)}\n\n"
+                reason_lines.append(Text(reason_text, style="red"))
+            conversation_lines.append(text_line)
+
+        if failing_tools and tool_definition_path:
+
+            inspector = DescriptionQualityInspector()
+
+            description_quality_inspection_lines = (
+                self.analyze_failing_tool_description_quality(
+                    inspector, tool_definition_path, set(failing_tools)
+                )
+            )
+
+            print_done()
+
+            if description_quality_inspection_lines:
+                reason_lines.extend(description_quality_inspection_lines)
+
+        conversation_panel = Panel(
+            Text().join(conversation_lines),
+            title="Conversation History",
+            border_style="blue",
         )
-        test_messages = []
-        with open(test_case_path, "r", encoding="utf-8") as f:
-            temp = json.load(f)
-            for entry in temp:
-                msg = ExtendedMessage(**entry)
-                test_messages.append(msg)
-
-        test_metrics_path = os.path.join(
-            config.data_path, "messages", f"{test_case_name}.metrics.json"
+        reason_panel = Panel(
+            Text().join(reason_lines), 
+            title="Analysis Results", 
+            border_style="red",
+        )
+
+        return Group(
+            conversation_panel,
+            reason_panel,
         )
-        with open(test_metrics_path, "r", encoding="utf-8") as f:
-            metrics = ToolCallAndRoutingMetrics(**json.load(f))
+
+    def analyze(self, config: AnalyzeConfig):
+        """
+        Analyze the results of the tool calls and routing metrics.
+        :param config: AnalyzeConfig object containing user provided paths for analysis.
+        """
+
+        def get_summary(summary_file_name: str = "summary_metrics.csv"):
+            summary = []
+
+            path_to_summary_file = os.path.join(config.data_path, summary_file_name)
+
+            with open(path_to_summary_file, "r") as f:
+                reader = csv.reader(f)
+                header = next(reader)
+                for row in reader:
+                    summary.append(dict(zip(header, row)))
+
+            return summary
+
+        def get_test_messages(test_case_name):
+            test_messages = []
+
+            test_case_path = os.path.join(
+                config.data_path, "messages", f"{test_case_name}.messages.analyze.json"
+            )
+
+            with open(test_case_path, "r", encoding="utf-8") as f:
+                temp = json.load(f)
+                for entry in temp:
+                    msg = ExtendedMessage(**entry)
+                    test_messages.append(msg)
+
+            return test_messages
+
+        def get_metrics(test_case_name):
+            test_metrics_path = os.path.join(
+                config.data_path, "messages", f"{test_case_name}.metrics.json"
+            )
+
+            with open(test_metrics_path, "r", encoding="utf-8") as f:
+                metrics = ToolCallAndRoutingMetrics(**json.load(f))
+
+            return metrics
+
+        summary = get_summary()
+
+        test_case_with_failed_tools = self._get_test_case_with_failed_tools(
+            summary=summary
+        )
+
+        if len(test_case_with_failed_tools) == 0:
+            header_table = Table(show_header=False, box=None)
+
+            header_table.add_row("No Tool Call Error found!")
+
+            panel = Panel(
+                header_table, title="[bold green]📋 Analysis Summary[/bold green]"
+            )
+
+            pretty_print(panel)
+
+        for test_case_entry in test_case_with_failed_tools:
+            test_case_name = test_case_entry["dataset_name"]
+
+            test_messages = get_test_messages(test_case_name=test_case_name)
+
+            metrics: ToolCallAndRoutingMetrics = get_metrics(
+                test_case_name=test_case_name
+            )
+
+            header_panel = self._create_header_analysis_panel(test_case_name, metrics)
+            pretty_print(header_panel)
+
+            tool_definition_path = config.tool_definition_path \
+                if config.tool_definition_path else None
+            
+            rendered_content = self.render(
+                data=test_messages,
+                tool_definition_path=tool_definition_path
+                )
+            pretty_print(rendered_content)
+
+            add_line_seperator(
+                self._generate_style_config()
+            )
+
+    def _create_header_analysis_panel(
+        self, test_case_name: str, metrics: ToolCallAndRoutingMetrics
+    ) -> Panel:
         header_table = Table(show_header=False, box=None)
+
         header_table.add_row(f"Test Case Name: {test_case_name}")
-        header_table.add_row((f"Expected Tool Calls: {metrics.expected_tool_calls}"))
+        header_table.add_row(f"Expected Tool Calls: {metrics.expected_tool_calls}")
         header_table.add_row(f"Correct Tool Calls: {metrics.correct_tool_calls}")
         header_table.add_row(f"Text Match: {metrics.text_match.value}")
-        header_table.add_row(
-            f"Journey Success: {metrics.is_success}"
-        )
+        header_table.add_row(f"Journey Success: {metrics.is_success}")
+
         header_panel = Panel(
             header_table, title="[bold green]📋 Analysis Summary[/bold green]"
         )
-        rich.print(header_panel)
-        layout = render(test_messages)
-        rich.print(layout)
 
+        return header_panel
+
+    def _get_test_case_with_failed_tools(self, summary) -> List:
+
+        test_case_with_failed_tools = []
+
+        for entry in summary:
+            test_case_name = entry["dataset_name"]
+
+            if test_case_name.lower().strip() == "summary (average)":
+                continue
+
+            if (
+                not entry["is_success"]
+                or float(entry["tool_calls_with_incorrect_parameter"]) > 0
+                or float(entry["tool_call_precision"]) < 1.0
+                or float(entry["tool_call_recall"]) < 1.0
+            ):
+
+                test_case_with_failed_tools.append(entry)
+
+        return test_case_with_failed_tools
+
+    def _get_tools_not_found_in_source(
+        self,
+        tools_to_analyze: List[str],
+        failing_tool_definitions: List[ToolDefinition],
+    ) -> Set[str]:
+
+        return set(tools_to_analyze) - {
+            tool_def.tool_name for tool_def in failing_tool_definitions
+        }
+
+    def _analyze_tool_definition(
+        self,
+        inspector: DescriptionQualityInspector,
+        tool_definition: ToolDefinition,
+        tool_definition_path: str,
+    ) -> List[Text]:
+
+        tool_name = tool_definition.tool_name
+        tool_desc = tool_definition.tool_description
+
+        tool_analysis = []
+
+        # missing description
+        if tool_desc is None:
+            tool_analysis.extend(
+                IncorrectParameterUtils.format_missing_description_message(
+                    tool_name=tool_name, tool_definition_path=tool_definition_path
+                )
+            )
+            return tool_analysis
+
+        # bad description
+        if inspector.detect_bad_description(tool_definition):
+            tool_analysis.extend(
+                IncorrectParameterUtils.format_bad_description_message(
+                    tool_name=tool_name, tool_desc=tool_desc
+                )
+            )
+            return tool_analysis
+
+        # good description
+        tool_analysis.append(
+            is_ok(message=f"The description for the `{tool_name}` looks sufficient.")
+        )
+        return tool_analysis
 
 if __name__ == "__main__":
-    analyze(CLI(AnalyzeConfig, as_positional=False))
+    dummy_analyzer = Analyzer()
+    dummy_analyzer.analyze(CLI(AnalyzeConfig, as_positional=False))

wxo_agentic_evaluation/arg_configs.py

@@ -1,6 +1,6 @@
 import os
 from dataclasses import dataclass, field
-from typing import List
+from typing import List, Optional, Union
 from wxo_agentic_evaluation import __file__
 
 root_dir = os.path.dirname(__file__)
@@ -43,9 +43,32 @@ class TestConfig:
     num_workers: int = 2
 
 
+@dataclass
+class AttackConfig:
+    attack_paths: List[str]
+    output_dir: str
+    auth_config: AuthConfig
+    provider_config: ProviderConfig = field(default_factory=ProviderConfig)
+    llm_user_config: LLMUserConfig = field(default_factory=LLMUserConfig)
+    enable_verbose_logging: bool = True
+    enable_manual_user_input: bool = False
+    num_workers: int = 2
+
+
+@dataclass
+class AttackGeneratorConfig:
+    attacks_list: Union[List[str], str]
+    datasets_path: Union[List[str], str]
+    agents_path: str
+    target_agent_name: str
+    output_dir: str = None
+    max_variants: int = None
+
+
 @dataclass
 class AnalyzeConfig:
     data_path: str
+    tool_definition_path: Optional[str] = None
 
 
 @dataclass
@@ -77,6 +100,10 @@ class ChatRecordingConfig:
     max_retries: int = 5
 
 
+@dataclass
+class QuickEvalConfig(TestConfig):
+    tools_path: str = None
+
 @dataclass
 class BatchAnnotateConfig:
     allowed_tools: List[str]

wxo_agentic_evaluation/description_quality_checker.py

@@ -0,0 +1,149 @@
+import os
+from pathlib import Path
+from typing import List
+import rich
+from enum import Enum
+
+from wxo_agentic_evaluation.service_provider import get_provider
+from wxo_agentic_evaluation.prompt.template_render import BadToolDescriptionRenderer
+from wxo_agentic_evaluation.tool_planner import (
+    parse_json_string,
+    extract_tool_signatures,
+    MISSING_DOCSTRING_PROMPT,
+)
+from wxo_agentic_evaluation.type import ToolDefinition
+from wxo_agentic_evaluation.utils.utils import safe_divide
+
+
+class ToolDescriptionIssue(Enum):
+    """
+    Represents the binary outcomes the LLM judge will classify in its assessment \
+    of the tool's description.
+    The presence of these issues in the tool's description indicates poor quality.
+    For more detail on what each issue indicates, please take a look at the template here: `wxo_agentic_evaluation/prompt/bad_tool_descriptions_prompt.jinja2`.
+    """
+
+    # TODO: Priority-based weighting of issues.
+    CONTAINS_REDUNDANT_INFORMATION = "contains_redundant_information"
+    USES_VAGUE_LANGUAGE = "uses_vague_language"
+    DOES_NOT_HELP_IN_IDENTIFYING_TOOL_UNIQUELY = (
+        "does_not_help_in_identifying_tool_uniquely"
+    )
+    PROVIDES_NO_NEW_INFORMATION = "provides_no_new_information"
+    DOES_NOT_CONVEY_TOOL_PURPOSE = "does_not_convey_tool_purpose"
+
+
+class DescriptionQualityInspector:
+    DEFAULT_CLASSIFICATION_THRESHOLD = (
+        40.0  # 2/5 issues detected. A higher score indicates a worse description.
+    )
+    CLASSIFICATION_SCORE_THRESHOLD = float(
+        os.getenv("CLASSIFICATION_SCORE_THRESHOLD", DEFAULT_CLASSIFICATION_THRESHOLD)
+    )
+
+    LLM_MODEL_ID = "meta-llama/llama-3-2-90b-vision-instruct"
+    LLM_PARAMS = {
+        "min_new_tokens": 128,
+        "decoding_method": "greedy",
+        "max_new_tokens": 512,
+    }
+
+    WORST_POSSIBLE_EVAL_OUTCOME = len(
+        ToolDescriptionIssue
+    )  # the final score used for classification is normalized against this value.
+
+    root_dir = os.path.dirname(__file__)
+    BAD_TOOL_DESCRIPTIONS_DETECTOR_PATH = os.path.join(
+        root_dir, "prompt", "bad_tool_descriptions_prompt.jinja2"
+    )
+
+    def __init__(self, llm_client=None):
+        if llm_client is None:
+            llm_client = get_provider(
+                model_id=self.LLM_MODEL_ID,
+                params=self.LLM_PARAMS,
+            )
+        self.llm_client = llm_client
+        self.template = BadToolDescriptionRenderer(
+            self.BAD_TOOL_DESCRIPTIONS_DETECTOR_PATH
+        )
+        self.cached_response = (
+            None  # this is used in the unit-tests for nuanced analysis of the response.
+        )
+
+    @staticmethod
+    def extract_tool_desc_from_tool_source(
+        tool_source: Path, failing_tools: List[str]
+    ) -> List[ToolDefinition]:
+        """
+        Parses the tool source file to extract the tool description.
+        Wraps the description along with the tool name, and args into a `ToolDefinition` for all `failing_tools`.
+        This `ToolDefinition` is later rendered into the judge's prompt template for evaluation.
+        Args:
+            tool_source (Path): The path to the tool source file/dir containing `.py` tools.
+            failing_tools (List[str]): List of tool names that failed during inference.
+        Returns:
+            List[ToolDefinition]: The extracted tool definition(s) or [] if the file contains no @tool decorators.
+        """
+        all_tool_data = extract_tool_signatures(tool_source)
+
+        tool_definitions = []
+        for tool_data in all_tool_data:
+            tool_name = tool_data["Function Name"]
+            if tool_name in failing_tools:
+                tool_definitions.append(
+                    ToolDefinition(
+                        tool_name=tool_name,
+                        tool_description=(
+                            tool_data["Docstring"]
+                            if tool_data["Docstring"] != MISSING_DOCSTRING_PROMPT
+                            else None
+                        ),
+                        tool_params=tool_data["Arguments"],
+                    )
+                )
+        return tool_definitions
+
+    def detect_bad_description(self, tool_definition: ToolDefinition) -> bool:
+        """
+        Detects if a tool description is 'bad' using an LLM judge.
+        A 'bad' description is one that:
+            - does not describe the tool's functionality/use-case clearly
+            - does not provide sufficient detail for an agent to understand how to use the tool
+            - does not distinguish the tool from other tools
+            For the exact definition of a 'bad' description, refer to `ToolDescriptionIssue` Enum.
+        Args:
+            tool_definition (ToolDefinition): The definition of the tool to evaluate.
+        Returns:
+            bool: True if the description is 'bad', False otherwise.
+        """
+        prompt = self.template.render(tool_definition=tool_definition)
+        response = self.llm_client.query(prompt)
+
+        # parse JSON objects from cleaned text
+        json_objects = parse_json_string(response)
+
+        # pick the first JSON object
+        if json_objects:
+            response_data = json_objects[0]
+            self.cached_response = response_data
+        else:
+            return False  # likely some unexpected parsing issue, in this case - flags description as good.
+
+        # calculate weighted score
+        final_description_score = self._calculate_score(response_data=response_data)
+
+        return final_description_score >= self.CLASSIFICATION_SCORE_THRESHOLD
+
+    def _calculate_score(self, response_data: dict) -> float:
+        """
+        Calculates a final score for the tool description.
+        This score is used to finally classify a 'good' or 'bad' description.
+        :param response_data: Parsed JSON response returned by the LLM judge.
+        """
+        detected_issues = sum(
+            1
+            for issue in ToolDescriptionIssue
+            if response_data.get(issue.value, "FALSE").upper() == "TRUE"
+        )
+        return safe_divide(detected_issues, self.WORST_POSSIBLE_EVAL_OUTCOME) * 100