ibm-watsonx-orchestrate-evaluation-framework 1.0.7__py3-none-any.whl → 1.0.9__py3-none-any.whl

wxo_agentic_evaluation/utils/rich_utils.py

@@ -0,0 +1,174 @@
+from rich.text import Text
+from typing import Optional, List, Any
+import rich
+
+
+def pretty_print(content: Any, style: Optional[str] = None):
+    """
+    Utility function for stylized prints.
+    Please refer to: https://rich.readthedocs.io/en/stable/appendix/colors.html for valid  `style` strings.
+    NOTE:
+        Rich allows for nested [style][/style] tags within a string.
+        This utility only applies an outermost style wrapper using the passed `style` (ONLY for a string `content`).
+
+    :param content: The content to be printed
+    :param style: a valid `rich` colour.
+    """
+    if isinstance(content, str):
+        if style:
+            rich.print(f"[{style}]{content}[/{style}]")
+        else:
+            rich.print(content)
+    else:
+        rich.print(content)
+
+
+def warn(
+    message: str,
+    style: Optional[str] = "bold yellow",
+    prompt: Optional[str] = "WARNING ⚠️ :",
+) -> Text:
+    """Utility function for formatting a warning message."""
+    return Text(f"{prompt}{message}\n\n", style=style)
+
+
+def is_ok(
+    message: str, style: Optional[str] = "bold green", prompt: Optional[str] = "OK ✅ :"
+) -> Text:
+    """Utility function for formatting an OK message."""
+    return Text(f"{prompt}{message}\n\n", style=style)
+
+
+def print_done(prompt: Optional[str] = "Done ✅", style: Optional[str] = "bold cyan"):
+    """
+    Prints a prompt indicating completion of a process/routine.
+    :param prompt: default is `"Done ✅"`
+    :param style: The style for the text (default is bold cyan).
+    """
+    pretty_print(content=prompt, style=style)
+
+
+def print_success(
+    message: str,
+    style: Optional[str] = "bold green",
+    prompt: Optional[str] = "✅ PASSED",
+):
+    """
+    Prints a success message.
+    :param message: a statement that is printed alongside a PASSED outcome.
+    :param style: The style for the text (default is bold green).
+    :param prompt: The prompt to display before the message (default is "✅ PASSED").
+    """
+    pretty_print(content=f"{prompt} - {message}", style=style)
+
+
+def print_failure(
+    message: str, style: Optional[str] = "bold red", prompt: Optional[str] = "❌ FAILED"
+):
+    """
+    Prints a failure message.
+    :param message: a statement that is printed alongside a FAILED outcome.
+    :param style: The style for the text (default is bold red).
+    :param prompt: The prompt to display before the message (default is "❌ FAILED").
+    """
+    pretty_print(content=f"{prompt} - {message}", style=style)
+
+
+class IncorrectParameterUtils:
+    """
+    Utility functions for handling warning and suggestion messages related to bad parameters in tool descriptions.
+    These are primarily used for providing feedback on incorrect parameter usage by the assistant in `analyze_run`.
+    """
+
+    @staticmethod
+    def suggest(message: str, style: Optional[str] = "green") -> Text:
+        """
+        Used for formatting a suggestion message for improving agent behaviour relating to bad parameter usage.
+        :param message: The suggestion message to display.
+        :param style: The style for the text (default is green).
+        :return: A rich Text object styled as a suggestion.
+        """
+        return Text(
+            f"💡 {message}\n✅ A good description is insightful of the tool's purpose, and clarifies parameter usage to the assistant.\n\n",
+            style=style,
+        )
+
+    @staticmethod
+    def format_missing_description_message(
+        tool_definition_path: str, tool_name: str
+    ) -> List[Text]:
+
+        return [
+            warn(
+                f"Tool description for '{tool_name}' not found in file: '{tool_definition_path}'"
+            ),
+            IncorrectParameterUtils.suggest(
+                f"Please consider adding a description for '{tool_name}'."
+            ),
+        ]
+
+    @staticmethod
+    def format_bad_description_message(tool_name: str, tool_desc: str) -> List[Text]:
+
+        return [
+            warn(
+                f"Tool description for '{tool_name}' may be incomplete or unclear: '{tool_desc.strip()}'."
+            ),
+            IncorrectParameterUtils.suggest(
+                f"Please consider making the description for '{tool_name}' more informative on parameter usage."
+            ),
+        ]
+
+
+class TestingUtils:
+    """
+    Provides a collection of formatted messages that can be used in testing workflows.
+    """
+
+    @staticmethod
+    def print_test_header(
+        test_case_count: int,
+        test_description: str,
+        style: Optional[str] = "bold cyan",
+        prompt: Optional[str] = "\n⚙️ Testing",
+    ):
+        """
+        Print formatted test suite header.
+        :param test_case_count: # of test-cases.
+        :param test_description: a short statement explaining what is being examined.
+        For example, this can be read as: `"{\n⚙️ Testing} {20} {good tool descriptions}"`.
+        """
+        pretty_print(
+            content=f"{prompt} {test_case_count} {test_description}", style=style
+        )
+
+    @staticmethod
+    def print_error_details(
+        expected: List[str], detected: List[str], style: Optional[str] = "bold red"
+    ):
+        """
+        Print detailed error information.
+        An error in this context can be an assertion mis-match.
+        Use this function to display the delta.
+        :param expected: the expected outcome.
+        :param detected: the actual/observed outcome.
+        """
+        pretty_print(content=f"   Expected: {expected}", style=style)
+        pretty_print(content=f"   Detected: {detected}", style=style)
+
+    @staticmethod
+    def print_failure_summary(
+        failed_cases: List[str],
+        prompt: Optional[str] = "Failed cases",
+        style: Optional[str] = "bold red",
+    ):
+        """
+        Print summary of all failures.
+        List out the specific cases that failed the test.
+        :param failed_cases: List of failed case names, this list is iterated over to print/list all failures.
+        :param style: The style for the text (default is bold red).
+        """
+        if failed_cases:
+            pretty_print(content=f"{prompt} ({len(failed_cases)}):", style=style)
+            for case in failed_cases:
+                pretty_print(content=f"  - {case}", style=style)

wxo_agentic_evaluation/utils/rouge_score.py

@@ -0,0 +1,23 @@
+def lcs_length(x, y):
+    """Compute the length of the Longest Common Subsequence (LCS)."""
+    m, n = len(x), len(y)
+    dp = [[0] * (n + 1) for _ in range(m + 1)]
+    for i in range(m):
+        for j in range(n):
+            if x[i] == y[j]:
+                dp[i + 1][j + 1] = dp[i][j] + 1
+            else:
+                dp[i + 1][j + 1] = max(dp[i][j + 1], dp[i + 1][j])
+    return dp[m][n]
+
+
+def rouge_l_recall(prediction, reference):
+    """Compute ROUGE-L recall. No stemming."""
+    pred_tokens = prediction.split()
+    ref_tokens = reference.split()
+
+    lcs = lcs_length(pred_tokens, ref_tokens)
+    if len(pred_tokens) == 0:
+        return 0.0
+
+    return lcs / len(pred_tokens)

wxo_agentic_evaluation/utils/utils.py

@@ -5,15 +5,47 @@ from rich.panel import Panel
 from rich.rule import Rule
 from rich import box
 from rich import print
+import re
+from rich.style import Style
 
-from typing import List
+from typing import List, Optional, Union
+import json
+import yaml
+import glob
+import os
 
 from wxo_agentic_evaluation.metrics.llm_as_judge import Faithfulness
-from wxo_agentic_evaluation.metrics.metrics import KnowledgeBaseMetricSummary
-from wxo_agentic_evaluation.type import ConversationalConfidenceThresholdScore
+from wxo_agentic_evaluation.metrics.metrics import KnowledgeBaseMetricSummary, ReferenceLessEvalMetrics
+from wxo_agentic_evaluation.type import ConversationalConfidenceThresholdScore, Message
 
 console = Console()
 
+class AttackResultsTable:
+    def __init__(self, attack_results: dict):
+        self.table = Table(
+            title="Attack Results",
+            box=box.ROUNDED,
+            show_lines=True,
+        )
+        self.table.add_column("Attack Category", style="magenta")
+        self.table.add_column("Count", style="cyan")
+        self.table.add_column("Success Rate", style="green")
+
+        # Extract values
+        n_on_policy = attack_results.get("n_on_policy_attacks", 0)
+        n_off_policy = attack_results.get("n_off_policy_attacks", 0)
+        n_on_policy_successful = attack_results.get("n_on_policy_successful", 0)
+        n_off_policy_successful = attack_results.get("n_off_policy_successful", 0)
+
+        # Calculate success rates
+        on_policy_rate = f"{round(100 * safe_divide(n_on_policy_successful, n_on_policy))}%" if n_on_policy else "0%"
+        off_policy_rate = f"{round(100 * safe_divide(n_off_policy_successful, n_off_policy))}%" if n_off_policy else "0%"
+
+        self.table.add_row("On Policy", str(n_on_policy), on_policy_rate)
+        self.table.add_row("Off Policy", str(n_off_policy), off_policy_rate)
+
+    def print(self):
+        console.print(self.table)
 
 class AgentMetricsTable:
     def __init__(self, data):
@@ -70,8 +102,22 @@ def is_ibm_cloud_url(service_url: str) -> bool:
     return ".cloud.ibm.com" in hostname
 
 
-def add_line_seperator():
-    console.print(Rule(style="grey42"))
+def add_line_seperator(
+        style_config: Optional[
+            Union[str,Style]
+            ]=None,
+):
+    
+    if not style_config:
+        style="grey42"
+    else:
+        style=style_config
+    
+    console.print(
+        Rule(
+        style=style,
+        )
+    )
 
 
 class FaithfulnessTable:
@@ -183,3 +229,119 @@ class SummaryPanel:
 
     def print(self):
         console.print(self.table)
+
+
+class Tokenizer:
+    PATTERN = r"""
+            \w+(?=n't)|              # Words before n't contractions (e.g., "do" in "don't")
+            n't|                     # n't contractions themselves
+            \w+(?=')|                # Words before apostrophes (e.g., "I" in "I'm")
+            '|                       # Apostrophes as separate tokens
+            \w+|                     # Regular words (letters, numbers, underscores)
+            [^\w\s]                  # Punctuation marks (anything that's not word chars or whitespace)
+        """
+    
+    def __init__(self):
+        self.compiled_pattern = re.compile(
+            self.PATTERN,
+            re.VERBOSE | re.IGNORECASE
+        )
+    
+    def __call__(self, text: str) -> List[str]:
+        """
+        Tokenizes text by splitting on punctuation and handling contractions.
+
+        Args:
+            text: Input text to tokenize.
+        
+        Returns:
+            List of tokenized words (lowercase, no punctuation).
+    
+        Examples:
+            - "I'm fine"      -> ['i', 'm', 'fine']
+            - "don't go"      -> ['do', "n't", 'go']  
+            - "Hello, world!" -> ['hello', 'world']
+        """
+        
+        tokens = self.compiled_pattern.findall(
+            text
+        )
+    
+        return self._clean_tokens(tokens)
+    
+    def _clean_tokens(self, raw_tokens: List[str]) -> List[str]:
+        """
+        Applies some basic post-processing to tokenized messages.
+
+        Args:
+            raw_tokens: list of tokens extracted from a message.
+        """
+        
+        filtered_tokens = [
+            token.lower() \
+            for token in raw_tokens \
+            if token.strip() \
+            and not (len(token) == 1 and not token.isalnum())
+        ]
+
+        return filtered_tokens
+
+
+class ReferencelessEvalPanel:
+    def __init__(self, referenceless_metrics: List[ReferenceLessEvalMetrics]):
+        self.table = Table(
+            title="Quick Evaluation Summary Metrics",
+            box=box.ROUNDED,
+            show_lines=True,
+        )
+
+        self.table.add_column("Dataset", style="yellow", justify="center")
+        self.table.add_column("Tool Calls", style="deep_sky_blue1", justify="center")
+        self.table.add_column("Successful Tool Calls", style="magenta", justify="center")
+        self.table.add_column("Tool Calls Failed due to Schema Mismatch", style="deep_sky_blue1", justify="center")
+        self.table.add_column("Tool Calls Failed due to Hallucination", style="magenta", justify="center")
+
+        for metric in referenceless_metrics:
+            self.table.add_row(
+                str(metric.dataset_name),
+                str(metric.number_of_tool_calls),
+                str(metric.number_of_successful_tool_calls),
+                str(metric.number_of_static_failed_tool_calls),
+                str(metric.number_of_semantic_failed_tool_calls)
+            )
+
+    def print(self):
+        console.print(self.table)
+
+# Function to load messages from JSON file
+def load_messages(file_path):
+    with open(file_path, "r") as f:
+        try:
+            message_data = json.load(f)
+            messages = []
+            for msg in message_data:
+                messages.append(Message.model_validate(msg))
+
+            return messages
+
+        except Exception as e:
+            print(file_path)
+            print(e)
+            return None
+
+
+def load_agents(agents_path: str):
+    agents_json = glob.glob(os.path.join(agents_path, "*.json"))
+    agents_yaml = glob.glob(os.path.join(agents_path, "*.yaml"))
+
+    agents = []
+
+    for agent_path in agents_json:
+        with open(agent_path, "r") as f:
+            agents.append(json.load(f))
+    
+    for agent_path in agents_yaml:
+        with open(agent_path, "r") as f:
+            agents.append(yaml.safe_load(f))
+    
+    return agents

ibm_watsonx_orchestrate_evaluation_framework-1.0.7.dist-info/RECORD

@@ -1,56 +0,0 @@
-wxo_agentic_evaluation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wxo_agentic_evaluation/analyze_run.py,sha256=C4HowEukNMM-H8FkRcHRqkiNYIQVCoTKbBLiqr1cFRM,4332
-wxo_agentic_evaluation/annotate.py,sha256=nxYMc6gwfQ-GuNjCPFtbX_-Es5-9XDdbXpMH89yRDdc,1228
-wxo_agentic_evaluation/arg_configs.py,sha256=Nc-Z9hG5ZgHAJIdLqUDv-Ct7Wkxvs_VGy-A3JwkC-PI,2265
-wxo_agentic_evaluation/batch_annotate.py,sha256=44K4DUI498uaLIWUn3nz82AKcU6VnCjrExoG6GpPHoM,6323
-wxo_agentic_evaluation/data_annotator.py,sha256=6cUUpCTFSs36VF3wICLXWrWbEUJz6v-PzPeuzO9S1k8,8310
-wxo_agentic_evaluation/evaluation_package.py,sha256=jOSe-TCJdAWCk1sWpRYfi_EMkZERrVf5swm-bxfozzc,21333
-wxo_agentic_evaluation/inference_backend.py,sha256=fhEB1kaNN-A08RtJglBiv3QL_8nq8m-g7xbF4WbHAvU,25691
-wxo_agentic_evaluation/llm_matching.py,sha256=l010exoMmsvTIAVHCm-Ok0diyeQogjCmemUb9rJLe6A,1477
-wxo_agentic_evaluation/llm_rag_eval.py,sha256=vsNGz1cFE5QGdhnfrx-iJq1r6q8tSI9Ef1mzuhoHElg,1642
-wxo_agentic_evaluation/llm_user.py,sha256=0zSsyEM7pYQtLcfbnu0gEIkosHDwntOZY84Ito6__SM,1407
-wxo_agentic_evaluation/main.py,sha256=tRXVle2o1JhwJZOTpqdsOzBOpxPYxAH5ziZkbCmzfyU,11470
-wxo_agentic_evaluation/record_chat.py,sha256=uFdbLt4HaMREN3q4HHAA1ZvtjoLdiBEyxPd9Eoc6svc,8103
-wxo_agentic_evaluation/resource_map.py,sha256=11qF1oJDwGNWOLYFVsIPsR66JK4eD0cqVOBKreK2mPQ,1644
-wxo_agentic_evaluation/service_instance.py,sha256=yt7XpwheaRRG8Ri4TFIS5G2p5mnCwvNgj6T7bDF5uTU,6494
-wxo_agentic_evaluation/test_prompt.py,sha256=ksteXCs9iDQPMETc4Hb7JAXHhxz2r678U6-sgZJAO28,3924
-wxo_agentic_evaluation/tool_planner.py,sha256=JW5o0VYaaUorB3FBcrwLzgG3-iqEWrqjVhh82u7x8YM,12960
-wxo_agentic_evaluation/type.py,sha256=uVKim70XgPW-3L7Z0yRO07wAH9xa-NcjfaiIyPhYMR0,3413
-wxo_agentic_evaluation/analytics/tools/analyzer.py,sha256=IPX_lAFujjPVI9fhXTNohXTxTmpqRhfzQygCWDYHBHg,18125
-wxo_agentic_evaluation/analytics/tools/main.py,sha256=ocwPUlEjyK7PMdXBg5OM2DVDQBcaHT4UjR4ZmEhR0C4,6567
-wxo_agentic_evaluation/analytics/tools/types.py,sha256=IFLKI1CCQwPR2iWjif8AqL_TEq--VbLwdwnMqfJujBw,4461
-wxo_agentic_evaluation/analytics/tools/ux.py,sha256=EaWNvsq68X_i2H4pQ2fABtXEEmk3ZXqaMrTs42_7MwE,18347
-wxo_agentic_evaluation/external_agent/__init__.py,sha256=LY3gMNzfIEwjpQkx5_2iZFHGQiUL4ymEkKL1dc2uKq4,1491
-wxo_agentic_evaluation/external_agent/external_validate.py,sha256=xW8tqPcm8JYvveSxf-oFCajvF5J8ORaK23YXu-LuFmc,4142
-wxo_agentic_evaluation/external_agent/performance_test.py,sha256=bCXUsW0OeUzwfSSYObgfAmEU5vARkD-PblYU-mU9aPY,2507
-wxo_agentic_evaluation/external_agent/types.py,sha256=4kfWD_ZyGZmpbib33gCxEuKS4HLb7CEtferlQgQe7uk,1624
-wxo_agentic_evaluation/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wxo_agentic_evaluation/metrics/llm_as_judge.py,sha256=bybJQfVWiVh3BoFEZjdBmU9EQO9Ukheu3YWmkI9b1ks,1218
-wxo_agentic_evaluation/metrics/metrics.py,sha256=9O2m6T2iW-PMjGrTdMbOHP2Pr4RN0NwbEp6YgFpTi3I,5572
-wxo_agentic_evaluation/prompt/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wxo_agentic_evaluation/prompt/answer_relevancy_prompt.jinja2,sha256=vLrMWce-5HlvniCQdtifnl-YdbJfT8-oixzfwulZs98,3839
-wxo_agentic_evaluation/prompt/args_extractor_prompt.jinja2,sha256=0qBicXFcc6AA3mQNLPVRmFsnuYaCABJXgZkIH9fO0Js,952
-wxo_agentic_evaluation/prompt/batch_testcase_prompt.jinja2,sha256=QXuk2ecnEPPRCPoWZJyrtb1gAVuIPljB91YoqPBp2Dk,1896
-wxo_agentic_evaluation/prompt/faithfulness_prompt.jinja2,sha256=DW9OdjeZJbOWrngRqTAVD4w0va_HtA2FR4G1POIIamM,2524
-wxo_agentic_evaluation/prompt/keyword_matching_prompt.jinja2,sha256=7mTkSrppjgPluUAIMTWaT30K7M4J4hyR_LjSjW1Ofq0,1290
-wxo_agentic_evaluation/prompt/keywords_generation_prompt.jinja2,sha256=PiCjr1ag44Jk5xD3F24fLD_bOGYh2sF0i5miY4OrVlc,1890
-wxo_agentic_evaluation/prompt/llama_user_prompt.jinja2,sha256=nDfCD0o9cRYmsgIjzD-RZNQxotlvuqrzdsZIY-vT794,684
-wxo_agentic_evaluation/prompt/semantic_matching_prompt.jinja2,sha256=MltPfEXYyOwEC2xNLl7UsFTxNbr8CwHaEcPqtvKE2r8,2749
-wxo_agentic_evaluation/prompt/starting_sentence_generation_prompt.jinja2,sha256=m_l6f7acfnWJmGQ0mXAy85oLGLgzhVhoz7UL1FVYq8A,4908
-wxo_agentic_evaluation/prompt/story_generation_prompt.jinja2,sha256=_DxjkFoHpNTmdVSUzUrUdwn4Cng7nAGqkMnm0ScOH1w,4191
-wxo_agentic_evaluation/prompt/template_render.py,sha256=FVH5ew2TofC5LGqQzqNj90unrxooUZv_5XxJzVdz8uM,3563
-wxo_agentic_evaluation/prompt/tool_chain_agent.jinja2,sha256=9RcIjLYoOvtFsf-RgyMfMcj2Fe8fq1wGkE4nG1zamYY,297
-wxo_agentic_evaluation/prompt/tool_planner.jinja2,sha256=Ln43kwfSX50B1VBsT-MY1TCE0o8pGFh8aQJAzZfGkpI,3239
-wxo_agentic_evaluation/prompt/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-wxo_agentic_evaluation/prompt/examples/data_simple.json,sha256=XXF-Pn-mosklC9Ch7coyaJxosFNnl3OkHSW3YPuiKMM,2333
-wxo_agentic_evaluation/service_provider/__init__.py,sha256=EaY4jjKp58M3W8N3b3a8PNC2S81xA7YV2_QkTIy9DfI,1600
-wxo_agentic_evaluation/service_provider/model_proxy_provider.py,sha256=Y36Ryv4nPG8RdVP_zsQsRlEWv8F_hGi7-wOppWPQTwc,4026
-wxo_agentic_evaluation/service_provider/ollama_provider.py,sha256=HMHQVUGFbLSQI1dhysAn70ozJl90yRg-CbNd4vsz-Dc,1116
-wxo_agentic_evaluation/service_provider/provider.py,sha256=MsnRzLYAaQiU6y6xf6eId7kn6-CetQuNZl00EP-Nl28,417
-wxo_agentic_evaluation/service_provider/watsonx_provider.py,sha256=iKVkWs4PRTM_S0TIdPgQ9NFQWPlDvcEvuHpQlIPzO10,6216
-wxo_agentic_evaluation/utils/__init__.py,sha256=QMxk6hx1CDvCBLFh40WpPZmqFNJtDqwXP7S7cXD6NQE,145
-wxo_agentic_evaluation/utils/utils.py,sha256=JYZQZ-OBy43gAWg9S7duJi9StRApGJATs2JUsW1l30M,6057
-ibm_watsonx_orchestrate_evaluation_framework-1.0.7.dist-info/METADATA,sha256=wz60je0UK3ogKLH9qiDLS808j57cfWOosONyCuQR95g,18051
-ibm_watsonx_orchestrate_evaluation_framework-1.0.7.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-ibm_watsonx_orchestrate_evaluation_framework-1.0.7.dist-info/top_level.txt,sha256=2okpqtpxyqHoLyb2msio4pzqSg7yPSzwI7ekks96wYE,23
-ibm_watsonx_orchestrate_evaluation_framework-1.0.7.dist-info/RECORD,,