ibm-watsonx-orchestrate-evaluation-framework 1.0.0__py3-none-any.whl

wxo_agentic_evaluation/llm_user.py

@@ -0,0 +1,38 @@
+from typing import List, TypeVar
+from wxo_agentic_evaluation.type import Message, ContentType
+from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.prompt.template_render import JinjaTemplateRenderer
+
+T = TypeVar("T", bound=JinjaTemplateRenderer)
+
+
+class LLMUser:
+    def __init__(
+        self, wai_client: WatsonXProvider, template: T, user_response_style: List[str]
+    ):
+        self.wai_client = wai_client
+        self.prompt_template = template
+        self.user_response_style = (
+            [] if user_response_style is None else user_response_style
+        )
+
+    def generate_user_input(
+        self, user_story, conversation_history: List[Message]
+    ) -> Message | None:
+        # the tool response is already summarized, we don't need that to take over the chat history context window
+        prompt_input = self.prompt_template.render(
+            conversation_history=[
+                entry
+                for entry in conversation_history
+                if entry.type != ContentType.tool_response
+            ],
+            user_story=user_story,
+            user_response_style=self.user_response_style,
+        )
+        user_input = self.wai_client.query(prompt_input)
+        user_input = Message(
+            role="user",
+            content=user_input["generated_text"].strip(),
+            type=ContentType.text,
+        )
+        return user_input

wxo_agentic_evaluation/main.py

@@ -0,0 +1,231 @@
+from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.llm_user import LLMUser
+from wxo_agentic_evaluation.prompt.template_render import LlamaUserTemplateRenderer
+from wxo_agentic_evaluation.inference_backend import (
+    EvaluationController,
+    get_wxo_inference_backend,
+)
+from wxo_agentic_evaluation.evaluation_package import EvaluationPackage
+from wxo_agentic_evaluation.type import EvaluationData
+
+from wxo_agentic_evaluation.arg_configs import TestConfig
+from wxo_agentic_evaluation.utils.utils import (
+    create_table,
+    create_average_row,
+    SummaryPanel,
+)
+from wxo_agentic_evaluation.utils import json_dump
+from wxo_agentic_evaluation.metrics.metrics import KnowledgeBaseMetricSummary
+import os
+import json
+
+import yaml
+import dataclasses
+import glob
+import rich
+import csv
+from rich.progress import Progress
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+from jsonargparse import CLI
+
+
+def process_test_case(task_n, test_case, config, inference_backend, llm_user):
+    summary_results_for_path = []
+    tc_name = os.path.basename(test_case).replace(".json", "")
+    with open(test_case, "r") as f:
+        test_case: EvaluationData = EvaluationData.model_validate(json.load(f))
+
+    evaluation_controller = EvaluationController(
+        wxo_inference_backend=inference_backend, llm_user=llm_user, config=config
+    )
+    rich.print(f"[bold magenta]Running test case: {tc_name}[/bold magenta]")
+    history, call_tracker, conversational_search_data = evaluation_controller.run(
+        task_n,
+        test_case.story,
+        agent_name=test_case.agent,
+        starting_user_input=test_case.starting_sentence,
+    )
+    result = list()
+    for message in history:
+        result.append(message.model_dump())
+
+    json_dump(
+        os.path.join(config.output_dir, "messages", tc_name + ".messages.json"), result
+    )
+
+    if len(conversational_search_data) > 0:
+        fn = tc_name + ".retrieval_context.json"
+        out_folder = Path(config.output_dir) / "knowledge_base_metrics"
+        out_folder.mkdir(exist_ok=True)
+        rc = [context.model_dump() for context in conversational_search_data]
+        json_dump(out_folder / fn, rc)
+
+    # If data annotation run, skip summary generation
+    if config.data_annotation_run:
+        return summary_results_for_path  # empty result set, skip summary
+
+    evaluation_package = EvaluationPackage(
+        test_case_name=tc_name,
+        messages=history,
+        ground_truth=test_case,
+        conversational_search_data=conversational_search_data,
+    )
+    (
+        tool_call_metrics,
+        keyword_semantic_matches,
+        knowledge_base_metrics,
+        messages_with_reason,
+        metrics,
+    ) = evaluation_package.generate_summary()
+    temp = []
+    for message in messages_with_reason:
+        temp.append(message.model_dump())
+    json_dump(
+        os.path.join(config.output_dir, "messages", tc_name + ".messages.analyze.json"),
+        temp,
+    )
+
+    json_dump(
+        os.path.join(config.output_dir, "messages", tc_name + ".metrics.json"),
+        metrics.model_dump(),
+    )
+
+    tool_call_metrics["Avg Resp Time (Secs)"] = (
+        sum(call_tracker.generic) + sum(call_tracker.tool_call)
+    ) / (len(call_tracker.generic) + len(call_tracker.tool_call))
+    tool_call_metrics["Avg Resp Time (Secs)"] = round(
+        tool_call_metrics["Avg Resp Time (Secs)"], 2
+    )
+
+    summary_results_for_path.append((tool_call_metrics, knowledge_base_metrics))
+
+    return summary_results_for_path
+
+
+def main(config: TestConfig):
+    executor = ThreadPoolExecutor(max_workers=config.num_workers)
+    wai_client = WatsonXProvider(model_id=config.llm_user_config.model_id)
+    inference_backend = get_wxo_inference_backend(
+        config.auth_config.url, config.auth_config.tenant_name, config.auth_config.token
+    )
+
+    llm_user = LLMUser(
+        wai_client=wai_client,
+        template=LlamaUserTemplateRenderer(config.llm_user_config.prompt_config),
+        user_response_style=config.llm_user_config.user_response_style,
+    )
+
+    print(f"Running evaluation with tenant {config.auth_config.tenant_name}")
+
+    results_list = []
+
+    knowledge_base_output_folder = Path(config.output_dir) / "knowledge_base_metrics"
+    knowledge_base_output_folder.mkdir(exist_ok=True, parents=True)
+    detailed_rag_output_file = (
+        knowledge_base_output_folder / "knowledge_base_detailed_metrics.json"
+    )
+    summary_rag_output_file = (
+        Path(config.output_dir) / "knowledge_base_summary_metrics.json"
+    )
+
+    os.makedirs(os.path.join(config.output_dir, "messages"), exist_ok=True)
+    available_res = set()
+    if config.skip_available_results:
+        available_res = set(
+            [
+                os.path.basename(f).replace(".messages", "")
+                for f in glob.glob(
+                    os.path.join(config.output_dir, "messages", "*.messages.json")
+                )
+            ]
+        )
+
+    test_cases = []
+    for test_path in config.test_paths:
+        if os.path.isdir(test_path):
+            test_path = os.path.join(test_path, "*.json")
+        test_cases.extend(sorted(glob.glob(test_path)))
+    
+    futures = []
+    task_n = 0
+    for test_case in test_cases:
+        if not test_case.endswith(".json") or test_case.endswith("agent.json"):
+            continue
+        if config.skip_available_results:
+            if test_case in available_res:
+                print(f"Skipping test case {test_case} as results already exist.")
+                continue
+
+        future = executor.submit(
+            process_test_case,
+            task_n,
+            test_case,
+            config,
+            inference_backend,
+            llm_user,
+        )
+
+        futures.append((test_case, future))
+        task_n += 1
+
+    if futures:
+        with Progress() as progress:
+            task1 = progress.add_task(
+                f"[purple]Evaluating {len(futures)} tasks...", total=len(futures)
+            )
+            for test_case, future in futures:
+                try:
+                    results_list.extend(future.result())
+                except Exception as e:
+                    rich.print(f"test case {test_case} fails with {e}")
+                finally:
+                    progress.update(task1, advance=1)
+
+    tool_call_metrics = [metric[0] for metric in results_list]
+    knowledge_base_metrics = [metric[1] for metric in results_list]
+
+    rag_metric_summary = KnowledgeBaseMetricSummary(
+        knowledge_base_metrics=knowledge_base_metrics
+    )
+    SummaryPanel(rag_metric_summary).print()
+
+    with open(detailed_rag_output_file, "w+", encoding="utf-8") as f:
+        json.dump(rag_metric_summary.model_dump(by_alias=True)["detailed"], f, indent=4)
+
+    with open(summary_rag_output_file, "w+", encoding="utf-8") as f:
+        json.dump(rag_metric_summary.model_dump(by_alias=True)["summary"], f, indent=4)
+
+    if len(tool_call_metrics) > 0:
+        # remove the average row if exist
+        tool_call_metrics = [
+            row for row in tool_call_metrics if row["Dataset"] != "Summary (Average)"
+        ]
+        avg_row = create_average_row(tool_call_metrics)
+        tool_call_metrics.append(avg_row)
+    
+    tool_call_table = create_table(tool_call_metrics)
+    
+    if tool_call_table:
+        tool_call_table.print()
+
+    if len(tool_call_metrics) > 0:
+        output_file = os.path.join(config.output_dir, "summary_metrics.csv")
+        header = list(tool_call_metrics[0].keys())
+
+        with open(output_file, "w") as file:
+            csv_writer = csv.writer(file)
+            csv_writer.writerow(header)
+            for entry in tool_call_metrics:
+                csv_writer.writerow([entry[name] for name in header])
+
+    with open(
+        os.path.join(config.output_dir, "config.yml"), "w", encoding="utf-8"
+    ) as f:
+        yaml.safe_dump(dataclasses.asdict(config), f)
+
+    print(f"Results saved to {config.output_dir}")
+
+
+if __name__ == "__main__":
+    main(CLI(TestConfig, as_positional=False))

wxo_agentic_evaluation/metrics/llm_as_judge.py

@@ -0,0 +1,46 @@
+from pydantic import BaseModel, computed_field
+
+from abc import abstractmethod
+from functools import cached_property
+
+
+class BaseLLMJudgeMetric(BaseModel):
+    @abstractmethod
+    def table(self):
+        raise NotImplementedError("Method is not implemented")
+
+
+class Faithfulness(BaseLLMJudgeMetric):
+    faithfulness_score: str | float
+    evidence: list
+    reason: str
+
+    def table(self):
+        return {
+            "evidence": ",".join(self.evidence),
+            "reason": self.reason,
+            "faithfulness_score": str(self.faithfulness_score),
+        }
+
+
+class AnswerRelevancy(BaseLLMJudgeMetric):
+    answer_relevancy: list
+
+    @computed_field
+    @cached_property
+    def answer_relevancy_score(self) -> str:
+        total_num_statements = len(self.answer_relevancy)
+        yes_statements = list(
+            filter(
+                lambda item: item["relevant"].lower().strip() == "yes",
+                self.answer_relevancy,
+            )
+        )
+
+        return str(round(len(yes_statements) / total_num_statements, 3))
+
+    def table(self):
+        return {
+            "answer_relevancy": self.answer_relevancy,
+            "answer_relevancy_score": self.answer_relevancy_score,
+        }

wxo_agentic_evaluation/metrics/metrics.py

@@ -0,0 +1,101 @@
+from typing import List, Mapping, Any
+
+import numpy as np
+from pydantic import BaseModel, computed_field
+
+from wxo_agentic_evaluation.metrics.llm_as_judge import Faithfulness, AnswerRelevancy
+from wxo_agentic_evaluation.type import ConversationalConfidenceThresholdScore
+
+
+class KnowledgeBaseMetrics(BaseModel):
+    dataset_name: str = None
+    knowledge_base_name: str = (
+        None  # in the message response body it is represented as "tool_name"
+    )
+    tool_call_id: str = None
+    faithfulness: Faithfulness = None
+    answer_relevancy: AnswerRelevancy = None
+    confidence_scores: ConversationalConfidenceThresholdScore = None
+
+
+class KnowledgeBaseMetricSummary(BaseModel):
+    knowledge_base_metrics: List[List[KnowledgeBaseMetrics]]
+
+    @computed_field(alias="detailed")
+    @property
+    def groupby_dataset(self) -> Mapping[str, Any]:
+        groupby = {}
+        for metric in self.knowledge_base_metrics:
+            for row in metric:
+                name = row.dataset_name
+                tool_call_id = row.tool_call_id
+                knowledge_base_name = row.knowledge_base_name
+                faithfulness = row.faithfulness
+                confidence_scores = row.confidence_scores
+                answer_relevancy = row.answer_relevancy
+
+                if name not in groupby:
+                    groupby[name] = {
+                        "knowledge_base_name": [knowledge_base_name],
+                        "faithfulness": [faithfulness],
+                        "confidence_scores": [confidence_scores],
+                        "tool_call_id": [tool_call_id],
+                        "answer_relevancy": [answer_relevancy],
+                        "number_of_calls": 1,
+                    }
+                else:
+                    values = groupby[name]
+                    values.get("knowledge_base_name").append(knowledge_base_name)
+                    values.get("faithfulness").append(faithfulness)
+                    values.get("answer_relevancy").append(answer_relevancy)
+                    values.get("confidence_scores").append(confidence_scores)
+                    values.get("tool_call_id").append(tool_call_id)
+                    values["number_of_calls"] += 1
+                    groupby[name] = values
+
+        return groupby
+
+    @computed_field(alias="summary")
+    @property
+    def average(self) -> Mapping[str, Any]:
+        summary = {}
+        for dataset, metric in self.groupby_dataset.items():
+            average_metric = {}
+            average_metric["average_faithfulness"] = np.average(
+                [
+                    float(faithfulness.faithfulness_score)
+                    for faithfulness in metric["faithfulness"]
+                ]
+            )
+            average_metric["average_response_confidence"] = np.average(
+                [
+                    float(confidence_score.response_confidence)
+                    for confidence_score in metric["confidence_scores"]
+                ]
+            )
+            average_metric["average_retrieval_confidence"] = np.average(
+                [
+                    float(confidence_score.retrieval_confidence)
+                    for confidence_score in metric["confidence_scores"]
+                ]
+            )
+            average_metric["average_answer_relevancy"] = np.average(
+                [
+                    float(answer_relevancy.answer_relevancy_score)
+                    for answer_relevancy in metric["answer_relevancy"]
+                ]
+            )
+            average_metric["number_of_calls"] = metric["number_of_calls"]
+            average_metric["knowledge_bases_called"] = ", ".join(
+                set(metric["knowledge_base_name"])
+            )
+            summary[dataset] = average_metric
+
+        return summary
+
+
+class KeywordSemanticSearchMetric(BaseModel):
+    keyword_match: bool
+    semantic_match: bool
+    message: str
+    goal_detail: str

wxo_agentic_evaluation/prompt/answer_relevancy_prompt.jinja2

@@ -0,0 +1,120 @@
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+
+You are an evaluation agent that judges how relevant statements within a provided answer are in addressing the provided question. 
+In addition to the provided question, you *might* be given previous interactions between a user and an assistant that led up to the given question. 
+
+Your evaluation task can be broken down into the following four steps:
+1. Contextualization: Contextualize the question with the prior history if it's provided.
+2. Extraction: Extract the statements from the answer.
+3. Relevant: Determine if the statement is relevant in addressing the question.
+4. Reasoning: Provide a reason why a statement within an answer was relevant or not.
+
+Let's breakdown each of these stages into more detail.
+
+### Contextualization
+Like mentioned, you *might* be provided the prior interactions between a user and an assistant. 
+If the prior interactions between the user and assistant is provided, it should serve to add more detail and illustrate what the question is about if there is some ambiguity.
+If provided, the prior interactions look like this:
+
+#### Example 1 of Prior Interaction
+User: sample user text
+Assistant sample response
+User: sample user text 2
+Assistant sample response 2
+
+#### Example 2 of Prior Interaction
+User: sample user text
+Assistant sample response
+
+-- End of Examples --
+
+### Extraction Stage
+An answer can have many statements within in it. A statement that is ambigious or unclear in either wording or meaning also counts as a statement.
+Provided an answer, extract the statements from the answer into a list.
+
+
+### Relevance Stage
+For each statement, determine if the statement is relevant to answering the question. To determine if the statement is relevant ask yourself these questions:
+1. Does the statement contribute to answering the question?
+2. Does the statement match the question's topic?
+
+For each statement, output either "yes", "no", or "not sure" when determining the relevancy of the statement.
+
+### Reasoning Stage
+For each statement that recieved a "no" or "not sure" when determining relevancy, output a concise statement explaining your reasoning. Be concise in your reasoning!
+For statements that recieved a "yes", you can have an empty string for this field.
+
+----
+
+Now let's talk about the output format.
+
+## Output 
+Respond in a JSON formatted list. Each item in the list should have the following fields:
+- statement: this field contains *a* extracted statement.
+- relevant: this field contains your determination on if the statement is relevant to the question. Remember, valid fields for this are "yes", "no", or "not sure".
+- reason: this field contains your reasoning. Remember, provide reasoning ONLY if a statement recieved "no" or "not sure" for its relevancy. Otherwise this field is an empty string.
+
+These are some examples of valid JSON output
+#### Example 1
+[
+    {
+        "statement": "example statement",
+        "relevant": "yes",
+        "reason": ""
+
+    },
+    {
+        "statement": "another example statement" ,
+        "relevant": "no",
+        "reason": "placeholder text for reason"
+    },
+    {
+        "statement": "one more statement" ,
+        "relevant": "not sure",
+        "reason": "placeholder text for reasoning"
+    }
+
+]
+
+#### Example 2
+[
+    {
+        "statement": "example statement",
+        "relevant": "yes",
+        "reason": ""
+
+    }
+
+]
+
+-- End of examples --
+
+DO NOT PROVIDE ADDITIONAL COMMENTARY, EXPLANATIONS, OR OUTPUTS other than what is explicitly required above.
+
+<|eot_id|>
+
+---
+
+<|start_header_id|>user<|end_header_id|>
+
+
+Evaluate the following answer against the question.
+
+Answer:
+{{ answer }}
+
+Question:
+{{ question }}
+
+Previous Interactions Between a User and an Assistant:
+{{ context }}
+
+<|eot_id|>
+
+
+<|start_header_id|>assistant<|end_header_id|>
+
+
+
+
+

wxo_agentic_evaluation/prompt/batch_testcase_prompt.jinja2

@@ -0,0 +1,51 @@
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+You are a test scenario generator.
+
+Agent name: {{ agent_name }}
+Agent capabilities:
+{{ tool_blocks }}
+
+Use the given user story to generate {{ num_variants }} test cases. DO NOT rewrite the story — use it exactly as provided.
+You must generate at least one **SIMPLE** and one **COMPLEX** test case:
+
+- In the SIMPLE version:
+  - The `starting_sentence` must include **all** input values (e.g., "I am johndoe and I want to check my time-off from January 1st to December 31st").
+
+- In the COMPLEX version:
+  - The `starting_sentence` should **not include any input values**, only the intent (e.g., "I'd like to check my time-off schedule").
+
+The `starting_sentence` values must be distinct in tone and structure, while preserving the original intent and using the story’s exact vocabulary for roles, entities, and other key concepts.
+
+In the `goal_details` list, the **only** step with `"type": "text"` must be the final one, and it must have `"name": "summarize"`. No other `goal_detail` should have `"type": "text"`.
+
+Story:
+"{{ story }}"
+
+Generate {{ num_variants }} test cases with:
+- agent (must match agent name)
+- story (use as-is)
+- starting_sentence 
+- goals (tool dependency structure)
+- goal_details (ordered tool invocations, ending with a 'summarize' text step)
+
+Use only these tool input examples:
+{{ tool_inputs_str }}
+
+Please use the following format for your response:
+{% raw %}
+[
+  {{test_case_1}},
+  {{test_case_2}},
+  {{test_case_3}}
+]
+{% endraw %}
+
+The final summarize step must use actual values from tool outputs (no placeholders).
+
+Here is one complete example to follow:
+{{ example_str }}
+<|eot_id|>
+<|start_header_id|>user<|end_header_id|>
+Story: "{{ story }}"
+<|eot_id|>
+<|start_header_id|>assistant<|end_header_id|>

wxo_agentic_evaluation/prompt/examples/data_simple.json

@@ -0,0 +1,93 @@
+{
+  "agent": "hr_agent",
+  "story": "My username is nwaters. I want to find out your time-off schedule from: 2025-01-01 to: 2025-12-31 of all my direct reports.",
+  "starting_sentence": "I'd like to check my direct reports' time-off schedule.",
+  "goals": {
+    "fetch_assignment_id-1": [
+      "list_direct_reports-1"
+    ],
+    "list_direct_reports-1": [
+      "fetch_assignment_id-2",
+      "fetch_assignment_id-3"
+    ],
+    "fetch_assignment_id-2": [
+      "retrieve_timeoff_schedule-1"
+    ],
+    "fetch_assignment_id-3": [
+      "retrieve_timeoff_schedulet-2"
+    ],
+    "retrieve_timeoff_schedule-1": [
+      "summarize"
+    ],
+    "retrieve_timeoff_schedule-2": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "fetch_assignment_id-1",
+      "tool_name": "fetch_assignment_id",
+      "args": {
+        "username": "nwaters"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "list_direct_reports-1",
+      "tool_name": "list_direct_reports",
+      "args": {
+        "manager_assignment_id": "15778303"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "fetch_assignment_id-2",
+      "tool_name": "fetch_assignment_id",
+      "args": {
+        "username": "johndoe"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "fetch_assignment_id-3",
+      "tool_name": "fetch_assignment_id",
+      "args": {
+        "username": "nken"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "retrieve_timeoff_schedule-1",
+      "tool_name": "retrieve_timeoff_schedule",
+      "args": {
+        "assignment_id": "15338303",
+        "start_date": "2025-01-01",
+        "end_date": "2025-12-31"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "retrieve_timeoff_schedule-2",
+      "tool_name": "retrieve_timeoff_schedule",
+      "args": {
+        "assignment_id": "15338304",
+        "start_date": "2025-01-01",
+        "end_date": "2025-12-31"
+      }
+    },
+    {
+      "type": "text",
+      "name": "summarize",
+      "response": "Your direct reports' time-off schedules for 2025-01-01 to 2025-12-31 are: johndoe - 2025-04-11, 2025-03-11, 2025-01-01; nken - 2025-01-15, 2025-02-20",
+      "keywords": [
+        "2025-04-11",
+        "2025-03-11",
+        "2025-01-01",
+        "2025-01-15",
+        "2025-02-20"
+      ]
+    }
+  ],
+  "mine_fields": []
+}