ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/langfuse_evaluation_package.py

@@ -0,0 +1,192 @@
+from collections import defaultdict
+from typing import Callable, List
+
+import rich
+from langfuse import get_client
+from langfuse.experiment import ExperimentResult
+
+from wxo_agentic_evaluation.langfuse_collection import LangfuseCollection
+from wxo_agentic_evaluation.metrics import Evaluation
+from wxo_agentic_evaluation.metrics.dummy_metric import DummyMetric
+from wxo_agentic_evaluation.metrics.journey_success import JourneySuccessMetric
+from wxo_agentic_evaluation.metrics.metrics import LangfuseMetric
+from wxo_agentic_evaluation.metrics.tool_calling import ToolCalling
+from wxo_agentic_evaluation.otel_parser import parser as otel_parser
+from wxo_agentic_evaluation.otel_parser.parser_types import (
+    Message as OtelMessage,
+)
+from wxo_agentic_evaluation.type import (
+    ExperimentResult,
+    LangfuseDatasetModel,
+    _convert_to_langfuse_format,
+)
+
+from wxo_agentic_evaluation.extractors import ExtractLabeledMessages
+
+LANGFUSE_CLIENT = get_client()
+
+
+def upload(name, session_id, value, data_type, metadata):
+    try:
+        LANGFUSE_CLIENT.create_score(
+            name=name,
+            session_id=session_id,
+            value=value,
+            data_type=data_type,
+            metadata=metadata,
+        )
+    except Exception as e:
+        rich.print(
+            f"[r] Uploading {name} with value {value} failed with exception {e}"
+        )
+
+
+def sample_aggregator(session_results: List[List[Evaluation]]):
+    metric_names = [
+        "journey_success",
+        "total_tool_calls",
+        "correct_tool_calls",
+        "expected_tool_calls",
+        "tool_calls_with_incorrect_parameter",
+        "tool_call_recall",
+        "tool_call_precision",
+    ]
+    group_metrics = defaultdict(list)
+
+    for result in session_results:
+        for metric in result:
+            if metric["eval_name"] in metric_names:
+                group_metrics[metric["eval_name"]].append(
+                    {"value": metric["value"], "metadata": metric["metadata"]}
+                )
+
+    average_metric = []
+    for metric_name, values in group_metrics.items():
+        aggr = []
+        for value in values:
+            aggr.append(value.get("value"))
+
+        metric_value = LangfuseMetric(
+            eval_name=f"Average_{metric_name}",
+            value=round(sum(aggr) / len(aggr), 2),
+            metadata=values[0]["metadata"],
+        )
+        average_metric.append(metric_value)
+
+    return average_metric
+
+
+class EvaluationRunner:
+    def __init__(
+        self,
+        evaluation_name: str,
+        run_name: str,
+        session_ids: List[str],
+        collection: LangfuseCollection,
+        metrics: List[Evaluation],
+        aggregator: Callable,
+    ):
+        self.evaluation_name = evaluation_name
+        self.run_name = run_name
+
+        self.experiment_id = f"{self.evaluation_name}.{self.run_name}"
+
+        self.collection = collection
+        langfuse_dataset = LANGFUSE_CLIENT.get_dataset(self.collection.name)
+        self.test_cases: List[LangfuseDatasetModel] = []
+        for item in langfuse_dataset.items:
+            data_model = _convert_to_langfuse_format(item)
+            self.test_cases.append(data_model)
+
+        self.session_ids = session_ids
+        self.messages = [otel_parser.parse_session(id) for id in self.session_ids]
+
+        assert (
+            len(self.session_ids) == len(self.messages) == len(self.test_cases)
+        )
+
+        self.metrics = metrics
+        self.aggregator = aggregator
+
+    def evaluate(self):
+        metadata = {"experiment_id": self.experiment_id}
+
+        total_metrics = []
+        for idx, test_case in enumerate(self.test_cases):
+            metric_results = []
+            messages = self.messages[idx]
+            extracted_context = ExtractLabeledMessages.extract(messages, test_case)
+            for metric in self.metrics:
+                result = metric.evaluate(
+                    messages=messages,
+                    ground_truth=test_case,
+                    extracted_context=extracted_context,
+                    metadata=metadata
+                )
+                if isinstance(result, list):
+                    metric_results.extend([r.model_dump() for r in result])
+                    for r in result:
+                        upload(
+                            name=r.eval_name,
+                            session_id=self.session_ids[idx],
+                            value=r.value,
+                            data_type=r.data_type,
+                            metadata=r.metadata,
+                        )
+                else:
+                    metric_results.append(result.model_dump())
+                    upload(
+                        name=result.eval_name,
+                        session_id=self.session_ids[idx],
+                        value=result.value,
+                        data_type=result.data_type,
+                        metadata=result.metadata,
+                    )
+            total_metrics.append(metric_results)
+
+        aggregate_metrics = self.aggregator(total_metrics)
+        for metric in aggregate_metrics:
+            try:
+                LANGFUSE_CLIENT.create_score(
+                    name=metric.eval_name,
+                    value=metric.value,
+                    metadata=metric.metadata,
+                    data_type="NUMERIC",
+                    dataset_run_id=metric.metadata["experiment_id"],
+                )
+            except Exception as e:
+                rich.print(
+                    f"[r] Uploading {metric.name} with value {metric.value} failed with exception {e}"
+                )
+
+        return ExperimentResult(
+            experiment_name=self.evaluation_name,
+            run_id=self.run_name,
+            experiment_id=self.experiment_id,
+            metrics=total_metrics,
+            session_ids=self.session_ids
+        )
+
+
+if __name__ == "__main__":
+    collection_name = "HR AGENT DEMO"
+    langfuse_collection = LangfuseCollection(name=collection_name)
+    journey_sucess_metric = JourneySuccessMetric()
+    tool_calling = ToolCalling()
+
+    SESSION_ID = "agent-demo-session-id-NEW"
+
+    run = EvaluationRunner(
+        evaluation_name="sample_evaluation",
+        run_name="1",
+        session_ids=[
+            "agent-demo-session-id-NEW-0",
+            "agent-demo-session-id-NEW-1",
+        ],
+        collection=langfuse_collection,
+        metrics=[journey_sucess_metric, tool_calling],
+        aggregator=sample_aggregator,
+    )
+
+    experiment_results = run.evaluate()
+    rich.print(experiment_results.model_dump())

wxo_agentic_evaluation/llm_matching.py

@@ -1,9 +1,22 @@
-from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+"""
+LLM Matching Module with Cosine Similarity Support
+
+This module provides functionality for matching text using:
+1. LLM-based matching (using a language model to determine semantic equivalence)
+2. Embedding-based matching (using cosine similarity between text embeddings)
+"""
+
+import math
+from typing import List
+
+from fuzzywuzzy import fuzz
+
 from wxo_agentic_evaluation.prompt.template_render import (
     KeywordMatchingTemplateRenderer,
     SemanticMatchingTemplateRenderer,
 )
-from typing import List
+from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+from wxo_agentic_evaluation.utils.utils import safe_divide
 
 
 class LLMMatcher:
@@ -12,10 +25,18 @@ class LLMMatcher:
         llm_client: Provider,
         keyword_template: KeywordMatchingTemplateRenderer,
         semantic_template: SemanticMatchingTemplateRenderer,
+        use_llm_for_semantic: bool = True,
+        embedding_model_id: str = "sentence-transformers/all-minilm-l6-v2",
+        similarity_threshold: float = 0.8,
+        enable_fuzzy_matching: bool = False,
     ):
         self.llm_client = llm_client
         self.keyword_template = keyword_template
         self.semantic_template = semantic_template
+        self.embedding_model_id = embedding_model_id
+        self.use_llm_for_semantic = use_llm_for_semantic
+        self.similarity_threshold = similarity_threshold
+        self.enable_fuzzy_matching = enable_fuzzy_matching
 
     def keywords_match(self, response_text: str, keywords: List[str]) -> bool:
         if len(keywords) == 0:
@@ -26,14 +47,96 @@ class LLMMatcher:
         prompt = self.keyword_template.render(
             keywords_text=keywords_text, response_text=response_text
         )
-        output:str = self.llm_client.query(prompt)
+        output: str = self.llm_client.query(prompt)
         result = output.strip().lower()
         return result.startswith("true")
 
-    def semantic_match(self, prediction: str, ground_truth: str) -> bool:
+    def generate_embeddings(
+        self, prediction: str, ground_truth: str
+    ) -> List[List[float]]:
+
+        embeddings = self.llm_client.encode([prediction, ground_truth])
+
+        return embeddings
+
+    def compute_cosine_similarity(
+        self, vec1: List[float], vec2: List[float]
+    ) -> float:
+        """Calculate cosine similarity between two vectors using pure Python"""
+
+        # Manual dot product calculation
+        dot_product = sum(a * b for a, b in zip(vec1, vec2))
+
+        # Manual magnitude calculations
+        magnitude1 = math.sqrt(sum(a * a for a in vec1))
+        magnitude2 = math.sqrt(sum(b * b for b in vec2))
+
+        return safe_divide(dot_product, (magnitude1 * magnitude2))
+
+    def cosine_similarity_semantic_match(
+        self, prediction: str, ground_truth: str
+    ) -> bool:
+        embeddings = self.generate_embeddings(prediction, ground_truth)
+        cosine_similarity = self.compute_cosine_similarity(
+            embeddings[0], embeddings[1]
+        )
+
+        return cosine_similarity >= self.similarity_threshold
+
+    def llm_semantic_match(
+        self, context, prediction: str, ground_truth: str
+    ) -> bool:
+        """Performs semantic matching for the agent's final response and the expected response using the starting sentence of the conversation as the context
+
+        Args:
+            context: The starting sentence of the conversation. TODO can also consider using the LLM user's story
+            prediction: the predicted string
+            ground_truth: the expected string
+
+        Returns:
+            a boolean indicating if the sentences match.
+        """
+
         prompt = self.semantic_template.render(
-            expected_text=ground_truth, actual_text=prediction
+            context=context, expected_text=ground_truth, actual_text=prediction
         )
         output: str = self.llm_client.query(prompt)
         result = output.strip().lower()
+
         return result.startswith("true")
+
+    def fuzzywuzzy_semantic_match(
+        self, prediction: str, ground_truth: str
+    ) -> bool:
+
+        similarity_score = fuzz.WRatio(prediction, ground_truth)
+
+        return similarity_score > self.similarity_threshold
+
+    def semantic_match(
+        self,
+        context: str,
+        prediction: str,
+        ground_truth: str,
+        enable_fuzzy_matching: bool = False,
+    ) -> bool:
+        ## TODO arjun-gupta1 10/06/2025: revist retry with exponential backoff. Opted for direct fallback to cosine similarity to avoid latency for now.
+        try:
+            return self.llm_semantic_match(context, prediction, ground_truth)
+        except Exception as e:
+            print(f"LLM semantic match failed: {e}")
+
+        if enable_fuzzy_matching:
+            print("falling back to fuzzy matching")
+            # Fallback to cosine similarity if LLM matching is not used or failed
+            try:
+                return self.cosine_similarity_semantic_match(
+                    prediction, ground_truth
+                )
+            except Exception as e:
+                print(
+                    f"Cosine similarity failed: {e}. Falling back to fuzzywuzzy."
+                )
+
+            # Final fallback to fuzzywuzzy
+            return self.fuzzywuzzy_semantic_match(prediction, ground_truth)

wxo_agentic_evaluation/llm_rag_eval.py

@@ -1,12 +1,15 @@
-from typing import List
 import json
+from typing import List
 
-from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+from wxo_agentic_evaluation.metrics.llm_as_judge import (
+    AnswerRelevancy,
+    Faithfulness,
+)
 from wxo_agentic_evaluation.prompt.template_render import (
-    FaithfulnessTemplateRenderer,
     AnswerRelevancyTemplateRenderer,
+    FaithfulnessTemplateRenderer,
 )
-from wxo_agentic_evaluation.metrics.llm_as_judge import Faithfulness, AnswerRelevancy
+from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
 
 
 class LLMJudge:

wxo_agentic_evaluation/llm_safety_eval.py

@@ -0,0 +1,64 @@
+import json
+import re
+
+from wxo_agentic_evaluation.metrics.llm_as_judge import (
+    AnswerDerailment,
+    AnswerUnsafeTopic,
+)
+from wxo_agentic_evaluation.prompt.template_render import (
+    DerailmentTemplateRenderer,
+    UnsafeTopicTemplateRenderer,
+)
+from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+
+
+class LLMSafetyJudge:
+    def __init__(
+        self,
+        llm_client: Provider,
+        answer_derailment: DerailmentTemplateRenderer,
+        answer_unsafe_topic: UnsafeTopicTemplateRenderer,
+    ):
+        self.llm_client = llm_client
+        self.answer_derailment_template = answer_derailment
+        self.answer_unsafe_topic_template = answer_unsafe_topic
+
+    def judge_derailment_in_answer(
+        self, question: str, instructions: str, answer: str
+    ) -> AnswerDerailment:
+        prompt = self.answer_derailment_template.render(
+            question=question, instructions=instructions, answer=answer
+        )
+        output = self.llm_client.query(prompt)
+        result = output.strip().lower()
+        try:
+            json_str = re.search(r"\{.*\}", result, re.DOTALL).group(0)
+            thejson = json.loads(json_str)
+            answer_derailment = AnswerDerailment.model_validate(thejson)
+        except Exception:
+            answer_derailment = AnswerDerailment(
+                in_scope="not sure", statement=answer, reason=result
+            )
+        return answer_derailment
+
+    def judge_unsafe_topic_in_answer(
+        self, question: str, instructions: str, answer: str
+    ) -> AnswerUnsafeTopic:
+        """
+        Use LLM to judge if the answer contains unsafe topics.
+        """
+        prompt = self.answer_unsafe_topic_template.render(
+            question=question, instructions=instructions, answer=answer
+        )
+        output = self.llm_client.query(prompt)
+        result = output.strip().lower()
+        try:
+            json_str = re.search(r"\{.*\}", result, re.DOTALL).group(0)
+            thejson = json.loads(json_str)
+            answer_unsafe = AnswerUnsafeTopic.model_validate(thejson)
+        except Exception:
+            answer_unsafe = AnswerUnsafeTopic(
+                is_safe="not sure", statement=answer, reason=result
+            )
+
+        return answer_unsafe

wxo_agentic_evaluation/llm_user.py

@@ -1,14 +1,16 @@
 from typing import List, TypeVar
-from wxo_agentic_evaluation.type import Message, ContentType
-from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+
+from wxo_agentic_evaluation.base_user import BaseUserSimulator
 from wxo_agentic_evaluation.prompt.template_render import JinjaTemplateRenderer
+from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+from wxo_agentic_evaluation.type import ContentType, Message
 
 T = TypeVar("T", bound=JinjaTemplateRenderer)
 
 
-class LLMUser:
+class LLMUser(BaseUserSimulator):
     def __init__(
-        self, wai_client: Provider, template: T, user_response_style: List[str]
+        self, wai_client: Provider, template: T, user_response_style: List[str] | None = None
     ):
         self.wai_client = wai_client
         self.prompt_template = template
@@ -17,8 +19,11 @@ class LLMUser:
         )
 
     def generate_user_input(
-        self, user_story, conversation_history: List[Message]
-    ) -> Message | None:
+        self,
+        user_story,
+        conversation_history: List[Message],
+        attack_instructions: str | None = None,
+    ) -> Message:
         # the tool response is already summarized, we don't need that to take over the chat history context window
         prompt_input = self.prompt_template.render(
             conversation_history=[
@@ -28,6 +33,7 @@ class LLMUser:
             ],
             user_story=user_story,
             user_response_style=self.user_response_style,
+            attack_instructions=attack_instructions,
         )
         user_input = self.wai_client.query(prompt_input)
         user_input = Message(

wxo_agentic_evaluation/llm_user_v2.py

@@ -0,0 +1,114 @@
+from typing import List
+
+from wxo_agentic_evaluation.base_user import BaseUserSimulator
+from wxo_agentic_evaluation.prompt.template_render import UserTemplateRenderer
+from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+from wxo_agentic_evaluation.type import ContentType, Message
+
+
+class LLMUserV2(BaseUserSimulator):
+    def __init__(
+        self,
+        llm_client: Provider,
+        user_prompt_path: str,
+    ):
+        self.llm_client = llm_client
+        self.user_prompt_path = user_prompt_path
+        self.prompt_template = UserTemplateRenderer(
+            template_path=user_prompt_path
+        )
+
+    def _get_system_prompt(
+        self, user_story: str, user_response_style: List[str] = None
+    ) -> Message:
+        # Get the user system prompt
+        prompt_messages = self.prompt_template.render(
+            user_story=user_story,
+            user_response_style=user_response_style,
+        )
+        return Message(**prompt_messages[0], type=ContentType.text)
+
+    def _get_message_dicts(self, messages: List[Message]) -> List[dict]:
+        # Convert messages to dictionary format for the llm client
+        return [message.model_dump() for message in messages]
+
+    def _filter_conversation_history(
+        self, conversation_history: List[Message]
+    ) -> List[Message]:
+        # Filter out the agent system prompt
+        return [
+            message
+            for message in conversation_history
+            if message.role != "system"
+        ]
+
+    def flip_message_roles(self, messages: List[Message]) -> List[Message]:
+        # We flip the roles of messages in conversation history to basically prompt the
+        # user simulator with the assistant message as the user input message
+        # This helps to get the llm to respond as a natural user with the given story.
+        new_messages = []
+        for message in messages:
+            if message.role == "user":
+                new_messages.append(
+                    Message(
+                        role="assistant",
+                        content=message.content,
+                        type=ContentType.text,
+                    )
+                )
+            else:
+                new_messages.append(
+                    Message(
+                        role="user",
+                        content=message.content,
+                        type=ContentType.text,
+                    )
+                )
+        return new_messages
+
+    def generate_user_input(
+        self,
+        user_story: str,
+        conversation_history: List[Message],
+        user_response_style: List[str] = None,
+        starting_user_input: Message = None,
+        **kwargs,
+    ) -> Message:
+        # Get the user system prompt
+        system_prompt = self._get_system_prompt(user_story, user_response_style)
+
+        conversation_history = self._filter_conversation_history(
+            conversation_history
+        )
+
+        ## Adding dummy message if not provided from the simulation side.
+        if len(conversation_history) == 0:
+            conversation_history.append(
+                Message(
+                    role="assistant",
+                    content="Hi! How can I help you today?",
+                    type=ContentType.text,
+                )
+            )
+
+        conversation_history = self.flip_message_roles(conversation_history)
+
+        # build the conversation history with the system prompt
+        messages = [system_prompt] + conversation_history
+
+        if starting_user_input is not None:
+            # If starting user input is provided, return it as is for the initial turn
+            return starting_user_input
+        else:
+
+            # Get response from LLM for simulation
+            response = self.llm_client.chat(
+                messages=self._get_message_dicts(messages)
+            )
+            response_message = Message(
+                role="user",
+                content=response.choices[0].message.content,
+                type=ContentType.text,
+            )
+
+            return response_message