ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/external_agent/external_validate.py

@@ -41,15 +41,15 @@ class ExternalAgentValidation:
         data = b""
         for chunk in resp:
             for line in chunk.splitlines(True):
-                if line.startswith(b"data:"):
-                    line = line.replace(b"data:", b"")
-                if line.strip() == b"[DONE]":
-                    return
+                if line.startswith(b"event:"):
+                    continue
                 data += line
                 if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")):
                     # NOTE: edge case, "data" can be sent in two different chunks
                     if data.startswith(b"data:"):
                         data = data.replace(b"data:", b"")
+                    if data.strip() == b"[DONE]":
+                        return
                     yield data
                     data = b""
         if data:
@@ -74,7 +74,9 @@ class ExternalAgentValidation:
         payload = {"stream": True}
         payload["messages"] = messages
         resp = requests.post(
-            url=self.service_url, headers=self.header, json=payload
+            url=self.service_url,
+            headers=self.header,
+            json=payload,
         )
         success, logged_events = self._validate_streaming_response(resp)
 

wxo_agentic_evaluation/external_agent/types.py

@@ -1,4 +1,4 @@
-from typing import Any, List, Literal, Mapping, Union
+from typing import Any, List, Literal, Mapping, Optional, Union
 
 from pydantic import BaseModel
 
@@ -46,7 +46,7 @@ class ThreadRunStepDeltaChoice(BaseModel):
 class BaseEventData(BaseModel):
     id: str
     object: str
-    thread_id: str
+    thread_id: Optional[str] = None
     model: str | None = None
     created: int | None = None
 
@@ -62,13 +62,7 @@ class ThreadRunStepDeltaData(BaseEventData):
 
 
 class UniversalData(BaseEventData):
-    object: Union[
-        Literal["thread.message.delta"],
-        Literal["thread.run.step.delta"],
-        Literal["thread.run.step.created"],
-        Literal["thread.run.step.completed"],
-    ]
-    choices: List[ThreadMessageDeltaChoice]
+    object: Optional[str]
     choices: List[Union[ThreadMessageDeltaChoice, dict]]
 
 

wxo_agentic_evaluation/extractors/__init__.py

@@ -0,0 +1,3 @@
+from wxo_agentic_evaluation.extractors.labeled_messages import (
+    ExtractLabeledMessages,
+)

wxo_agentic_evaluation/extractors/extractor_base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from wxo_agentic_evaluation.type import Message
+
+
+class Extractor(ABC):
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Unique name for the extractor."""
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def extract(
+        messages: list[Message],
+        **kwargs,
+    ) -> Any:
+        """Extract data from messages."""
+        raise NotImplementedError

wxo_agentic_evaluation/extractors/labeled_messages.py

@@ -0,0 +1,47 @@
+import json
+from typing import Any, List, Mapping
+
+from wxo_agentic_evaluation.extractors.extractor_base import Extractor
+from wxo_agentic_evaluation.type import ContentType, GoalDetail, Message
+
+
+class ExtractLabeledMessages(Extractor):
+    def name(self):
+        return "Labelled Messages"
+
+    def extract(
+        messages: List[Message],
+        ground_truth,
+        **kwargs,
+    ) -> Any:
+
+        tool_dictionary = (
+            {
+                goal_detail.name: goal_detail
+                for goal_detail in ground_truth.goal_details
+                if goal_detail.type == ContentType.tool_call
+            }
+            if ground_truth.goal_details
+            else {}
+        )
+        labeled_messages = {}
+        for idx, message in enumerate(messages):
+            # TODO: investigate this logic - `message` body might not be consistent across providers
+            if not (message.role == "assistant" and message.tool_calls):
+                continue
+            try:
+                msg_tool_call = message.tool_calls[0].function
+            except Exception:
+                # ignore malformed tool_call content
+                continue
+
+            matching_goal_details = [
+                gd
+                for gd in tool_dictionary.values()
+                if gd.tool_name == msg_tool_call.name
+            ]
+
+            if matching_goal_details:
+                labeled_messages[idx] = matching_goal_details
+
+        return {"labeled_messages": labeled_messages}

wxo_agentic_evaluation/hr_agent_langgraph.py

@@ -0,0 +1,68 @@
+from langchain.tools import tool
+from langchain.agents import create_agent
+from langchain_openai import ChatOpenAI
+
+import json
+from datetime import datetime
+
+@tool
+def get_assignment_id_hr_usecase(username: str) -> str:
+    """
+    get the assignment id from username
+    :param username: username of the employee
+    """
+    if username=="nwaters":
+        return "15778303"
+    if username=="johndoe":
+        return "15338303"
+    return "not found"
+
+def validate_datetime(date_text):
+    try:
+        format = "%Y-%m-%d"
+        datetime.strptime(date_text, format)
+        return True
+    except ValueError:
+        return False
+
+
+@tool
+def get_timeoff_schedule_hr_usecase(assignment_id: str, start_date: str, end_date: str) -> str:
+    """
+    get timeoff schedule for employee based on assignment id, start date and end date
+    :param assignment_id: assignment_id of the user
+    :param start_date: start date of the timeoff scheduel, in YYYY-MM-DD format
+    :param assignment_id: end date of the timeoff scheduel, in YYYY-MM-DD format
+    """
+
+    if not validate_datetime(start_date):
+        return f"Incorrect date format {start_date}, should be YYYY-MM-DD"
+    if not validate_datetime(end_date):
+        return f"Incorrect date format {end_date}, should be YYYY-MM-DD"
+    if assignment_id=="15338303":
+        return json.dumps(["20250411", "20250311", "20250101"])
+    if assignment_id=="15778303":
+        return json.dumps(["20250105"])
+    return []
+
+
+@tool
+def get_direct_reports_hr_usecase(username: str) -> str:
+    """
+    get direct reports for a given username
+    :param assignment_id: assignment_id of the user
+    """
+
+    return json.dumps(["nwaters", "johndoe"])
+
+
+llm = ChatOpenAI(model="gpt-4o-mini")
+tools = [get_assignment_id_hr_usecase, get_timeoff_schedule_hr_usecase, get_direct_reports_hr_usecase]
+system_prompt="""You are an HR Agent that can answer questions related to timeoff and holiday calendar. Use the tools provided to answer the user's question. If you do not have enough information to answer the question, say so.  If you need more information, ask follow up questions."""
+
+agent = create_agent(
+    tools=tools,
+    model=llm,
+    system_prompt=system_prompt
+)
+

wxo_agentic_evaluation/langfuse_collection.py

@@ -0,0 +1,60 @@
+import json
+from typing import List, Mapping, Union
+
+import rich
+
+from wxo_agentic_evaluation.type import (
+    LangfuseCollectionModel,
+    LangfuseDatasetModel,
+)
+
+
+class LangfuseCollection:
+    def __init__(self, name, description="", metadata: Mapping[str, str] = {}):
+        self.name = name
+        self.description = description
+        self.metadata = metadata
+
+    def upload(self, paths: Union[str, List[str]]):
+        from langfuse import get_client
+
+        langfuse_client = get_client()
+
+        datasets = []
+        if isinstance(paths, str):
+            paths = [paths]
+
+        for path in paths:
+            with open(path, encoding="utf-8") as f:
+                dataset = json.load(f)
+                dataset = LangfuseDatasetModel(
+                    starting_sentence=dataset.get("starting_sentence", ""),
+                    story=dataset.get("story", ""),
+                    goals=dataset.get("goals"),
+                    goal_details=dataset.get("goal_details"),
+                    agent=dataset.get("agent")
+                )
+                datasets.append(dataset)
+
+        collection = LangfuseCollectionModel(
+            collection_name=self.name,
+            collection_description=self.description,
+            datasets=datasets,
+            metadata=self.metadata,
+        )
+
+        rich.print(
+            f"[g] Uploading {len(collection.datasets)} datasets to '{collection.collection_name}'"
+        )
+        langfuse_client.create_dataset(
+            name=collection.collection_name,
+            description=collection.collection_description,
+            metadata=collection.metadata,
+        )
+
+        for dataset in collection.datasets:
+            langfuse_client.create_dataset_item(
+                dataset_name=collection.collection_name,
+                input=dataset.langfuse_input,
+                expected_output=dataset.langfuse_output,
+            )

wxo_agentic_evaluation/langfuse_evaluation_package.py

@@ -0,0 +1,192 @@
+from collections import defaultdict
+from typing import Callable, List
+
+import rich
+from langfuse import get_client
+from langfuse.experiment import ExperimentResult
+
+from wxo_agentic_evaluation.langfuse_collection import LangfuseCollection
+from wxo_agentic_evaluation.metrics import Evaluation
+from wxo_agentic_evaluation.metrics.dummy_metric import DummyMetric
+from wxo_agentic_evaluation.metrics.journey_success import JourneySuccessMetric
+from wxo_agentic_evaluation.metrics.metrics import LangfuseMetric
+from wxo_agentic_evaluation.metrics.tool_calling import ToolCalling
+from wxo_agentic_evaluation.otel_parser import parser as otel_parser
+from wxo_agentic_evaluation.otel_parser.parser_types import (
+    Message as OtelMessage,
+)
+from wxo_agentic_evaluation.type import (
+    ExperimentResult,
+    LangfuseDatasetModel,
+    _convert_to_langfuse_format,
+)
+
+from wxo_agentic_evaluation.extractors import ExtractLabeledMessages
+
+LANGFUSE_CLIENT = get_client()
+
+
+def upload(name, session_id, value, data_type, metadata):
+    try:
+        LANGFUSE_CLIENT.create_score(
+            name=name,
+            session_id=session_id,
+            value=value,
+            data_type=data_type,
+            metadata=metadata,
+        )
+    except Exception as e:
+        rich.print(
+            f"[r] Uploading {name} with value {value} failed with exception {e}"
+        )
+
+
+def sample_aggregator(session_results: List[List[Evaluation]]):
+    metric_names = [
+        "journey_success",
+        "total_tool_calls",
+        "correct_tool_calls",
+        "expected_tool_calls",
+        "tool_calls_with_incorrect_parameter",
+        "tool_call_recall",
+        "tool_call_precision",
+    ]
+    group_metrics = defaultdict(list)
+
+    for result in session_results:
+        for metric in result:
+            if metric["eval_name"] in metric_names:
+                group_metrics[metric["eval_name"]].append(
+                    {"value": metric["value"], "metadata": metric["metadata"]}
+                )
+
+    average_metric = []
+    for metric_name, values in group_metrics.items():
+        aggr = []
+        for value in values:
+            aggr.append(value.get("value"))
+
+        metric_value = LangfuseMetric(
+            eval_name=f"Average_{metric_name}",
+            value=round(sum(aggr) / len(aggr), 2),
+            metadata=values[0]["metadata"],
+        )
+        average_metric.append(metric_value)
+
+    return average_metric
+
+
+class EvaluationRunner:
+    def __init__(
+        self,
+        evaluation_name: str,
+        run_name: str,
+        session_ids: List[str],
+        collection: LangfuseCollection,
+        metrics: List[Evaluation],
+        aggregator: Callable,
+    ):
+        self.evaluation_name = evaluation_name
+        self.run_name = run_name
+
+        self.experiment_id = f"{self.evaluation_name}.{self.run_name}"
+
+        self.collection = collection
+        langfuse_dataset = LANGFUSE_CLIENT.get_dataset(self.collection.name)
+        self.test_cases: List[LangfuseDatasetModel] = []
+        for item in langfuse_dataset.items:
+            data_model = _convert_to_langfuse_format(item)
+            self.test_cases.append(data_model)
+
+        self.session_ids = session_ids
+        self.messages = [otel_parser.parse_session(id) for id in self.session_ids]
+
+        assert (
+            len(self.session_ids) == len(self.messages) == len(self.test_cases)
+        )
+
+        self.metrics = metrics
+        self.aggregator = aggregator
+
+    def evaluate(self):
+        metadata = {"experiment_id": self.experiment_id}
+
+        total_metrics = []
+        for idx, test_case in enumerate(self.test_cases):
+            metric_results = []
+            messages = self.messages[idx]
+            extracted_context = ExtractLabeledMessages.extract(messages, test_case)
+            for metric in self.metrics:
+                result = metric.evaluate(
+                    messages=messages,
+                    ground_truth=test_case,
+                    extracted_context=extracted_context,
+                    metadata=metadata
+                )
+                if isinstance(result, list):
+                    metric_results.extend([r.model_dump() for r in result])
+                    for r in result:
+                        upload(
+                            name=r.eval_name,
+                            session_id=self.session_ids[idx],
+                            value=r.value,
+                            data_type=r.data_type,
+                            metadata=r.metadata,
+                        )
+                else:
+                    metric_results.append(result.model_dump())
+                    upload(
+                        name=result.eval_name,
+                        session_id=self.session_ids[idx],
+                        value=result.value,
+                        data_type=result.data_type,
+                        metadata=result.metadata,
+                    )
+            total_metrics.append(metric_results)
+
+        aggregate_metrics = self.aggregator(total_metrics)
+        for metric in aggregate_metrics:
+            try:
+                LANGFUSE_CLIENT.create_score(
+                    name=metric.eval_name,
+                    value=metric.value,
+                    metadata=metric.metadata,
+                    data_type="NUMERIC",
+                    dataset_run_id=metric.metadata["experiment_id"],
+                )
+            except Exception as e:
+                rich.print(
+                    f"[r] Uploading {metric.name} with value {metric.value} failed with exception {e}"
+                )
+
+        return ExperimentResult(
+            experiment_name=self.evaluation_name,
+            run_id=self.run_name,
+            experiment_id=self.experiment_id,
+            metrics=total_metrics,
+            session_ids=self.session_ids
+        )
+
+
+if __name__ == "__main__":
+    collection_name = "HR AGENT DEMO"
+    langfuse_collection = LangfuseCollection(name=collection_name)
+    journey_sucess_metric = JourneySuccessMetric()
+    tool_calling = ToolCalling()
+
+    SESSION_ID = "agent-demo-session-id-NEW"
+
+    run = EvaluationRunner(
+        evaluation_name="sample_evaluation",
+        run_name="1",
+        session_ids=[
+            "agent-demo-session-id-NEW-0",
+            "agent-demo-session-id-NEW-1",
+        ],
+        collection=langfuse_collection,
+        metrics=[journey_sucess_metric, tool_calling],
+        aggregator=sample_aggregator,
+    )
+
+    experiment_results = run.evaluate()
+    rich.print(experiment_results.model_dump())

wxo_agentic_evaluation/llm_matching.py

@@ -1,10 +1,22 @@
+"""
+LLM Matching Module with Cosine Similarity Support
+
+This module provides functionality for matching text using:
+1. LLM-based matching (using a language model to determine semantic equivalence)
+2. Embedding-based matching (using cosine similarity between text embeddings)
+"""
+
+import math
 from typing import List
 
+from fuzzywuzzy import fuzz
+
 from wxo_agentic_evaluation.prompt.template_render import (
     KeywordMatchingTemplateRenderer,
     SemanticMatchingTemplateRenderer,
 )
 from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+from wxo_agentic_evaluation.utils.utils import safe_divide
 
 
 class LLMMatcher:
@@ -13,10 +25,18 @@ class LLMMatcher:
         llm_client: Provider,
         keyword_template: KeywordMatchingTemplateRenderer,
         semantic_template: SemanticMatchingTemplateRenderer,
+        use_llm_for_semantic: bool = True,
+        embedding_model_id: str = "sentence-transformers/all-minilm-l6-v2",
+        similarity_threshold: float = 0.8,
+        enable_fuzzy_matching: bool = False,
     ):
         self.llm_client = llm_client
         self.keyword_template = keyword_template
         self.semantic_template = semantic_template
+        self.embedding_model_id = embedding_model_id
+        self.use_llm_for_semantic = use_llm_for_semantic
+        self.similarity_threshold = similarity_threshold
+        self.enable_fuzzy_matching = enable_fuzzy_matching
 
     def keywords_match(self, response_text: str, keywords: List[str]) -> bool:
         if len(keywords) == 0:
@@ -31,10 +51,92 @@ class LLMMatcher:
         result = output.strip().lower()
         return result.startswith("true")
 
-    def semantic_match(self, prediction: str, ground_truth: str) -> bool:
+    def generate_embeddings(
+        self, prediction: str, ground_truth: str
+    ) -> List[List[float]]:
+
+        embeddings = self.llm_client.encode([prediction, ground_truth])
+
+        return embeddings
+
+    def compute_cosine_similarity(
+        self, vec1: List[float], vec2: List[float]
+    ) -> float:
+        """Calculate cosine similarity between two vectors using pure Python"""
+
+        # Manual dot product calculation
+        dot_product = sum(a * b for a, b in zip(vec1, vec2))
+
+        # Manual magnitude calculations
+        magnitude1 = math.sqrt(sum(a * a for a in vec1))
+        magnitude2 = math.sqrt(sum(b * b for b in vec2))
+
+        return safe_divide(dot_product, (magnitude1 * magnitude2))
+
+    def cosine_similarity_semantic_match(
+        self, prediction: str, ground_truth: str
+    ) -> bool:
+        embeddings = self.generate_embeddings(prediction, ground_truth)
+        cosine_similarity = self.compute_cosine_similarity(
+            embeddings[0], embeddings[1]
+        )
+
+        return cosine_similarity >= self.similarity_threshold
+
+    def llm_semantic_match(
+        self, context, prediction: str, ground_truth: str
+    ) -> bool:
+        """Performs semantic matching for the agent's final response and the expected response using the starting sentence of the conversation as the context
+
+        Args:
+            context: The starting sentence of the conversation. TODO can also consider using the LLM user's story
+            prediction: the predicted string
+            ground_truth: the expected string
+
+        Returns:
+            a boolean indicating if the sentences match.
+        """
+
         prompt = self.semantic_template.render(
-            expected_text=ground_truth, actual_text=prediction
+            context=context, expected_text=ground_truth, actual_text=prediction
         )
         output: str = self.llm_client.query(prompt)
         result = output.strip().lower()
+
         return result.startswith("true")
+
+    def fuzzywuzzy_semantic_match(
+        self, prediction: str, ground_truth: str
+    ) -> bool:
+
+        similarity_score = fuzz.WRatio(prediction, ground_truth)
+
+        return similarity_score > self.similarity_threshold
+
+    def semantic_match(
+        self,
+        context: str,
+        prediction: str,
+        ground_truth: str,
+        enable_fuzzy_matching: bool = False,
+    ) -> bool:
+        ## TODO arjun-gupta1 10/06/2025: revist retry with exponential backoff. Opted for direct fallback to cosine similarity to avoid latency for now.
+        try:
+            return self.llm_semantic_match(context, prediction, ground_truth)
+        except Exception as e:
+            print(f"LLM semantic match failed: {e}")
+
+        if enable_fuzzy_matching:
+            print("falling back to fuzzy matching")
+            # Fallback to cosine similarity if LLM matching is not used or failed
+            try:
+                return self.cosine_similarity_semantic_match(
+                    prediction, ground_truth
+                )
+            except Exception as e:
+                print(
+                    f"Cosine similarity failed: {e}. Falling back to fuzzywuzzy."
+                )
+
+            # Final fallback to fuzzywuzzy
+            return self.fuzzywuzzy_semantic_match(prediction, ground_truth)

wxo_agentic_evaluation/llm_safety_eval.py

@@ -0,0 +1,64 @@
+import json
+import re
+
+from wxo_agentic_evaluation.metrics.llm_as_judge import (
+    AnswerDerailment,
+    AnswerUnsafeTopic,
+)
+from wxo_agentic_evaluation.prompt.template_render import (
+    DerailmentTemplateRenderer,
+    UnsafeTopicTemplateRenderer,
+)
+from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
+
+
+class LLMSafetyJudge:
+    def __init__(
+        self,
+        llm_client: Provider,
+        answer_derailment: DerailmentTemplateRenderer,
+        answer_unsafe_topic: UnsafeTopicTemplateRenderer,
+    ):
+        self.llm_client = llm_client
+        self.answer_derailment_template = answer_derailment
+        self.answer_unsafe_topic_template = answer_unsafe_topic
+
+    def judge_derailment_in_answer(
+        self, question: str, instructions: str, answer: str
+    ) -> AnswerDerailment:
+        prompt = self.answer_derailment_template.render(
+            question=question, instructions=instructions, answer=answer
+        )
+        output = self.llm_client.query(prompt)
+        result = output.strip().lower()
+        try:
+            json_str = re.search(r"\{.*\}", result, re.DOTALL).group(0)
+            thejson = json.loads(json_str)
+            answer_derailment = AnswerDerailment.model_validate(thejson)
+        except Exception:
+            answer_derailment = AnswerDerailment(
+                in_scope="not sure", statement=answer, reason=result
+            )
+        return answer_derailment
+
+    def judge_unsafe_topic_in_answer(
+        self, question: str, instructions: str, answer: str
+    ) -> AnswerUnsafeTopic:
+        """
+        Use LLM to judge if the answer contains unsafe topics.
+        """
+        prompt = self.answer_unsafe_topic_template.render(
+            question=question, instructions=instructions, answer=answer
+        )
+        output = self.llm_client.query(prompt)
+        result = output.strip().lower()
+        try:
+            json_str = re.search(r"\{.*\}", result, re.DOTALL).group(0)
+            thejson = json.loads(json_str)
+            answer_unsafe = AnswerUnsafeTopic.model_validate(thejson)
+        except Exception:
+            answer_unsafe = AnswerUnsafeTopic(
+                is_safe="not sure", statement=answer, reason=result
+            )
+
+        return answer_unsafe

wxo_agentic_evaluation/llm_user.py

@@ -1,5 +1,6 @@
 from typing import List, TypeVar
 
+from wxo_agentic_evaluation.base_user import BaseUserSimulator
 from wxo_agentic_evaluation.prompt.template_render import JinjaTemplateRenderer
 from wxo_agentic_evaluation.service_provider.watsonx_provider import Provider
 from wxo_agentic_evaluation.type import ContentType, Message
@@ -7,9 +8,9 @@ from wxo_agentic_evaluation.type import ContentType, Message
 T = TypeVar("T", bound=JinjaTemplateRenderer)
 
 
-class LLMUser:
+class LLMUser(BaseUserSimulator):
     def __init__(
-        self, wai_client: Provider, template: T, user_response_style: List[str]
+        self, wai_client: Provider, template: T, user_response_style: List[str] | None = None
     ):
         self.wai_client = wai_client
         self.prompt_template = template
@@ -21,8 +22,8 @@ class LLMUser:
         self,
         user_story,
         conversation_history: List[Message],
-        attack_instructions: str = None,
-    ) -> Message | None:
+        attack_instructions: str | None = None,
+    ) -> Message:
         # the tool response is already summarized, we don't need that to take over the chat history context window
         prompt_input = self.prompt_template.render(
             conversation_history=[