ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/type.py

@@ -1,6 +1,21 @@
-from typing import Dict, List, Union, Any, Optional
-from pydantic import BaseModel, computed_field, ConfigDict
-from enum import StrEnum
+from enum import Enum, StrEnum
+from hashlib import md5
+from typing import Any, Dict, List, Literal, Mapping, Optional, Union
+
+from pydantic import (
+    BaseModel,
+    ConfigDict,
+    Field,
+    computed_field,
+    model_validator,
+)
+
+
+class CallTracker(BaseModel):
+    tool_call: List = []
+    tool_response: List = []
+    generic: List = []
+    metadata: Dict[str, Any] = Field(default={})
 
 
 class EventTypes(StrEnum):
@@ -20,6 +35,16 @@ class ContentType(StrEnum):
     conversational_search = "conversational_search"
 
 
+class AttackCategory(StrEnum):
+    on_policy = "on_policy"
+    off_policy = "off_policy"
+
+
+class Roles(Enum):
+    ASSISTANT = "assistant"
+    USER = "user"
+
+
 class ConversationalSearchCitations(BaseModel):
     url: str
     body: str
@@ -51,9 +76,13 @@ class ConversationalConfidenceThresholdScore(BaseModel):
     def table(self):
         return {
             "response_confidence": str(self.response_confidence),
-            "response_confidence_threshold": str(self.response_confidence_threshold),
+            "response_confidence_threshold": str(
+                self.response_confidence_threshold
+            ),
             "retrieval_confidence": str(self.retrieval_confidence),
-            "retrieval_confidence_threshold": str(self.retrieval_confidence_threshold),
+            "retrieval_confidence_threshold": str(
+                self.retrieval_confidence_threshold
+            ),
         }
 
 
@@ -79,10 +108,35 @@ class ConversationalSearch(BaseModel):
     response_length_option: str
 
 
+class OTelParserFunction(BaseModel):
+    """OpenAI chat completion function structure for OTel parser tool calls"""
+
+    name: str
+    arguments: str  # JSON string of arguments
+
+    model_config = ConfigDict(frozen=True)
+
+    def __str__(self):
+        return f"{self.name}:{self.arguments}"
+
+
+class OTelParserToolCall(BaseModel):
+    """OpenAI chat completion tool call structure for OTel parser"""
+
+    id: str
+    function: OTelParserFunction
+    type: Literal["function"] = "function"
+
+    model_config = ConfigDict(frozen=True)
+
+    def __str__(self):
+        return f"{self.id}:{self.type}:{self.function}"
+
+
 class Message(BaseModel):
     role: str
     content: Union[str, Dict[str, Any]]
-    type: ContentType
+    type: ContentType = None
     # event that produced the message
     event: Optional[str] = None
     # used to correlate the Message with the retrieval context (ConversationalSearch)
@@ -93,7 +147,32 @@ class Message(BaseModel):
 
 class ExtendedMessage(BaseModel):
     message: Message
-    reason: dict | None = None
+    reason: dict | list | None = None
+
+
+class OTelParserMessage(Message):
+    """Message class for OTel parser with OpenAI-compatible tool call fields.
+
+    Inherits from Message and adds structured tool call fields for compatibility
+    with OpenTelemetry trace parsing (LangGraph, Pydantic AI, etc.)
+    """
+
+    tool_calls: Optional[List[OTelParserToolCall]] = None
+    tool_call_id: Optional[str] = None
+
+    def hash(self) -> str:
+        """Generate hash for message deduplication"""
+        parts = [
+            self.role,
+            str(self.content) if self.content else "",
+            (
+                ":".join(str(tc) for tc in self.tool_calls)
+                if self.tool_calls
+                else ""
+            ),
+            self.tool_call_id or "",
+        ]
+        return md5(":".join(parts).encode("utf-8")).hexdigest()
 
 
 class KnowledgeBaseGoalDetail(BaseModel):
@@ -101,19 +180,143 @@ class KnowledgeBaseGoalDetail(BaseModel):
     metrics: list = []
 
 
+class MatchingStrategy(StrEnum):
+    """Argument matching strategy:\n
+    Strict: exact match\n
+    Optional: optional argument, exact match if the field exists\n
+    Fuzzy: semantic/similarity match\n"""
+
+    strict = "strict"
+    optional = "optional"
+    fuzzy = "fuzzy"
+
+
 class GoalDetail(BaseModel):
     name: str
-    tool_name: str = None
+    tool_name: Optional[str] = None
     type: ContentType
-    args: Dict = None
-    response: str = None
-    keywords: List = None
+    args: Optional[Dict] = None
+    # matching strategy defaults to `strict` matching if not specified in the test case
+    arg_matching: Optional[dict[str, MatchingStrategy]] = Field(
+        default_factory=dict
+    )
+    response: Optional[str] = None
+    keywords: Optional[List] = None
+
+    @model_validator(mode="after")
+    def validate_arg_matching(self):
+        for field in self.arg_matching:
+            if field not in self.args:
+                raise ValueError(
+                    f"{field} not in goal arguments for goal {self.name}"
+                )
+        return self
+
+
+class GoalDetailOrchestrate(GoalDetail):
     knowledge_base: KnowledgeBaseGoalDetail = KnowledgeBaseGoalDetail()
 
 
-class EvaluationData(BaseModel):
+class AttackData(BaseModel):
+    attack_category: AttackCategory
+    attack_type: str
+    attack_name: str
+    attack_instructions: str
+
+
+class AttackData(BaseModel):
     agent: str
-    goals: Dict
+    agents_list_or_path: Union[List[str], str]
+    attack_data: AttackData
     story: str
+    starting_sentence: str
+    goals: dict | None = None
+    goal_details: list[GoalDetail] | None = None
+
+
+class DatasetModel(BaseModel):
+    starting_sentence: str | None = None
+    story: str
+    goals: Mapping[str, Any]
     goal_details: List[GoalDetail]
-    starting_sentence: str = None
+    max_user_turns: int | None = None
+    agent: str | None = None
+
+
+class LangfuseDatasetModel(DatasetModel):
+    @computed_field
+    @property
+    def langfuse_input(self) -> Mapping[str, Any]:
+        input = {
+            "starting_sentence": self.starting_sentence,
+            "story": self.story,
+            "agent": self.agent
+        }
+
+        return input
+
+    @computed_field
+    @property
+    def langfuse_output(self) -> Mapping[str, Any]:
+        output = {"goals": self.goals, "goal_details": self.goal_details}
+
+        return output
+
+
+def _convert_to_langfuse_format(langfuse_row) -> LangfuseDatasetModel:
+    input = langfuse_row.input
+    output = langfuse_row.expected_output
+
+    for goal in output.get("goal_details"):
+        GoalDetail.model_validate(goal)
+
+    return LangfuseDatasetModel(
+        starting_sentence=input.get("starting_sentence"),
+        story=input.get("story"),
+        goals=output.get("goals"),
+        goal_details=[
+            GoalDetail.model_validate(goal)
+            for goal in output.get("goal_details")
+        ],
+    )
+
+
+class OrchestrateDataset(DatasetModel):
+    goal_details: List[GoalDetailOrchestrate]
+    agent: str
+
+
+class LangfuseCollectionModel(BaseModel):
+    collection_name: str
+    datasets: List[LangfuseDatasetModel]
+    collection_description: Optional[str] = ""
+    metadata: Optional[Mapping[str, str]] = None
+
+
+class ToolDefinition(BaseModel):
+    tool_description: Optional[str]
+    tool_name: str
+    tool_params: List[str]
+
+
+class ProviderInstancesCacheKey(BaseModel):
+    provider: str
+    hashed_args: str
+    hashed_kwargs: str
+
+    def __str__(self) -> str:
+        return f"{self.provider}|{self.hashed_args}|{self.hashed_kwargs}"
+
+
+class RuntimeResponse(BaseModel):
+    messages: List[Message]
+    thread_id: str | None = None
+    context: dict = Field(default={})
+
+
+class ExperimentResult(BaseModel):
+    experiment_name: str
+    run_id: str
+    experiment_id: str
+    metrics: list
+    session_ids: List[str]

wxo_agentic_evaluation/user_simulator/demo_usage_llm_user.py

@@ -0,0 +1,100 @@
+from wxo_agentic_evaluation.llm_user_v2 import LLMUser
+from wxo_agentic_evaluation.service_provider.portkey_provider import (
+    PortkeyProvider,
+)
+from openai import OpenAI
+import os
+import uuid
+
+from wxo_agentic_evaluation.type import Message, ContentType
+
+user_story = "Your user id is mia_li_3668. You want to fly from New York to Seattle on May 20 (one way). You do not want to fly before 11am est. You want to fly in economy. You prefer direct flights but one stopover also fine. If there are multiple options, you prefer the one with the lowest price. You have 3 baggages. You do not want insurance. You want to use your two certificates to pay. If only one certificate can be used, you prefer using the larger one, and pay the rest with your 7447 card. You are reactive to the agent and will not say anything that is not asked. Your birthday is in your user profile so you do not prefer to provide it."
+
+portkey_client = PortkeyProvider(
+    provider="@openai",
+    model_id="gpt-4o-mini",
+    api_key=os.environ.get("PORTKEY_API_KEY"),
+)
+
+user_response_style = [
+    "reactive to the agent and will not say anything that is not asked",
+    "replies only in very short sentences and few words",
+]
+
+user_agent = LLMUser(
+    llm_client=portkey_client,
+    user_prompt_path="../prompt/universal_user_template.jinja2",
+)
+
+agent = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+
+
+def get_agent_response(messages: list[dict]) -> str:
+
+    response = agent.chat.completions.create(
+        model="gpt-4o-mini", messages=messages
+    )
+    return response.choices[0].message.content
+
+
+starting_user_input = Message(
+    role="user", content="I want to fly.", type=ContentType.text
+)
+
+
+agent_system_prompt = Message(
+    role="system",
+    content="You are a helpful assistant. Keep your responses short and concise.",
+    type=ContentType.text,
+)
+
+session_id = str(uuid.uuid4())
+max_turns = 30
+conversation_history = []
+for i in range(max_turns):
+
+    if len(conversation_history) == 0:
+        conversation_history.append(agent_system_prompt)
+        conversation_history.append(
+            Message(
+                role="assistant",
+                content="Hi! How can I help you today?",
+                type=ContentType.text,
+            )
+        )
+
+        user_response = user_agent.generate_user_input(
+            user_story=user_story,
+            conversation_history=conversation_history,
+            user_response_style=user_response_style,
+            starting_user_input=starting_user_input,
+        )
+    else:
+        user_response = user_agent.generate_user_input(
+            user_story=user_story,
+            conversation_history=conversation_history,
+            user_response_style=user_response_style,
+            starting_user_input=None,
+        )
+
+    conversation_history.append(user_response)
+    print(f"User: {user_response.content}")
+
+    if "END" in user_response.content:
+        break
+
+    # Get agent response
+    agent_response_content = get_agent_response(
+        [msg.model_dump() for msg in conversation_history]
+    )
+    # agent_response_content = get_langflow_agent_response(conversation_history, session_id)
+    # agent_response_content = asyncio.run(get_langgraph_agent_response(conversation_history, session_id))
+    print(f"Agent: {agent_response_content}")
+
+    agent_response = Message(
+        role="assistant", content=agent_response_content, type=ContentType.text
+    )
+    conversation_history.append(agent_response)
+
+
+print(conversation_history)

wxo_agentic_evaluation/utils/__init__.py

@@ -1,6 +1,47 @@
 import json
+import os
+import tempfile
+from pathlib import Path
 
+from wxo_agentic_evaluation.utils.open_ai_tool_extractor import (
+    ToolExtractionOpenAIFormat,
+)
+from wxo_agentic_evaluation.utils.parsers import ReferencelessEvalParser
+from wxo_agentic_evaluation.utils.utils import (
+    N_A,
+    TestCaseResources,
+    add_line_seperator,
+    list_run_files,
+    load_run_metrics,
+)
 
-def json_dump(output_path, object):
-    with open(output_path, "w", encoding="utf-8") as f:
-        json.dump(object, f, indent=4)
+
+def json_dump(output_path, obj):
+    """
+    Atomically dump JSON to `output_path`.
+
+    - Writes to a temporary file first
+    - Then atomically replaces the target file
+    - Prevents corrupted/half-written JSON if process is interrupted
+    """
+    output_path = Path(output_path)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    fd, tmp_path = tempfile.mkstemp(
+        dir=output_path.parent,
+        prefix=output_path.stem,
+        suffix=".tmp",
+        text=True,
+    )
+    try:
+        with os.fdopen(fd, "w", encoding="utf-8") as f:
+            json.dump(obj, f, indent=4, ensure_ascii=False)
+            f.flush()
+            os.fsync(f.fileno())
+        os.replace(tmp_path, output_path)
+    except Exception:
+        try:
+            os.remove(tmp_path)
+        except OSError:
+            pass
+        raise

wxo_agentic_evaluation/utils/evaluation_discovery.py

@@ -0,0 +1,47 @@
+"""
+Evaluation discovery mechanism.
+
+This module provides functionality for discovering classes that inherit from Evaluation.
+"""
+
+import importlib.util
+import inspect
+import os
+
+
+def find_evaluation_subclasses(directory: str, base_class_name="Evaluation"):
+    """
+    Dynamically import Python files under 'directory' and find classes that
+    inherit from a class named 'Evaluation'. Returns a list of non-abstract
+    class objects.
+    """
+    subclasses = []
+
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".py") and not file.startswith("__"):
+                filepath = os.path.join(root, file)
+                module_name = os.path.splitext(os.path.basename(filepath))[0]
+
+                spec = importlib.util.spec_from_file_location(
+                    module_name, filepath
+                )
+                if spec and spec.loader:
+                    module = importlib.util.module_from_spec(spec)
+                    try:
+                        spec.loader.exec_module(module)
+                    except Exception as e:
+                        print(f"Skipping {filepath} due to import error: {e}")
+                        continue
+
+                    # Inspect for subclasses
+                    for name, obj in inspect.getmembers(
+                        module, inspect.isclass
+                    ):
+                        if any(
+                            base.__name__ == base_class_name
+                            for base in obj.__mro__[1:]
+                        ) and not inspect.isabstract(obj):
+                            subclasses.append(obj)
+
+    return subclasses

wxo_agentic_evaluation/utils/gateway_provider_utils.py

@@ -0,0 +1,39 @@
+import os
+from functools import lru_cache
+
+from wxo_agentic_evaluation.arg_configs import AuthConfig
+from wxo_agentic_evaluation.service_provider import USE_GATEWAY_MODEL_PROVIDER
+from wxo_agentic_evaluation.wxo_client import get_wxo_client
+
+WXO_AUTH_CONFIG_DEFAULTS = AuthConfig(
+    url=os.getenv("WXO_URL", "http://localhost:4321"),
+    tenant_name=os.getenv("WXO_TENANT", "wxo-dev"),
+    token=os.getenv("WXO_TOKEN", None),
+)
+
+
+@lru_cache(maxsize=1)
+def _get_cached_wxo_client():
+    # TODO: remove this once the client is implemented as a Singleton.
+    return get_wxo_client(
+        WXO_AUTH_CONFIG_DEFAULTS.url,
+        WXO_AUTH_CONFIG_DEFAULTS.tenant_name,
+        WXO_AUTH_CONFIG_DEFAULTS.token,
+    )
+
+
+def get_provider_kwargs(**base_kwargs: dict) -> dict:
+
+    if not USE_GATEWAY_MODEL_PROVIDER:
+        return base_kwargs
+
+    if "instance_url" in base_kwargs and "token" in base_kwargs:
+        return base_kwargs
+
+    wxo_client = _get_cached_wxo_client()
+
+    return {
+        **base_kwargs,
+        "instance_url": wxo_client.service_url,
+        "token": wxo_client.api_key,
+    }

wxo_agentic_evaluation/utils/messages_parser.py

@@ -0,0 +1,30 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from wxo_agentic_evaluation.type import ContentType, Message
+
+
+class ParsedMessages(BaseModel):
+    """
+    A parsed history of messages.
+    """
+
+    messages: list[Message] = Field(description="The list of messages")
+
+    @property
+    def user_input(self) -> Optional[str]:
+        """Find the original user message."""
+        for message in self.messages:
+            if message.role == "user" and message.type == ContentType.text:
+                return str(message.content)
+        return None
+
+    @property
+    def agent_response(self) -> Optional[str]:
+        """Find the most recent assistant message."""
+        messages_in_reverse = reversed(self.messages)
+        for message in messages_in_reverse:
+            if message.role == "assistant" and message.type == ContentType.text:
+                return str(message.content)
+        return None