ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/otel_parser/parser.py

@@ -0,0 +1,163 @@
+from dotenv import load_dotenv
+load_dotenv()
+
+from collections import defaultdict
+from functools import partial
+from typing import Dict, List, Any, Optional
+
+from langfuse import get_client
+langfuse_client = get_client()
+
+from wxo_agentic_evaluation.type import OTelParserMessage
+from . import langgraph_parser, pydantic_parser, langflow_parser, wxo_parser
+
+def parse_session(session_id):
+    traces = get_traces(session_id)
+    messages: list[OTelParserMessage] = []
+    for tr in traces.data:
+        trace_messages = parse_trace(tr)
+        messages = add_messages(messages, trace_messages)
+    return messages
+
+def parse_trace(trace):
+    messages: list[OTelParserMessage] = []
+    agent_framework = get_agent_framework(trace)
+    if agent_framework == 'langfuse.langgraph_agent': parser_func = langgraph_parser.parse_observations
+    elif agent_framework == 'langfuse.langflow': parser_func = partial(langflow_parser.parse_observations, dfs_callable=dfs_)
+    elif agent_framework == 'langgraph_agent': parser_func = langgraph_parser.parse_observations
+    elif agent_framework == 'pydantic_ai':
+        parser_func = pydantic_parser.parse_observations
+        sys_message = pydantic_parser.get_system_message(trace)
+        if sys_message: messages.append(sys_message)
+    else:
+        parser_func = None
+        parsers_to_try = [
+            partial(langflow_parser.parse_observations, dfs_callable=dfs_),
+            wxo_parser.parse_observations,
+        ]
+    observations = get_observations(trace.observations)
+    dfs_observations = dfs_(observations)
+    if parser_func:
+        parsed_messages = parser_func(observations, dfs_observations)
+        messages = add_messages(messages, parsed_messages)
+    else:
+        for parser_func in parsers_to_try:
+            try:
+                parsed_messages = parser_func(observations, dfs_observations)
+                if not parsed_messages: continue
+                messages = add_messages(messages, parsed_messages)
+                break
+            except Exception as e:
+                print(e)
+    return messages
+
+
+def add_messages(messages: list[OTelParserMessage], parsed_messages: list[OTelParserMessage]):
+    ret: list[OTelParserMessage] = []
+    seen: set[str] = set()
+    for msg in messages:
+        msg_hash = msg.hash()
+        if msg_hash in seen: continue
+        seen.add(msg_hash)
+        ret.append(msg)
+    for msg in parsed_messages:
+        msg_hash = msg.hash()
+        if msg_hash in seen: continue
+        seen.add(msg_hash)
+        ret.append(msg)
+    return ret
+
+def get_agent_framework(trace):
+    """
+    Supported frameworks:
+    - OpenInference
+        - Pydantic AI
+        - Langflow
+        - Langgraph Agent
+    - LangFuse
+        - Langflow
+        - LangGraph Agent
+    """
+    md_attrs = trace.metadata.get('attributes', {})
+    scope = trace.metadata.get('scope', {})
+    scope_name = scope.get('name', '')
+
+    if scope_name == 'langfuse-sdk':
+        if trace.name == 'LangGraph': return 'langfuse.langgraph_agent'
+
+    if 'langflow.project.name' in md_attrs.keys(): return 'langflow'
+    if 'pydantic-ai' in scope_name: return 'pydantic_ai'
+    if 'openinference.instrumentation.langchain' in scope_name:
+        # TODO: need to find a better way to detect Langgraph Agent
+        return 'langgraph_agent'
+
+    # check for langflow
+    # get observations for trace
+    observations = dfs_(get_observations(trace.observations))
+    for obs in observations:
+        if 'from_langflow_component' in obs.obs.metadata.keys(): return 'langfuse.langflow'
+    return "UNKNOWN"
+
+
+def get_traces(session_id):
+    traces = langfuse_client.api.trace.list(
+        session_id=session_id,
+        limit=100,
+    )
+    # sort by timestamp
+    traces.data.sort(key=lambda x: x.timestamp)
+    return traces
+
+
+def get_observations(observations_ids):
+    observations = [langfuse_client.api.observations.get(obs_id) for obs_id in observations_ids]
+    observations.sort(key=lambda x: x.start_time)
+    observation_tree = build_observation_forest(observations)
+    return observation_tree
+
+class ObsNode:
+    def __init__(self, obs: Any):
+        self.obs = obs
+        self.children: List["ObsNode"] = []
+        self.parent: Optional["ObsNode"] = None
+
+    def __repr__(self):
+        return f"ObsNode(id={self.obs.id}, type={self.obs.type}, name={self.obs.name})"
+
+def build_observation_forest(observations: List[Any]) -> List[ObsNode]:
+    """Return list of root nodes; each has .children forming a tree."""
+    nodes: Dict[str, ObsNode] = {}
+    children_by_parent: Dict[Optional[str], List[ObsNode]] = defaultdict(list)
+
+    # 1. Create nodes for each observation
+    for o in observations:
+        node = ObsNode(o)
+        nodes[o.id] = node
+        parent_id = getattr(o, "parent_observation_id", None)
+        children_by_parent[parent_id].append(node)
+
+    # 2. Attach children to parents
+    for parent_id, child_nodes in children_by_parent.items():
+        if parent_id is None:
+            continue
+        parent_node = nodes.get(parent_id)
+        if parent_node:
+            parent_node.children.extend(child_nodes)
+            for child_node in child_nodes: child_node.parent = parent_node
+
+    # 3. Roots are those with parent_observation_id == None
+    roots = children_by_parent[None]
+    return roots
+
+def dfs_(observation_tree: List[ObsNode]):
+    ret =[]
+    for node in observation_tree:
+        ret.append(node)
+        ret.extend(dfs_(node.children))
+    return ret
+
+
+if __name__ == "__main__":
+    messages = parse_session(session_id="93a24957-dd7a-425d-b821-88de49940a6e")
+
+    print(messages)

wxo_agentic_evaluation/otel_parser/parser_types.py

@@ -0,0 +1,38 @@
+import dataclasses
+from typing import Literal
+from hashlib import md5
+
+# TODO: This is not a fully chat completions message.
+
+
+@dataclasses.dataclass
+class Function:
+    name: str
+    arguments: str
+
+    def __str__(self):
+        return f"{self.name}:{self.arguments}"
+
+
+@dataclasses.dataclass
+class ToolCall:
+    id: str
+    function: "Function"
+    type: Literal["function"] = "function"
+
+    def __str__(self):
+        return f"{self.id}:{self.type}:{self.function}"
+
+
+@dataclasses.dataclass
+class Message:
+    role: Literal["user", "assistant", "tool", "system"]
+    content: str | None = None
+    tool_calls: list["ToolCall"] | None = None
+    tool_call_id: str | None = None
+
+    def __str__(self):
+        return f'{self.role}:{self.content}:{":".join(map(str, self.tool_calls or []))}:{self.tool_call_id}'
+
+    def hash(self):
+        return md5(self.__str__().encode("utf-8")).hexdigest()

wxo_agentic_evaluation/otel_parser/pydantic_parser.py

@@ -0,0 +1,50 @@
+import json
+from wxo_agentic_evaluation.type import OTelParserMessage, OTelParserToolCall, OTelParserFunction, ContentType
+
+
+def get_system_message(trace):
+    sys_instruction_json = trace.metadata.get('attributes', {}).get('gen_ai.system_instructions')
+    if not sys_instruction_json: return None
+    instruction = json.loads(sys_instruction_json)[0]['content']
+    return OTelParserMessage(role='system', content=instruction, type=ContentType.text)
+
+def parse_observations(observation_tree, dfs_observations):
+    messages = []
+    for obs in dfs_observations:
+        if obs.obs.type == 'GENERATION':
+            messages.extend(_get_messages(obs.obs.input))
+            messages.extend(_get_messages(obs.obs.output))
+    return messages
+
+
+def _get_messages(data):
+    messages = []
+    for msg in data:
+        if msg['role'] == 'user':
+            parts = msg['parts']
+            for part in parts:
+                content_type = part['type']
+                if content_type == 'text':
+                    content = part['content']
+                    messages.append(OTelParserMessage(role='user', content=content, type=ContentType.text))
+                elif content_type == 'tool_call_response':
+                    tool_call_id = part['id']
+                    result = json.dumps(part['result'])
+                    messages.append(OTelParserMessage(role='tool', tool_call_id=tool_call_id, content=result, type=ContentType.tool_response))
+        elif msg['role'] == 'assistant':
+            parts = msg['parts']
+            # TODO: assuming that only one message is present in the assistant parts
+            content_type = parts[0]['type']
+            if content_type == 'text':
+                content = parts[0]['content']
+                messages.append(OTelParserMessage(role='assistant', content=content, type=ContentType.text))
+            elif content_type == 'tool_call':
+                tool_calls = []
+                for part in parts:
+                    tool_call_id = part['id']
+                    tool_call_name = part['name']
+                    tool_call_arguments = part['arguments']
+                    tool_call = OTelParserToolCall(id=tool_call_id, function=OTelParserFunction(name=tool_call_name, arguments=tool_call_arguments))
+                    tool_calls.append(tool_call)
+                messages.append(OTelParserMessage(role='assistant', content='', tool_calls=tool_calls, type=ContentType.tool_call))
+    return messages

wxo_agentic_evaluation/otel_parser/utils.py

@@ -0,0 +1,15 @@
+import json
+from .parser_types import ToolCall, Function
+
+def to_tool_call(tool_call):
+    return ToolCall(
+        id=tool_call['id'],
+        type=tool_call['type'],
+        function=to_function(tool_call)
+    )
+
+def to_function(func):
+    return Function(
+        name=func['name'],
+        arguments=json.dumps(func['args'])
+    )

wxo_agentic_evaluation/otel_parser/wxo_parser.py

@@ -0,0 +1,39 @@
+import json
+from wxo_agentic_evaluation.type import OTelParserMessage, OTelParserToolCall, OTelParserFunction, ContentType
+
+def parse_observations(observation_tree, dfs_observations):
+    messages = []
+    for node in dfs_observations:
+        if node.obs.type == 'GENERATION':
+            if node.parent.obs.name == 'invoke_agent':
+                messages.extend(_get_input_messages(node.parent.obs.input))
+                messages.extend(_get_output_message(node.parent.obs.output))
+    return messages
+
+def _get_input_messages(data):
+    messages = []
+    for msg in data['messages']:
+        if msg['type'] == 'system': messages.append(OTelParserMessage(role='system', content=msg['content'], type=ContentType.text))
+        elif msg['type'] == 'human': messages.append(OTelParserMessage(role='user', content=msg['content'], type=ContentType.text))
+        elif msg['type'] == 'tool': messages.append(OTelParserMessage(role='tool', content=msg['content'], tool_call_id=msg['tool_call_id']))
+        elif msg['type'] == 'ai':
+            if msg.get('additional_kwargs', {}).get('tool_calls', None) is not None:
+                msg_tool_calls = msg['additional_kwargs']['tool_calls']
+                tool_calls = []
+                for tc in msg_tool_calls:
+                    tool_calls.append(OTelParserToolCall(id=tc['id'], function=OTelParserFunction(name=tc['function']['name'], arguments=tc['function']['arguments'])))
+            else:
+                tool_calls = None
+            messages.append(OTelParserMessage(role='assistant', content=msg['content'], tool_calls=tool_calls, type=ContentType.tool_call))
+    return messages
+
+
+def _get_output_message(data):
+    role = 'assistant'
+    content = data.get('content', '')
+    if data.get('tool_calls', None):
+        tool_calls = []
+        for tc in data['tool_calls']:
+            tool_calls.append(OTelParserToolCall(id=tc['id'], function=OTelParserFunction(name=tc['name'], arguments=json.dumps(tc.get('args', {})))))
+    else: tool_calls = None
+    return [OTelParserMessage(role='assistant', content=content, tool_calls=tool_calls)]

wxo_agentic_evaluation/otel_support/evaluate_tau.py

@@ -0,0 +1,101 @@
+import json
+
+from wxo_agentic_evaluation.evaluation_package import EvaluationPackage
+from wxo_agentic_evaluation.type import (
+    ContentType,
+    EvaluationData,
+    EventTypes,
+    Message,
+)
+
+with open(
+    "/Users/haodeqi/git/tau-bench/historical_trajectories/gpt-4o-airline.json",
+    "r",
+) as f:
+    test_data = json.load(f)
+
+
+goal_temp = []
+
+goals = {}
+goal_details = []
+
+i = 0
+for action in test_data[0]["info"]["task"]["actions"]:
+    goal_temp.append(action["name"] + f"_{i}")
+    goal_detail = {
+        "type": "tool_call",
+        "name": action["name"] + f"_{i}",
+        "tool_name": action["name"],
+        "args": {k: str(v) for k, v in action["kwargs"].items()},
+    }
+    goal_details.append(goal_detail)
+
+if len(goal_temp) == 1:
+    goals[goal_temp[0]] = []
+else:
+    for i in range(len(goal_temp) - 1):
+        goals.update({goal_temp[i]: goal_temp[i + 1]})
+
+gt_data = {
+    "agent": "airline_agent",
+    "goals": goals,
+    "goal_details": goal_details,
+    "story": test_data[0]["info"]["task"]["instruction"],
+    "starting_sentence": "",
+}
+print("2")
+gt_data = EvaluationData.model_validate(gt_data)
+
+tc_name = "airline_1"
+
+print(test_data[0]["traj"][0])
+
+history = []
+for msg in test_data[0]["traj"]:
+    if msg["role"] == "tool":
+        print(msg["content"])
+        history.append(
+            Message(
+                role=msg["role"],
+                content=json.dumps(
+                    {
+                        "type": "tool_call",
+                        "args": json.loads(msg["content"]),
+                        "name": msg["name"],
+                        "tool_call_id": msg["tool_call_id"],
+                    }
+                ),
+                type=ContentType.tool_call,
+                event=EventTypes.message_created,
+            )
+        )
+    else:
+        history.append(
+            Message(
+                role=msg["role"],
+                content=str(msg["content"]),
+                type=ContentType.text,
+                event=EventTypes.message_created,
+            )
+        )
+
+print(f"length of history {history}")
+
+evaluation_package = EvaluationPackage(
+    test_case_name=tc_name,
+    messages=history,
+    ground_truth=gt_data,
+    conversational_search_data=None,
+    resource_map=None,
+)
+print("1")
+(
+    keyword_semantic_matches,
+    knowledge_base_metrics,
+    messages_with_reason,
+    metrics,
+) = evaluation_package.generate_summary()
+
+
+print(metrics)

wxo_agentic_evaluation/otel_support/otel_message_conversion.py

@@ -0,0 +1,29 @@
+import json
+from typing import Any, Dict, List, Optional, Union
+
+from wxo_agentic_evaluation.type import ContentType, EventTypes, Message
+
+# with open("src/wxo_agentic_evaluation/otel_support/collie_example.json", "r") as f:
+#     data = json.load(f)
+#
+# otel_traces = data["calls"][-1]["messages"]
+
+
+def convert_otel_to_message(otel_traces):
+    history = []
+    for row in otel_traces:
+        print(row)
+        content = row["content"]
+        print(row.keys())
+        role = row.get("role", "assistant")
+
+        history.append(
+            Message(
+                role=role,
+                content=content,
+                type=ContentType.text,
+                event=EventTypes.message_created,
+            )
+        )
+
+    return history