ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/otel_parser/langflow_parser.py

@@ -0,0 +1,86 @@
+import json
+from wxo_agentic_evaluation.type import OTelParserMessage, OTelParserToolCall, OTelParserFunction, ContentType
+
+def parse_observations(observation_tree, dfs_observations, dfs_callable: callable):
+    messages = []
+    for node in dfs_observations:
+        # assume there will only be one AgentExecutor in the trace! 
+        if node.obs.name == 'AgentExecutor': return _parse_agent_executor(node.children, dfs_callable(node.children))
+    return messages
+
+
+def _parse_agent_executor(observation_tree, dfs_observations):
+    messages = []
+    for node in dfs_observations:
+        if node.obs.type == 'GENERATION':
+            print(node.obs.id)
+            messages.extend(_get_messages(node.obs.input))
+            # get intemediate steps from parent
+            messages.extend(_get_intermediate_steps(node.parent))
+            messages.extend(_get_messages([node.obs.output]))
+    return messages
+
+
+def _get_messages(data):
+    messages = []
+    for msg in data:
+        if msg['role'] == 'system': messages.append(OTelParserMessage(role='system', content=msg['content'], type=ContentType.text))
+        elif msg['role'] == 'user':
+            content = ''
+            if isinstance(msg['content'], list):
+                content = []
+                for item in msg['content']:
+                    if item['type'] == ['text']: content.append(item['text'])
+                content = ' '.join(content)
+            elif isinstance(msg['content'], str):
+                content = msg['content']
+
+            messages.append(OTelParserMessage(role='user', content=content, type=ContentType.text))
+        elif msg['role'] == 'assistant':
+            content = msg['content'] or ''
+            additional_kwargs = msg.get('additional_kwargs', {})
+            tool_calls = None
+            if 'tool_calls' in additional_kwargs:
+                tool_calls = []
+                for tc in additional_kwargs['tool_calls']:
+                    id_ = tc['id']
+                    function = OTelParserFunction(name=tc['function']['name'], arguments=tc['function']['arguments'])
+                    tool_calls.append(OTelParserToolCall(id=id_, function=function))
+            messages.append(OTelParserMessage(role='assistant', content=content, tool_calls=tool_calls, type=ContentType.tool_call))
+    return messages
+
+def _get_intermediate_steps(node):
+    messages = []
+    tool_calls_n_responses = node.obs.input['intermediate_steps']
+    for tc, tr in tool_calls_n_responses:
+        if 'tool' in tc and 'tool_input' in tc and 'tool_call_id' in tc:
+            tool_call_id = tc['tool_call_id']
+            if isinstance(tr, str):
+                messages.append(OTelParserMessage(role='tool', content=tr, tool_call_id=tool_call_id, type=ContentType.tool_response))
+                continue
+            elif (isinstance(tr, dict) and 'content' not in tr):
+                messages.append(OTelParserMessage(role='tool', content=json.dumps(tr), tool_call_id=tool_call_id, type=ContentType.tool_response))
+                continue
+            elif isinstance(tr, dict) and 'content' in tr:
+                content = tr['content']
+                if isinstance(content, str):
+                    messages.append(OTelParserMessage(role='tool', content=content, tool_call_id=tool_call_id, type=ContentType.tool_response))
+                    continue
+                elif isinstance(content, list):
+                    for part in content:
+                        if isinstance(part, dict) and part['type'] == 'text':
+                            text = part['text']
+                            if isinstance(text, dict): text = json.dumps(text)
+                            messages.append(OTelParserMessage(role='tool', content=text, tool_call_id=tool_call_id, type=ContentType.tool_response))
+                            continue
+                        else:
+                            raise ValueError(f"Unexpected part type: {type(part)} or part[type] '{part['type']}' != 'text'")
+                else:
+                    raise ValueError(f"Unexpected content type: {type(content)}")
+            else:
+                raise ValueError(f"Unexpected tool response: Type: {type(tr)}, Value: {tr}")
+
+        else:
+            print('Tool Call:', tc)
+            print('Tool Response:', tr)
+    return messages

wxo_agentic_evaluation/otel_parser/langgraph_parser.py

@@ -0,0 +1,61 @@
+import json
+from wxo_agentic_evaluation.type import OTelParserMessage, OTelParserToolCall, OTelParserFunction, ContentType
+
+def parse_observations(observation_tree, dfs_observations):
+    messages = []
+    is_first_generation = True
+    for obs in dfs_observations:
+        if obs.obs.type == 'GENERATION':
+            if is_first_generation:
+                messages.extend(_get_input_message(obs))
+                is_first_generation = False
+            parent = obs.parent
+            if parent.obs.type == 'CHAIN':
+                # TODO: messages is a list. confirm, we will only see one message in the list.
+                msg = parent.obs.output['messages'][0]
+                content = msg['content'] or ''
+                msg_type = ContentType.text
+                tool_calls = msg['tool_calls'] or None
+                if tool_calls is not None:
+                    msg_type = ContentType.tool_call
+                    tool_calls = [_to_tool_call(tc) for tc in tool_calls]
+                messages.append(OTelParserMessage(role='assistant', content=content, tool_calls=tool_calls, type=msg_type))
+        elif obs.obs.type == 'TOOL':
+            parent_node = obs.parent
+            if parent_node.obs.type == 'CHAIN':
+                for tool_response in parent_node.obs.output['messages']:
+                    messages.append(OTelParserMessage(role='tool', content=tool_response['content'], tool_call_id=tool_response['tool_call_id'], type=ContentType.tool_response))
+    return messages
+
+
+def _get_input_message(obs_node):
+    ret = []
+    parent = obs_node.parent
+    if parent.obs.type == 'CHAIN':
+        for msg in parent.obs.input['messages']:
+            if msg['type'] == 'system': ret.append(OTelParserMessage(role='system', content=msg['content'], type=ContentType.text))
+            elif msg['type'] == 'human': ret.append(OTelParserMessage(role='user', content=msg['content'], type=ContentType.text))
+            elif msg['type'] == 'tool': ret.append(OTelParserMessage(role='tool', content=msg['content'], tool_call_id=msg['tool_call_id'], type=ContentType.tool_response))
+            elif msg['type'] == 'ai':
+                content = msg['content'] or ''
+                tool_calls = msg['tool_calls'] or None
+                msg_type = ContentType.text
+                if tool_calls is not None:
+                    msg_type = ContentType.tool_call
+                    tool_calls = [_to_tool_call(tc) for tc in tool_calls]
+                ret.append(OTelParserMessage(role='assistant', content=content, tool_calls=tool_calls, type=msg_type))
+    return ret
+
+
+def _to_tool_call(tool_call):
+    return OTelParserToolCall(
+        id=tool_call['id'],
+        type='function',  # OTelParserToolCall expects literal 'function'
+        function=_to_function(tool_call)
+    )
+
+def _to_function(func):
+    return OTelParserFunction(
+        name=func['name'],
+        arguments=json.dumps(func['args'])
+    )

wxo_agentic_evaluation/otel_parser/parser.py

@@ -0,0 +1,163 @@
+from dotenv import load_dotenv
+load_dotenv()
+
+from collections import defaultdict
+from functools import partial
+from typing import Dict, List, Any, Optional
+
+from langfuse import get_client
+langfuse_client = get_client()
+
+from wxo_agentic_evaluation.type import OTelParserMessage
+from . import langgraph_parser, pydantic_parser, langflow_parser, wxo_parser
+
+def parse_session(session_id):
+    traces = get_traces(session_id)
+    messages: list[OTelParserMessage] = []
+    for tr in traces.data:
+        trace_messages = parse_trace(tr)
+        messages = add_messages(messages, trace_messages)
+    return messages
+
+def parse_trace(trace):
+    messages: list[OTelParserMessage] = []
+    agent_framework = get_agent_framework(trace)
+    if agent_framework == 'langfuse.langgraph_agent': parser_func = langgraph_parser.parse_observations
+    elif agent_framework == 'langfuse.langflow': parser_func = partial(langflow_parser.parse_observations, dfs_callable=dfs_)
+    elif agent_framework == 'langgraph_agent': parser_func = langgraph_parser.parse_observations
+    elif agent_framework == 'pydantic_ai':
+        parser_func = pydantic_parser.parse_observations
+        sys_message = pydantic_parser.get_system_message(trace)
+        if sys_message: messages.append(sys_message)
+    else:
+        parser_func = None
+        parsers_to_try = [
+            partial(langflow_parser.parse_observations, dfs_callable=dfs_),
+            wxo_parser.parse_observations,
+        ]
+    observations = get_observations(trace.observations)
+    dfs_observations = dfs_(observations)
+    if parser_func:
+        parsed_messages = parser_func(observations, dfs_observations)
+        messages = add_messages(messages, parsed_messages)
+    else:
+        for parser_func in parsers_to_try:
+            try:
+                parsed_messages = parser_func(observations, dfs_observations)
+                if not parsed_messages: continue
+                messages = add_messages(messages, parsed_messages)
+                break
+            except Exception as e:
+                print(e)
+    return messages
+
+
+def add_messages(messages: list[OTelParserMessage], parsed_messages: list[OTelParserMessage]):
+    ret: list[OTelParserMessage] = []
+    seen: set[str] = set()
+    for msg in messages:
+        msg_hash = msg.hash()
+        if msg_hash in seen: continue
+        seen.add(msg_hash)
+        ret.append(msg)
+    for msg in parsed_messages:
+        msg_hash = msg.hash()
+        if msg_hash in seen: continue
+        seen.add(msg_hash)
+        ret.append(msg)
+    return ret
+
+def get_agent_framework(trace):
+    """
+    Supported frameworks:
+    - OpenInference
+        - Pydantic AI
+        - Langflow
+        - Langgraph Agent
+    - LangFuse
+        - Langflow
+        - LangGraph Agent
+    """
+    md_attrs = trace.metadata.get('attributes', {})
+    scope = trace.metadata.get('scope', {})
+    scope_name = scope.get('name', '')
+
+    if scope_name == 'langfuse-sdk':
+        if trace.name == 'LangGraph': return 'langfuse.langgraph_agent'
+
+    if 'langflow.project.name' in md_attrs.keys(): return 'langflow'
+    if 'pydantic-ai' in scope_name: return 'pydantic_ai'
+    if 'openinference.instrumentation.langchain' in scope_name:
+        # TODO: need to find a better way to detect Langgraph Agent
+        return 'langgraph_agent'
+
+    # check for langflow
+    # get observations for trace
+    observations = dfs_(get_observations(trace.observations))
+    for obs in observations:
+        if 'from_langflow_component' in obs.obs.metadata.keys(): return 'langfuse.langflow'
+    return "UNKNOWN"
+
+
+def get_traces(session_id):
+    traces = langfuse_client.api.trace.list(
+        session_id=session_id,
+        limit=100,
+    )
+    # sort by timestamp
+    traces.data.sort(key=lambda x: x.timestamp)
+    return traces
+
+
+def get_observations(observations_ids):
+    observations = [langfuse_client.api.observations.get(obs_id) for obs_id in observations_ids]
+    observations.sort(key=lambda x: x.start_time)
+    observation_tree = build_observation_forest(observations)
+    return observation_tree
+
+class ObsNode:
+    def __init__(self, obs: Any):
+        self.obs = obs
+        self.children: List["ObsNode"] = []
+        self.parent: Optional["ObsNode"] = None
+
+    def __repr__(self):
+        return f"ObsNode(id={self.obs.id}, type={self.obs.type}, name={self.obs.name})"
+
+def build_observation_forest(observations: List[Any]) -> List[ObsNode]:
+    """Return list of root nodes; each has .children forming a tree."""
+    nodes: Dict[str, ObsNode] = {}
+    children_by_parent: Dict[Optional[str], List[ObsNode]] = defaultdict(list)
+
+    # 1. Create nodes for each observation
+    for o in observations:
+        node = ObsNode(o)
+        nodes[o.id] = node
+        parent_id = getattr(o, "parent_observation_id", None)
+        children_by_parent[parent_id].append(node)
+
+    # 2. Attach children to parents
+    for parent_id, child_nodes in children_by_parent.items():
+        if parent_id is None:
+            continue
+        parent_node = nodes.get(parent_id)
+        if parent_node:
+            parent_node.children.extend(child_nodes)
+            for child_node in child_nodes: child_node.parent = parent_node
+
+    # 3. Roots are those with parent_observation_id == None
+    roots = children_by_parent[None]
+    return roots
+
+def dfs_(observation_tree: List[ObsNode]):
+    ret =[]
+    for node in observation_tree:
+        ret.append(node)
+        ret.extend(dfs_(node.children))
+    return ret
+
+
+if __name__ == "__main__":
+    messages = parse_session(session_id="93a24957-dd7a-425d-b821-88de49940a6e")
+
+    print(messages)

wxo_agentic_evaluation/otel_parser/parser_types.py

@@ -0,0 +1,38 @@
+import dataclasses
+from typing import Literal
+from hashlib import md5
+
+# TODO: This is not a fully chat completions message.
+
+
+@dataclasses.dataclass
+class Function:
+    name: str
+    arguments: str
+
+    def __str__(self):
+        return f"{self.name}:{self.arguments}"
+
+
+@dataclasses.dataclass
+class ToolCall:
+    id: str
+    function: "Function"
+    type: Literal["function"] = "function"
+
+    def __str__(self):
+        return f"{self.id}:{self.type}:{self.function}"
+
+
+@dataclasses.dataclass
+class Message:
+    role: Literal["user", "assistant", "tool", "system"]
+    content: str | None = None
+    tool_calls: list["ToolCall"] | None = None
+    tool_call_id: str | None = None
+
+    def __str__(self):
+        return f'{self.role}:{self.content}:{":".join(map(str, self.tool_calls or []))}:{self.tool_call_id}'
+
+    def hash(self):
+        return md5(self.__str__().encode("utf-8")).hexdigest()

wxo_agentic_evaluation/otel_parser/pydantic_parser.py

@@ -0,0 +1,50 @@
+import json
+from wxo_agentic_evaluation.type import OTelParserMessage, OTelParserToolCall, OTelParserFunction, ContentType
+
+
+def get_system_message(trace):
+    sys_instruction_json = trace.metadata.get('attributes', {}).get('gen_ai.system_instructions')
+    if not sys_instruction_json: return None
+    instruction = json.loads(sys_instruction_json)[0]['content']
+    return OTelParserMessage(role='system', content=instruction, type=ContentType.text)
+
+def parse_observations(observation_tree, dfs_observations):
+    messages = []
+    for obs in dfs_observations:
+        if obs.obs.type == 'GENERATION':
+            messages.extend(_get_messages(obs.obs.input))
+            messages.extend(_get_messages(obs.obs.output))
+    return messages
+
+
+def _get_messages(data):
+    messages = []
+    for msg in data:
+        if msg['role'] == 'user':
+            parts = msg['parts']
+            for part in parts:
+                content_type = part['type']
+                if content_type == 'text':
+                    content = part['content']
+                    messages.append(OTelParserMessage(role='user', content=content, type=ContentType.text))
+                elif content_type == 'tool_call_response':
+                    tool_call_id = part['id']
+                    result = json.dumps(part['result'])
+                    messages.append(OTelParserMessage(role='tool', tool_call_id=tool_call_id, content=result, type=ContentType.tool_response))
+        elif msg['role'] == 'assistant':
+            parts = msg['parts']
+            # TODO: assuming that only one message is present in the assistant parts
+            content_type = parts[0]['type']
+            if content_type == 'text':
+                content = parts[0]['content']
+                messages.append(OTelParserMessage(role='assistant', content=content, type=ContentType.text))
+            elif content_type == 'tool_call':
+                tool_calls = []
+                for part in parts:
+                    tool_call_id = part['id']
+                    tool_call_name = part['name']
+                    tool_call_arguments = part['arguments']
+                    tool_call = OTelParserToolCall(id=tool_call_id, function=OTelParserFunction(name=tool_call_name, arguments=tool_call_arguments))
+                    tool_calls.append(tool_call)
+                messages.append(OTelParserMessage(role='assistant', content='', tool_calls=tool_calls, type=ContentType.tool_call))
+    return messages

wxo_agentic_evaluation/otel_parser/utils.py

@@ -0,0 +1,15 @@
+import json
+from .parser_types import ToolCall, Function
+
+def to_tool_call(tool_call):
+    return ToolCall(
+        id=tool_call['id'],
+        type=tool_call['type'],
+        function=to_function(tool_call)
+    )
+
+def to_function(func):
+    return Function(
+        name=func['name'],
+        arguments=json.dumps(func['args'])
+    )

wxo_agentic_evaluation/otel_parser/wxo_parser.py

@@ -0,0 +1,39 @@
+import json
+from wxo_agentic_evaluation.type import OTelParserMessage, OTelParserToolCall, OTelParserFunction, ContentType
+
+def parse_observations(observation_tree, dfs_observations):
+    messages = []
+    for node in dfs_observations:
+        if node.obs.type == 'GENERATION':
+            if node.parent.obs.name == 'invoke_agent':
+                messages.extend(_get_input_messages(node.parent.obs.input))
+                messages.extend(_get_output_message(node.parent.obs.output))
+    return messages
+
+def _get_input_messages(data):
+    messages = []
+    for msg in data['messages']:
+        if msg['type'] == 'system': messages.append(OTelParserMessage(role='system', content=msg['content'], type=ContentType.text))
+        elif msg['type'] == 'human': messages.append(OTelParserMessage(role='user', content=msg['content'], type=ContentType.text))
+        elif msg['type'] == 'tool': messages.append(OTelParserMessage(role='tool', content=msg['content'], tool_call_id=msg['tool_call_id']))
+        elif msg['type'] == 'ai':
+            if msg.get('additional_kwargs', {}).get('tool_calls', None) is not None:
+                msg_tool_calls = msg['additional_kwargs']['tool_calls']
+                tool_calls = []
+                for tc in msg_tool_calls:
+                    tool_calls.append(OTelParserToolCall(id=tc['id'], function=OTelParserFunction(name=tc['function']['name'], arguments=tc['function']['arguments'])))
+            else:
+                tool_calls = None
+            messages.append(OTelParserMessage(role='assistant', content=msg['content'], tool_calls=tool_calls, type=ContentType.tool_call))
+    return messages
+
+
+def _get_output_message(data):
+    role = 'assistant'
+    content = data.get('content', '')
+    if data.get('tool_calls', None):
+        tool_calls = []
+        for tc in data['tool_calls']:
+            tool_calls.append(OTelParserToolCall(id=tc['id'], function=OTelParserFunction(name=tc['name'], arguments=json.dumps(tc.get('args', {})))))
+    else: tool_calls = None
+    return [OTelParserMessage(role='assistant', content=content, tool_calls=tool_calls)]

wxo_agentic_evaluation/otel_support/evaluate_tau.py

@@ -1,8 +1,17 @@
 import json
+
 from wxo_agentic_evaluation.evaluation_package import EvaluationPackage
-from wxo_agentic_evaluation.type import EvaluationData, Message, EventTypes, ContentType
+from wxo_agentic_evaluation.type import (
+    ContentType,
+    EvaluationData,
+    EventTypes,
+    Message,
+)
 
-with open("/Users/haodeqi/git/tau-bench/historical_trajectories/gpt-4o-airline.json", "r") as f:
+with open(
+    "/Users/haodeqi/git/tau-bench/historical_trajectories/gpt-4o-airline.json",
+    "r",
+) as f:
     test_data = json.load(f)
 
 
@@ -14,14 +23,19 @@ goal_details = []
 i = 0
 for action in test_data[0]["info"]["task"]["actions"]:
     goal_temp.append(action["name"] + f"_{i}")
-    goal_detail = {"type": "tool_call", "name": action["name"] + f"_{i}", "tool_name": action["name"], "args": {k: str(v) for k,v in action["kwargs"].items()}}
+    goal_detail = {
+        "type": "tool_call",
+        "name": action["name"] + f"_{i}",
+        "tool_name": action["name"],
+        "args": {k: str(v) for k, v in action["kwargs"].items()},
+    }
     goal_details.append(goal_detail)
 
 if len(goal_temp) == 1:
     goals[goal_temp[0]] = []
 else:
-    for i in range(len(goal_temp)-1):
-        goals.update({goal_temp[i]: goal_temp[i+1]})
+    for i in range(len(goal_temp) - 1):
+        goals.update({goal_temp[i]: goal_temp[i + 1]})
 
 gt_data = {
     "agent": "airline_agent",
@@ -41,10 +55,30 @@ history = []
 for msg in test_data[0]["traj"]:
     if msg["role"] == "tool":
         print(msg["content"])
-        history.append(Message(role=msg["role"], content=json.dumps({"type": "tool_call", "args": json.loads(msg["content"]), "name": msg["name"], "tool_call_id": msg["tool_call_id"]}), type=ContentType.tool_call,
-                               event=EventTypes.message_created))
+        history.append(
+            Message(
+                role=msg["role"],
+                content=json.dumps(
+                    {
+                        "type": "tool_call",
+                        "args": json.loads(msg["content"]),
+                        "name": msg["name"],
+                        "tool_call_id": msg["tool_call_id"],
+                    }
+                ),
+                type=ContentType.tool_call,
+                event=EventTypes.message_created,
+            )
+        )
     else:
-        history.append(Message(role=msg["role"], content=str(msg["content"]), type=ContentType.text, event=EventTypes.message_created))
+        history.append(
+            Message(
+                role=msg["role"],
+                content=str(msg["content"]),
+                type=ContentType.text,
+                event=EventTypes.message_created,
+            )
+        )
 
 print(f"length of history {history}")
 
@@ -53,7 +87,7 @@ evaluation_package = EvaluationPackage(
     messages=history,
     ground_truth=gt_data,
     conversational_search_data=None,
-    resource_map=None
+    resource_map=None,
 )
 print("1")
 (
@@ -64,4 +98,4 @@ print("1")
 ) = evaluation_package.generate_summary()
 
 
-print(metrics)
+print(metrics)

wxo_agentic_evaluation/otel_support/otel_message_conversion.py

@@ -1,6 +1,7 @@
-from typing import Any, Dict, List, Union, Optional
-from wxo_agentic_evaluation.type import Message, ContentType, EventTypes
 import json
+from typing import Any, Dict, List, Optional, Union
+
+from wxo_agentic_evaluation.type import ContentType, EventTypes, Message
 
 # with open("src/wxo_agentic_evaluation/otel_support/collie_example.json", "r") as f:
 #     data = json.load(f)
@@ -16,6 +17,13 @@ def convert_otel_to_message(otel_traces):
         print(row.keys())
         role = row.get("role", "assistant")
 
-        history.append(Message(role = role, content= content, type=ContentType.text, event=EventTypes.message_created))
+        history.append(
+            Message(
+                role=role,
+                content=content,
+                type=ContentType.text,
+                event=EventTypes.message_created,
+            )
+        )
 
-    return history
+    return history