ibm-watsonx-orchestrate-evaluation-framework 1.0.2__py3-none-any.whl → 1.0.4__py3-none-any.whl

wxo_agentic_evaluation/prompt/starting_sentence_generation_prompt.jinja2

@@ -0,0 +1,195 @@
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+Your task is to generate a starting sentence provided a user story.
+
+## Generation Guidelines
+The starting sentence is the first question a user would ask in order to achieve the provided story.
+The starting sentence shouldn't be the same as the story. The story is the objective of the conversation, while the starting sentence is the initial question a user would ask in order to achieve that objective.
+The generated starting sentence should be a concise, no longer than one sentence, and in first person.
+
+## Output:
+Respond in a JSON format with the following fields:
+- starting_sentence: this is the generated starting sentence 
+
+This is an example of a valid JSON output
+
+{
+    "starting_sentence": "placeholder text",
+}
+
+--
+
+<|start_header_id|>user<|end_header_id|>
+Input:
+{
+  "agent": "workday_employee_support_manager",
+  "goals": {
+    "get_user_workday_ids": [
+      "get_payslips"
+    ],
+    "get_payslips": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "get_user_workday_ids",
+      "tool_name": "get_user_workday_ids",
+      "args": {
+        "email": "abrennan@workday.net"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "get_payslips",
+      "tool_name": "get_payslips",
+      "args": {
+        "user_id": "6dcb8106e8b74b5aabb1fc3ab8ef2b92"
+      }
+    },
+    {
+      "type": "text",
+      "name": "summarize",
+      "response": "No payslips found for user with ID 6dcb8106e8b74b5aabb1fc3ab8ef2b92.",
+      "keywords": [
+        "payslips"
+      ]
+    }
+  ],
+  "story": "Your email id is abrennan@workday.net. You want to get your payslips",
+}
+<|eot_id|>
+
+<|start_header_id|>assistant<|end_header_id|>
+{"starting_sentence": "I work at Workday, I want to get my payslips. My email id is abrennan@workday.net"}
+<|eot_id|>
+
+
+<|start_header_id|>user<|end_header_id|>
+Input:
+{
+  "agent": "servicenow_asset_management_agent",
+  "goals": {
+    "get_assets": [
+      "update_an_asset"
+    ],
+    "update_an_asset": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "get_assets",
+      "tool_name": "get_assets",
+      "args": {}
+    },
+    {
+      "type": "tool_call",
+      "name": "update_an_asset",
+      "tool_name": "update_an_asset",
+      "args": {
+        "args": {
+          "current_name": "A100",
+          "new_name": "Macbook"
+        }
+      }
+    },
+    {
+      "name": "summarize",
+      "type": "text",
+      "response": "The asset A100 has been updated to Macbook.",
+      "keywords": [
+        "A100",
+        "Macbook"
+      ]
+    }
+  ],
+  "story": "You want to update an asset. The current asset name is A100. You want to update it to Macbook.",
+}
+<|eot_id|>
+<|start_header_id|>assistant<|end_header_id|>
+{"starting_sentence": "I want to update an asset"}
+<|eot_id|>
+
+<|start_header_id|>user<|end_header_id|>
+Input:
+{
+  "agent": "sales_research",
+  "goals": {
+    "search_company_by_typeahead": [
+      "get_news_and_media"
+    ],
+    "get_news_and_media": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "search_company_by_typeahead",
+      "tool_name": "search_company_by_typeahead",
+      "args": {
+        "query": "tech"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "get_news_and_media",
+      "tool_name": "get_news_and_media",
+      "args": {
+        "entity_name": "TechNova"
+      }
+    },
+    {
+      "name": "summarize",
+      "type": "text",
+      "response": "Here is the latest news on the tech company:\n\n| Company Name | Headline | Source | Date | Summary |\n| --- | --- | --- | --- | --- |\n| TechNova | TechNova Launches AI-driven Sales Platform | TechCrunch | 2025-06-15 | TechNova announced its new AI platform aimed at transforming enterprise sales operations. |",
+      "keywords": [
+        "TechNova",
+        "AI-driven Sales Platform",
+        "TechCrunch",
+        "2025-06-15"
+      ]
+    }
+  ],
+  "story": "You need the latest news on a company. The company name is TechNova.",
+}
+<|eot_id|>
+<|start_header_id|>assistant<|end_header_id|>
+{"starting_sentence": "I need the latest news on a company"}
+<|eot_id|>
+
+<|start_header_id|>user<|end_header_id|>
+Input:
+{
+    "agent": "question_answer_agent",
+    "goals": {
+        "summarize": []
+    },
+    "goal_details": [
+        {
+            "name": "summarize",
+            "type": "text",
+            "response": "The current PM of England is Keir Starmer",
+            "keywords": [
+                "Keir Starmer"
+            ]
+        }
+    ],
+    "story": "You want to know who the PM of England is.",
+}
+<|eot_id|>
+
+<|start_header_id|>assistant<|end_header_id|>
+{"starting_sentence": "Who is the PM of the England?"}
+<|eot_id|>
+
+---
+
+<|start_header_id|>assistant<|end_header_id|>
+# Now, evaluate the following:
+Input:
+{{ input_data }}
+Starting sentence:
+

wxo_agentic_evaluation/prompt/story_generation_prompt.jinja2

@@ -0,0 +1,154 @@
+<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+Your task is to generate a user story based on the provided information, focusing solely on the user's end goal and the essential context. Write a clear, concise narrative that:
+
+- Centers on the user's primary objective and the key actions they take to achieve it.
+- Includes only information directly relevant to understanding the user's goals and context.
+- Excludes intermediate steps or process details that do not directly impact the user's outcome.
+- Avoids repetition and omits any duplicate or unnecessary details.
+- Captures the essence of the user's intent and required actions without commentary or extraneous explanation.
+
+The final story should be brief, direct, and goal-oriented.
+<|eot_id|>
+
+<|start_header_id|>user<|end_header_id|>
+Here area a few examples of how to generate a user story based on the provided input data:
+# Example 1:
+Input data:
+{
+  "agent": "workday_employee_support_manager",
+  "goals": {
+    "get_user_workday_ids": [
+      "get_payslips"
+    ],
+    "get_payslips": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "get_user_workday_ids",
+      "tool_name": "get_user_workday_ids",
+      "args": {
+        "email": "abrennan@workday.net"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "get_payslips",
+      "tool_name": "get_payslips",
+      "args": {
+        "user_id": "6dcb8106e8b74b5aabb1fc3ab8ef2b92"
+      }
+    },
+    {
+      "type": "text",
+      "name": "summarize",
+      "response": "No payslips found for user with ID 6dcb8106e8b74b5aabb1fc3ab8ef2b92.",
+      "keywords": [
+        "payslips"
+      ]
+    }
+  ],
+  "story": "",
+  "starting_sentence": "I work at Workday, I want to get my payslips. My email id is abrennan@workday.net"
+}
+Story: Your email id is abrennan@workday.net. You want to get your payslips
+
+# Example 2:
+Input data:
+{
+  "agent": "servicenow_asset_management_agent",
+  "goals": {
+    "get_assets": [
+      "update_an_asset"
+    ],
+    "update_an_asset": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "get_assets",
+      "tool_name": "get_assets",
+      "args": {}
+    },
+    {
+      "type": "tool_call",
+      "name": "update_an_asset",
+      "tool_name": "update_an_asset",
+      "args": {
+        "args": {
+          "current_name": "A100",
+          "new_name": "Macbook"
+        }
+      }
+    },
+    {
+      "name": "summarize",
+      "type": "text",
+      "response": "The asset A100 has been updated to Macbook.",
+      "keywords": [
+        "A100",
+        "Macbook"
+      ]
+    }
+  ],
+  "story": "",
+  "starting_sentence": "I want to update an asset"
+}
+Story: You want to update an asset. The current asset name is A100. You want to update it to Macbook.
+
+# Example 3:
+Input data:
+{
+  "agent": "sales_research",
+  "goals": {
+    "search_company_by_typeahead": [
+      "get_news_and_media"
+    ],
+    "get_news_and_media": [
+      "summarize"
+    ]
+  },
+  "goal_details": [
+    {
+      "type": "tool_call",
+      "name": "search_company_by_typeahead",
+      "tool_name": "search_company_by_typeahead",
+      "args": {
+        "query": "tech"
+      }
+    },
+    {
+      "type": "tool_call",
+      "name": "get_news_and_media",
+      "tool_name": "get_news_and_media",
+      "args": {
+        "entity_name": "TechNova"
+      }
+    },
+    {
+      "name": "summarize",
+      "type": "text",
+      "response": "Here is the latest news on the tech company:\n\n| Company Name | Headline | Source | Date | Summary |\n| --- | --- | --- | --- | --- |\n| TechNova | TechNova Launches AI-driven Sales Platform | TechCrunch | 2025-06-15 | TechNova announced its new AI platform aimed at transforming enterprise sales operations. |",
+      "keywords": [
+        "TechNova",
+        "AI-driven Sales Platform",
+        "TechCrunch",
+        "2025-06-15"
+      ]
+    }
+  ],
+  "story": "",
+  "starting_sentence": "I need the latest news on a company"
+}
+Story: You need the latest news on a company. The company name is TechNova.
+
+<|eot_id|>
+<|start_header_id|>assistant<|end_header_id|>
+# Now, evaluate the following:
+Input data:
+{{ input_data }}
+Story:

wxo_agentic_evaluation/prompt/template_render.py

@@ -61,6 +61,14 @@ class ToolPlannerTemplateRenderer(JinjaTemplateRenderer):
             agent_name=agent_name,
             available_tools=available_tools,
         )
+    
+class ArgsExtractorTemplateRenderer(JinjaTemplateRenderer):
+    def render(self, tool_signature: str, step: dict, inputs: dict) -> str:
+        return super().render(
+            tool_signature=tool_signature,
+            step=step,
+            inputs=inputs,
+        )
 
 class ToolChainAgentTemplateRenderer(JinjaTemplateRenderer):
     def render(self, tool_call_history: List, available_tools:str) -> str:
@@ -88,3 +96,12 @@ class BatchTestCaseGeneratorTemplateRenderer(JinjaTemplateRenderer):
             num_variants=num_variants,
             example_str=example_str,
         )
+
+class StoryGenerationTemplateRenderer(JinjaTemplateRenderer):
+    def render(
+        self,
+        input_data: dict,
+    ) -> str:
+        return super().render(
+            input_data=input_data,
+        )

wxo_agentic_evaluation/prompt/tool_planner.jinja2

@@ -1,7 +1,7 @@
 <|begin_of_text|><|start_header_id|>system<|end_header_id|>
 You are a tool-planning assistant for an AI system.
 
-Your job is to extract and sequence tool calls based on user stories and available tools.
+Your job is to generate a sequence of tool calls based on user stories and available tools.
 
 Rules:
 - Use only the tools listed below.
@@ -11,12 +11,16 @@ Rules:
 - Output ONLY one valid JSON array.
 - DO NOT include extra text or wrap the output. Just return the JSON list.
 
-Available Tools:
-{{ available_tools }}
+<|eot_id|>
+
+<|start_header_id|>user<|end_header_id|>
+Here area a few examples of how to generate a squence of tool calls based on user stories and available tools.
 
 Example:
 Story: "Your username is nwaters. You want to find out your time-off schedule from: 2025-01-01 to: 2025-12-31."
-
+Available Tools:
+[{'Function Name': '_is_valid_date', 'Arguments': ['date_str'], 'Docstring': 'Check if the provided date string is in YYYY-MM-DD format.'}, {'Function Name': 'fetch_assignment_id', 'Arguments': ['username'], 'Docstring': "Return the assignment ID for a given employee username.\n\n:param username: Employee's username\n:return: Assignment ID as a string or 'not found'"}, {'Function Name': 'retrieve_timeoff_schedule', 'Arguments': ['assignment_id', 'start_date', 'end_date'], 'Docstring': 'Get time-off schedule for an employee within a given date range.\n\n:param assignment_id: Assignment ID of the employee\n:param start_date: Start date in YYYY-MM-DD format\n:param end_date: End date in YYYY-MM-DD format\n:return: JSON list of time-off dates or error message'}, {'Function Name': 'list_direct_reports', 'Arguments': ['manager_assignment_id'], 'Docstring': "Retrieve the list of direct report Employee's username for a specified manager's assignment ID.\n\n:param manager_assignment_id: Assignment ID of the manager as a string\n:return: JSON-encoded list of Employee's username who report to the manager"}, {'Function Name': 'get_address_type', 'Arguments': ['address_type_name'], 'Docstring': 'Retrieve a string address type ID based on a given address type name.\n\n:param address_type_name: Address type as a string\n:return: Corresponding string address_type_id or -1 if not found'}, {'Function Name': 'update_address', 'Arguments': ['address_type_id', 'assignment_id', 'new_address'], 'Docstring': "Update the address for an employee based on assignment ID and address type.\n\n:param address_type_id: String representing the address type\n:param assignment_id: Employee's assignment ID\n:param new_address: New address string\n:return: Success or error message"}]
+Result:
 [
   {
     "tool_name": "fetch_assignment_id",
@@ -34,7 +38,9 @@ Story: "Your username is nwaters. You want to find out your time-off schedule fr
   }
 ]
 <|eot_id|>
-<|start_header_id|>user<|end_header_id|>
-Story: "{{ user_story }}"
-<|eot_id|>
 <|start_header_id|>assistant<|end_header_id|>
+# Now, generate results for the following user story and available tools:
+Story: "{{ user_story }}"
+Available Tools:
+{{ available_tools }}
+Result:

wxo_agentic_evaluation/record_chat.py

@@ -11,19 +11,25 @@ from wxo_agentic_evaluation.inference_backend import (
 from wxo_agentic_evaluation.data_annotator import DataAnnotator
 from wxo_agentic_evaluation.utils.utils import is_saas_url
 from wxo_agentic_evaluation.service_instance import tenant_setup
+from wxo_agentic_evaluation.prompt.template_render import StoryGenerationTemplateRenderer
+from wxo_agentic_evaluation.service_provider import get_provider
+from wxo_agentic_evaluation import __file__
 
 import json
 import os
 import rich
 from datetime import datetime
 import time
-from typing import List
+from typing import List, Dict
+import hashlib
 from jsonargparse import CLI
 import warnings
 
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 warnings.filterwarnings("ignore", category=FutureWarning)
 
+root_dir = os.path.dirname(__file__)
+STORY_GENERATION_PROMPT_PATH = os.path.join(root_dir, "prompt", "story_generation_prompt.jinja2")
 
 def get_all_runs(wxo_client: WXOClient):
     limit = 20  # Maximum allowed limit per request
@@ -31,9 +37,11 @@ def get_all_runs(wxo_client: WXOClient):
     all_runs = []
 
     if is_saas_url(wxo_client.service_url):
-        path = "v1//orchestrate/runs"
+        # TO-DO: this is not validated after the v1 prefix change
+        # need additional validation
+        path = "v1/orchestrate/runs"
     else:
-        path = "/orchestrate/runs"
+        path = "v1/orchestrate/runs"
 
     initial_response = wxo_client.get(
         path, {"limit": limit, "offset": 0}
@@ -62,34 +70,60 @@ def get_all_runs(wxo_client: WXOClient):
     return all_runs
 
 
-def pull_messages_from_thread_id(thread_id: str, wxo_client: WXOClient):
-    inference_backend = WXOInferenceBackend(wxo_client=wxo_client)
-    messages = inference_backend.get_messages(thread_id)
-    return messages
+def generate_story(annotated_data: dict):
+    renderer = StoryGenerationTemplateRenderer(STORY_GENERATION_PROMPT_PATH)
+    provider = get_provider(
+        model_id="meta-llama/llama-3-405b-instruct",
+        params={"min_new_tokens": 0, "decoding_method": "greedy", "max_new_tokens": 256},
+    )
+    prompt = renderer.render(input_data=json.dumps(annotated_data, indent=2))
+    res = provider.query(prompt)
+    return res.strip()
 
 
 def annotate_messages(
-    messages: List[Message], keywords_generation_config: KeywordsGenerationConfig
+    agent_name: str, messages: List[Message], keywords_generation_config: KeywordsGenerationConfig
 ):
     annotator = DataAnnotator(
         messages=messages, keywords_generation_config=keywords_generation_config
     )
-    return annotator.generate()
+    annotated_data = annotator.generate()
+    if agent_name is not None:
+        annotated_data["agent"] = agent_name
+
+    annotated_data["story"] = generate_story(annotated_data)
+    
+    return annotated_data
+
+def has_messages_changed(
+    thread_id: str,
+    messages: List[Message],
+    previous_hashes: Dict[str, str],
+) -> bool:
+    # serialize just the message content
+    payload = [msg.model_dump() for msg in messages]
+    sig = json.dumps(payload, sort_keys=True, default=str)
+    h = hashlib.sha256(sig.encode()).hexdigest()
+
+    if previous_hashes.get(thread_id) != h:
+        previous_hashes[thread_id] = h
+        return True
+    return False
 
 
 def record_chats(config: ChatRecordingConfig):
     """Record chats in background mode"""
     start_time = datetime.utcnow()
     processed_threads = set()
+    previous_input_hash: dict[str, str] = {}
 
     rich.print(
         f"[green]INFO:[/green] Starting chat recording at {start_time}. Press Ctrl+C to stop."
     )
     if config.token is None:
-        token = tenant_setup(config.service_url, config.tenant_name)
-    else:
-        token = config.token
-    wxo_client = get_wxo_client(config.service_url, token)
+        config.token = tenant_setup(config.service_url, config.tenant_name)
+    wxo_client = get_wxo_client(config.service_url, config.tenant_name, config.token)
+    inference_backend = WXOInferenceBackend(wxo_client=wxo_client)
     try:
         while True:
             all_runs = get_all_runs(wxo_client)
@@ -98,7 +132,12 @@ def record_chats(config: ChatRecordingConfig):
             # Process only new runs that started after our recording began
             for run in all_runs:
                 thread_id = run.get("thread_id")
-                if thread_id in seen_threads:
+                try:
+                    agent_name = inference_backend.get_agent_name_from_thread_id(thread_id)
+                except Exception as e:
+                    rich.print(f"[yellow]WARNING:[/yellow]Failure in getting thread id {thread_id}")
+                    continue
+                if thread_id in seen_threads or agent_name is None:
                     continue
                 seen_threads.add(thread_id)
                 started_at = run.get("started_at")
@@ -119,33 +158,42 @@ def record_chats(config: ChatRecordingConfig):
                             rich.print(
                                 f"[green]INFO:[/green] Messages saved to: {os.path.join(config.output_dir, f'{thread_id}_messages.json')}"
                             )
-                            rich.print(
-                                f"[green]INFO:[/green] Annotations saved to: {os.path.join(config.output_dir, f'{thread_id}_annotated_data.json')}"
-                            )
+                            # rich.print(
+                            #     f"[green]INFO:[/green] Annotations saved to: {os.path.join(config.output_dir, f'{thread_id}_annotated_data.json')}"
+                            # )
                         processed_threads.add(thread_id)
 
                         try:
-                            messages = pull_messages_from_thread_id(
-                                thread_id, wxo_client
-                            )
+                            messages = inference_backend.get_messages(thread_id)
+
+                            if not has_messages_changed(
+                                thread_id,
+                                messages,
+                                previous_input_hash,
+                            ):
+                                continue
+                            
                             annotated_data = annotate_messages(
-                                messages, config.keywords_generation_config
+                                agent_name, messages, config.keywords_generation_config
                             )
 
                             messages_filename = os.path.join(
                                 config.output_dir, f"{thread_id}_messages.json"
                             )
-                            annotation_filename = os.path.join(
-                                config.output_dir, f"{thread_id}_annotated_data.json"
-                            )
 
                             with open(messages_filename, "w") as f:
                                 json.dump(
                                     [msg.model_dump() for msg in messages], f, indent=4
                                 )
 
-                            with open(annotation_filename, "w") as f:
-                                json.dump(annotated_data, f, indent=4)
+                            # TO-DO: we want some tracing but we also do not want to persist the file
+                            # in the same folder.
+                            # annotation_filename = os.path.join(
+                            #     config.output_dir, f"{thread_id}_annotated_data.json"
+                            # )
+
+                            # with open(annotation_filename, "w") as f:
+                            #     json.dump(annotated_data, f, indent=4)
                         except Exception as e:
                             rich.print(
                                 f"[red]ERROR:[/red] Failed to process thread {thread_id}: {str(e)}"

wxo_agentic_evaluation/resource_map.py

@@ -0,0 +1,47 @@
+from collections import defaultdict
+from wxo_agentic_evaluation.inference_backend import WXOClient, is_saas_url
+
+
+class ResourceMap:
+    def __init__(self, wxo_client: WXOClient):
+        self.wxo_client = wxo_client
+        self.agent2tools, self.tools2agents = self.init_mapping()
+        self.all_agents = list(self.agent2tools.keys())
+
+    def init_mapping(self):
+        agent2tools = defaultdict(set)
+        tools2agents = defaultdict(set)
+        if is_saas_url(self.wxo_client.service_url):
+            # TO-DO: this is not validated after the v1 prefix change
+            # need additional validation
+            tools_path = "v1/orchestrate/tools/"
+            agents_path = "v1/orchestrate/agents"
+        else:
+            tools_path = "v1/tools/"
+            agents_path = "v1/orchestrate/agents/"
+
+        tool_map = {}
+
+        resp = self.wxo_client.get(tools_path)
+        if resp.status_code == 200:
+            tools = resp.json()
+            tool_map = {tool["id"]: tool["name"] for tool in tools}
+        else:
+            resp.raise_for_status()
+
+        resp = self.wxo_client.get(agents_path)
+
+        if resp.status_code == 200:
+            agents = resp.json()
+            for agent in agents:
+                agent_name = agent["name"]
+                tools = [tool_map[id] for id in agent["tools"]]
+                for tool in tools:
+                    agent2tools[agent_name].add(tool)
+                    tools2agents[tool].add(agent_name)
+        else:
+            resp.raise_for_status()
+
+        agent2tools = dict(agent2tools)
+        tools2agents = dict(tools2agents)
+        return agent2tools, tools2agents

wxo_agentic_evaluation/service_provider/__init__.py

@@ -0,0 +1,35 @@
+from wxo_agentic_evaluation.service_provider.ollama_provider import OllamaProvider
+from wxo_agentic_evaluation.service_provider.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.service_provider.model_proxy_provider import ModelProxyProvider
+from wxo_agentic_evaluation.arg_configs import ProviderConfig
+
+import os
+
+def _instantiate_provider(config: ProviderConfig, **kwargs):
+    if config.provider == "watsonx":
+        return WatsonXProvider(model_id=config.model_id, **kwargs)
+    elif config.provider == "ollama":
+        return OllamaProvider(model_id=config.model_id, **kwargs)
+    elif config.provider == "model_proxy":
+        return ModelProxyProvider(model_id=config.model_id, **kwargs)
+    else:
+        raise RuntimeError(f"target provider is not supported {config.provider}")
+    
+def get_provider(config: ProviderConfig = None, model_id: str = None, **kwargs):
+    if config:
+        return _instantiate_provider(config, **kwargs)
+
+    if not model_id:
+        raise ValueError("model_id must be provided if config is not supplied")
+
+    if "WATSONX_APIKEY" in os.environ and "WATSONX_SPACE_ID" in os.environ:
+        config = ProviderConfig(provider="watsonx", model_id=model_id)
+        return _instantiate_provider(config, **kwargs)
+
+    if "WO_API_KEY" in os.environ and "WO_INSTANCE" in os.environ:
+        config = ProviderConfig(provider="model_proxy", model_id=model_id)
+        return _instantiate_provider(config, **kwargs)
+
+    raise RuntimeError(
+        "No provider found. Please either provide a config or set the required environment variables."
+    )