ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/external_agent/__init__.py

@@ -1,21 +1,28 @@
 import importlib.resources
 import json
+
 import rich
 
-from wxo_agentic_evaluation.prompt.template_render import StoryGenerationTemplateRenderer
-from wxo_agentic_evaluation.service_provider import get_provider, ProviderConfig
 from wxo_agentic_evaluation import prompt
+from wxo_agentic_evaluation.prompt.template_render import (
+    StoryGenerationTemplateRenderer,
+)
+from wxo_agentic_evaluation.service_provider import get_provider
 
 console = rich.console.Console()
 
+
 def starting_sentence_generation_prompt():
-    with importlib.resources.path(prompt, "starting_sentence_generation_prompt.jinja2") as fp:
+    with importlib.resources.path(
+        prompt, "starting_sentence_generation_prompt.jinja2"
+    ) as fp:
         # reuse the StoryGenerationTemplateRenderer class, even though we are generating a "starting_sentence" instead of a "story"
         # the starting sentence generation prompts uses the same input variable
         render = StoryGenerationTemplateRenderer(str(fp))
-    
+
     return render
 
+
 def generate_starting_sentence(annotated_data: dict):
     renderer = starting_sentence_generation_prompt()
     llm_decode_parameter = {
@@ -23,7 +30,9 @@ def generate_starting_sentence(annotated_data: dict):
         "decoding_method": "greedy",
         "max_new_tokens": 4096,
     }
-    wai_client = get_provider(config=ProviderConfig(), params=llm_decode_parameter)
+    wai_client = get_provider(
+        model_id="meta-llama/llama-3-405b-instruct", params=llm_decode_parameter
+    )
     prompt = renderer.render(input_data=json.dumps(annotated_data, indent=4))
     res = wai_client.query(prompt)
     res = res.strip()
@@ -33,5 +42,7 @@ def generate_starting_sentence(annotated_data: dict):
         res = json.loads(res)
         return res["starting_sentence"]
     except Exception:
-        console.log(f"The generated `starting_sentence` had incorrect format: '{res}'")
-        return res
+        console.log(
+            f"The generated `starting_sentence` had incorrect format: '{res}'"
+        )
+        return res

wxo_agentic_evaluation/external_agent/external_validate.py

@@ -1,15 +1,21 @@
+import json
 from typing import Generator
+
 import requests
-import json
 import rich
 
-from wxo_agentic_evaluation.external_agent.types import UniversalData, SchemaValidationResults
-
+from wxo_agentic_evaluation.external_agent.types import (
+    SchemaValidationResults,
+    UniversalData,
+)
 
 MESSAGES = [
     {"role": "user", "content": "what's the holiday is June 13th in us?"},
-    {"role": "assistant", "content": "tool_name: calendar_lookup, args {\"location\": \"USA\", \"data\": \"06-13-2025\"}}"},
-    {"role": "assistant", "content":"it's National Sewing Machine Day"}
+    {
+        "role": "assistant",
+        "content": 'tool_name: calendar_lookup, args {"location": "USA", "data": "06-13-2025"}}',
+    },
+    {"role": "assistant", "content": "it's National Sewing Machine Day"},
 ]
 
 
@@ -18,7 +24,7 @@ class ExternalAgentValidation:
         self.credential = credential
         self.auth_scheme = auth_scheme
         self.service_url = service_url
-    
+
     @property
     def header(self):
         header = {"Content-Type": "application/json"}
@@ -32,23 +38,23 @@ class ExternalAgentValidation:
         return header
 
     def _parse_streaming_events(self, resp: Generator[bytes, None, None]):
-        data = b''
+        data = b""
         for chunk in resp:
             for line in chunk.splitlines(True):
-                if line.startswith(b'data:'):
-                    line = line.replace(b'data:', b'')
-                if line.strip() == b'[DONE]':
-                    return
+                if line.startswith(b"event:"):
+                    continue
                 data += line
-                if data.endswith((b'\r\r', b'\n\n', b'\r\n\r\n')):
+                if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")):
                     # NOTE: edge case, "data" can be sent in two different chunks
-                    if data.startswith(b'data:'):
-                        data = data.replace(b'data:', b'')
+                    if data.startswith(b"data:"):
+                        data = data.replace(b"data:", b"")
+                    if data.strip() == b"[DONE]":
+                        return
                     yield data
-                    data = b''
+                    data = b""
         if data:
             yield data
-    
+
     def _validate_streaming_response(self, resp):
         success = True
         logged_events = []
@@ -61,52 +67,59 @@ class ExternalAgentValidation:
             except Exception as e:
                 success = False
                 break
-                
+
         return success, logged_events
 
     def _validate_schema_compliance(self, messages):
         payload = {"stream": True}
         payload["messages"] = messages
-        resp = requests.post(url=self.service_url, headers=self.header, json=payload)
+        resp = requests.post(
+            url=self.service_url,
+            headers=self.header,
+            json=payload,
+        )
         success, logged_events = self._validate_streaming_response(resp)
 
         msg = ", ".join([msg["content"] for msg in payload["messages"]])
 
         if success:
-            rich.print(f":white_check_mark: External Agent streaming response validation succeeded for '{msg}'.")
+            rich.print(
+                f":white_check_mark: External Agent streaming response validation succeeded for '{msg}'."
+            )
         else:
-            rich.print(f":heavy_exclamation_mark:Schema validation failed for messages: '{msg}':heavy_exclamation_mark:\n The last logged event was {logged_events[-1]}.\n")
+            rich.print(
+                f":heavy_exclamation_mark:Schema validation failed for messages: '{msg}':heavy_exclamation_mark:\n The last logged event was {logged_events[-1]}.\n"
+            )
 
         return success, logged_events
 
-    def call_validation(self, input_str: str, add_context: bool = False) -> SchemaValidationResults:
+    def call_validation(
+        self, input_str: str, add_context: bool = False
+    ) -> SchemaValidationResults:
         if add_context:
             return self.block_validation(input_str)
 
-        msg = {
-            "role": "user",
-            "content": input_str
-        }
-       
+        msg = {"role": "user", "content": input_str}
+
         success, logged_events = self._validate_schema_compliance([msg])
-        results = SchemaValidationResults(success=success, logged_events=logged_events, messages=[msg])
+        results = SchemaValidationResults(
+            success=success, logged_events=logged_events, messages=[msg]
+        )
 
         return results.model_dump()
-    
+
     def block_validation(self, input_str: str) -> SchemaValidationResults:
-        """ Tests a block of messages
-        """
+        """Tests a block of messages"""
         rich.print(
             f"[gold3]The following prebuilt messages, '{MESSAGES}' is prepended to the input message, '{input_str}'"
         )
 
-        msg = {
-            "role": "user",
-            "content": input_str
-        }
+        msg = {"role": "user", "content": input_str}
 
         messages = MESSAGES + [msg]
         success, logged_events = self._validate_schema_compliance(messages)
-        results = SchemaValidationResults(success=success, logged_events=logged_events, messages=messages)
+        results = SchemaValidationResults(
+            success=success, logged_events=logged_events, messages=messages
+        )
 
-        return results.model_dump()
+        return results.model_dump()

wxo_agentic_evaluation/external_agent/performance_test.py

@@ -1,10 +1,15 @@
-from typing import List, Mapping, Any
+from typing import Any, List, Mapping
+
 from rich.console import Console
 
-from wxo_agentic_evaluation.external_agent import generate_starting_sentence
 from wxo_agentic_evaluation.arg_configs import KeywordsGenerationConfig
-from wxo_agentic_evaluation.service_provider import get_provider, ProviderConfig
-from wxo_agentic_evaluation.data_annotator import KeywordsGenerationLLM, LlamaKeywordsGenerationTemplateRenderer
+from wxo_agentic_evaluation.data_annotator import (
+    KeywordsGenerationLLM,
+    LlamaKeywordsGenerationTemplateRenderer,
+)
+from wxo_agentic_evaluation.external_agent import generate_starting_sentence
+from wxo_agentic_evaluation.service_provider import get_provider
+
 
 class ExternalAgentPerformanceTest:
     def __init__(self, agent_name: str, test_data: List[str]):
@@ -12,55 +17,61 @@ class ExternalAgentPerformanceTest:
         self.goal_template = {
             "agent": agent_name,
             "goals": {"summarize": []},
-            "goal_details": [
-            ],
+            "goal_details": [],
             "story": "<placeholder>",
         }
 
         kw_gen_config = KeywordsGenerationConfig()
 
-        provider_config = ProviderConfig(model_id=kw_gen_config.model_id)
         llm_decode_parameter = {
             "min_new_tokens": 0,
             "decoding_method": "greedy",
             "max_new_tokens": 256,
         }
-        wai_client = get_provider(config=provider_config, params=llm_decode_parameter)
-        
+        wai_client = get_provider(
+            model_id=kw_gen_config.model_id, params=llm_decode_parameter
+        )
+
         self.kw_gen = KeywordsGenerationLLM(
             provider=wai_client,
             template=LlamaKeywordsGenerationTemplateRenderer(
                 kw_gen_config.prompt_config
-                ),
-            )
-    
+            ),
+        )
+
     def generate_tests(self) -> List[Mapping[str, Any]]:
         console = Console()
         goal_templates = []
 
-        with console.status("[gold3]Creating starting sentence for user story from input file for performance testing") as status:
+        with console.status(
+            "[gold3]Creating starting sentence for user story from input file for performance testing"
+        ) as status:
             for sentence, response in self.test_data:
                 goal_temp = self.goal_template.copy()
                 goal_temp["story"] = sentence
 
                 keywords = self.kw_gen.genereate_keywords(response)
                 summarize_step = {
-                        "name": "summarize",
-                        "type": "text",
-                        "response": response,
-                        "keywords": keywords
-                    }
+                    "name": "summarize",
+                    "type": "text",
+                    "response": response,
+                    "keywords": keywords,
+                }
                 goal_temp["goal_details"] = [summarize_step]
-                goal_temp["starting_sentence"] = generate_starting_sentence(goal_temp)
+                goal_temp["starting_sentence"] = generate_starting_sentence(
+                    goal_temp
+                )
 
                 goal_templates.append(goal_temp)
-        
+
             status.stop()
-            console.print("[bold green]Done creating starting sentence from provided input data")
+            console.print(
+                "[bold green]Done creating starting sentence from provided input data"
+            )
 
             return goal_templates
 
 
 if __name__ == "__main__":
     t = ExternalAgentPerformanceTest("test")
-    t.generate_tests()
+    t.generate_tests()

wxo_agentic_evaluation/external_agent/types.py

@@ -1,5 +1,6 @@
+from typing import Any, List, Literal, Mapping, Optional, Union
+
 from pydantic import BaseModel
-from typing import List, Union, Literal, Mapping, Any
 
 
 class ThinkingStepDetails(BaseModel):
@@ -25,7 +26,9 @@ class ToolResponseStepDetails(BaseModel):
     tool_call_id: str
 
 
-StepDetails = Union[ThinkingStepDetails, ToolCallsStepDetails, ToolResponseStepDetails]
+StepDetails = Union[
+    ThinkingStepDetails, ToolCallsStepDetails, ToolResponseStepDetails
+]
 
 
 class DeltaMessageChoice(BaseModel):
@@ -43,7 +46,7 @@ class ThreadRunStepDeltaChoice(BaseModel):
 class BaseEventData(BaseModel):
     id: str
     object: str
-    thread_id: str
+    thread_id: Optional[str] = None
     model: str | None = None
     created: int | None = None
 
@@ -59,13 +62,11 @@ class ThreadRunStepDeltaData(BaseEventData):
 
 
 class UniversalData(BaseEventData):
-    object: Union[Literal["thread.message.delta"], Literal["thread.run.step.delta"],
-                  Literal["thread.run.step.created"], Literal["thread.run.step.completed"]]
-    choices: List[ThreadMessageDeltaChoice]
+    object: Optional[str]
     choices: List[Union[ThreadMessageDeltaChoice, dict]]
 
 
 class SchemaValidationResults(BaseModel):
     success: bool
     logged_events: List[str]
-    messages: List[Mapping[Any, Any]]
+    messages: List[Mapping[Any, Any]]

wxo_agentic_evaluation/extractors/__init__.py

@@ -0,0 +1,3 @@
+from wxo_agentic_evaluation.extractors.labeled_messages import (
+    ExtractLabeledMessages,
+)

wxo_agentic_evaluation/extractors/extractor_base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from wxo_agentic_evaluation.type import Message
+
+
+class Extractor(ABC):
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Unique name for the extractor."""
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def extract(
+        messages: list[Message],
+        **kwargs,
+    ) -> Any:
+        """Extract data from messages."""
+        raise NotImplementedError

wxo_agentic_evaluation/extractors/labeled_messages.py

@@ -0,0 +1,47 @@
+import json
+from typing import Any, List, Mapping
+
+from wxo_agentic_evaluation.extractors.extractor_base import Extractor
+from wxo_agentic_evaluation.type import ContentType, GoalDetail, Message
+
+
+class ExtractLabeledMessages(Extractor):
+    def name(self):
+        return "Labelled Messages"
+
+    def extract(
+        messages: List[Message],
+        ground_truth,
+        **kwargs,
+    ) -> Any:
+
+        tool_dictionary = (
+            {
+                goal_detail.name: goal_detail
+                for goal_detail in ground_truth.goal_details
+                if goal_detail.type == ContentType.tool_call
+            }
+            if ground_truth.goal_details
+            else {}
+        )
+        labeled_messages = {}
+        for idx, message in enumerate(messages):
+            # TODO: investigate this logic - `message` body might not be consistent across providers
+            if not (message.role == "assistant" and message.tool_calls):
+                continue
+            try:
+                msg_tool_call = message.tool_calls[0].function
+            except Exception:
+                # ignore malformed tool_call content
+                continue
+
+            matching_goal_details = [
+                gd
+                for gd in tool_dictionary.values()
+                if gd.tool_name == msg_tool_call.name
+            ]
+
+            if matching_goal_details:
+                labeled_messages[idx] = matching_goal_details
+
+        return {"labeled_messages": labeled_messages}

wxo_agentic_evaluation/hr_agent_langgraph.py

@@ -0,0 +1,68 @@
+from langchain.tools import tool
+from langchain.agents import create_agent
+from langchain_openai import ChatOpenAI
+
+import json
+from datetime import datetime
+
+@tool
+def get_assignment_id_hr_usecase(username: str) -> str:
+    """
+    get the assignment id from username
+    :param username: username of the employee
+    """
+    if username=="nwaters":
+        return "15778303"
+    if username=="johndoe":
+        return "15338303"
+    return "not found"
+
+def validate_datetime(date_text):
+    try:
+        format = "%Y-%m-%d"
+        datetime.strptime(date_text, format)
+        return True
+    except ValueError:
+        return False
+
+
+@tool
+def get_timeoff_schedule_hr_usecase(assignment_id: str, start_date: str, end_date: str) -> str:
+    """
+    get timeoff schedule for employee based on assignment id, start date and end date
+    :param assignment_id: assignment_id of the user
+    :param start_date: start date of the timeoff scheduel, in YYYY-MM-DD format
+    :param assignment_id: end date of the timeoff scheduel, in YYYY-MM-DD format
+    """
+
+    if not validate_datetime(start_date):
+        return f"Incorrect date format {start_date}, should be YYYY-MM-DD"
+    if not validate_datetime(end_date):
+        return f"Incorrect date format {end_date}, should be YYYY-MM-DD"
+    if assignment_id=="15338303":
+        return json.dumps(["20250411", "20250311", "20250101"])
+    if assignment_id=="15778303":
+        return json.dumps(["20250105"])
+    return []
+
+
+@tool
+def get_direct_reports_hr_usecase(username: str) -> str:
+    """
+    get direct reports for a given username
+    :param assignment_id: assignment_id of the user
+    """
+
+    return json.dumps(["nwaters", "johndoe"])
+
+
+llm = ChatOpenAI(model="gpt-4o-mini")
+tools = [get_assignment_id_hr_usecase, get_timeoff_schedule_hr_usecase, get_direct_reports_hr_usecase]
+system_prompt="""You are an HR Agent that can answer questions related to timeoff and holiday calendar. Use the tools provided to answer the user's question. If you do not have enough information to answer the question, say so.  If you need more information, ask follow up questions."""
+
+agent = create_agent(
+    tools=tools,
+    model=llm,
+    system_prompt=system_prompt
+)
+

wxo_agentic_evaluation/langfuse_collection.py

@@ -0,0 +1,60 @@
+import json
+from typing import List, Mapping, Union
+
+import rich
+
+from wxo_agentic_evaluation.type import (
+    LangfuseCollectionModel,
+    LangfuseDatasetModel,
+)
+
+
+class LangfuseCollection:
+    def __init__(self, name, description="", metadata: Mapping[str, str] = {}):
+        self.name = name
+        self.description = description
+        self.metadata = metadata
+
+    def upload(self, paths: Union[str, List[str]]):
+        from langfuse import get_client
+
+        langfuse_client = get_client()
+
+        datasets = []
+        if isinstance(paths, str):
+            paths = [paths]
+
+        for path in paths:
+            with open(path, encoding="utf-8") as f:
+                dataset = json.load(f)
+                dataset = LangfuseDatasetModel(
+                    starting_sentence=dataset.get("starting_sentence", ""),
+                    story=dataset.get("story", ""),
+                    goals=dataset.get("goals"),
+                    goal_details=dataset.get("goal_details"),
+                    agent=dataset.get("agent")
+                )
+                datasets.append(dataset)
+
+        collection = LangfuseCollectionModel(
+            collection_name=self.name,
+            collection_description=self.description,
+            datasets=datasets,
+            metadata=self.metadata,
+        )
+
+        rich.print(
+            f"[g] Uploading {len(collection.datasets)} datasets to '{collection.collection_name}'"
+        )
+        langfuse_client.create_dataset(
+            name=collection.collection_name,
+            description=collection.collection_description,
+            metadata=collection.metadata,
+        )
+
+        for dataset in collection.datasets:
+            langfuse_client.create_dataset_item(
+                dataset_name=collection.collection_name,
+                input=dataset.langfuse_input,
+                expected_output=dataset.langfuse_output,
+            )