ibm-watsonx-orchestrate-evaluation-framework 1.0.0__py3-none-any.whl

wxo_agentic_evaluation/inference_backend.py

@@ -0,0 +1,601 @@
+import requests
+import os
+import yaml
+import json
+import rich
+import time
+from pydantic import BaseModel
+from typing import List, Generator, Dict, Tuple, Mapping, Any
+
+from wxo_agentic_evaluation.type import (
+    ContentType,
+    Message,
+    ConversationalSearch,
+    ConversationalSearchCitations,
+    ConversationalSearchResultMetadata,
+    ConversationalConfidenceThresholdScore,
+    ConversationalSearchResults,
+    ConversationSearchMetadata,
+)
+
+from wxo_agentic_evaluation.llm_user import LLMUser
+from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.prompt.template_render import LlamaUserTemplateRenderer
+from wxo_agentic_evaluation.arg_configs import TestConfig
+from wxo_agentic_evaluation.service_instance import tenant_setup
+from wxo_agentic_evaluation.utils.utils import is_saas_url
+
+
+def is_end(user_input: Message):
+    if "END" in user_input.content.strip():
+        return True
+    return False
+
+
+def is_transfer_response(step_detail: Dict):
+    if step_detail["type"] == "tool_response" and step_detail["name"].startswith(
+        "transfer_to_"
+    ):
+        return True
+    return False
+
+
+class CallTracker(BaseModel):
+    tool_call: List = []
+    tool_response: List = []
+    generic: List = []
+
+
+class WXOClient:
+    def __init__(self, service_url, api_key):
+        self.service_url = service_url
+        self.api_key = api_key
+
+    def _get_headers(self) -> dict:
+        headers = {}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        return headers
+
+    def post(self, payload: dict, path: str, stream=False):
+        url = f"{self.service_url}/{path}"
+        return requests.post(
+            url=url, headers=self._get_headers(), json=payload, stream=stream
+        )
+
+    def get(self, path: str, params: dict = None):
+        url = f"{self.service_url}/{path}"
+        return requests.get(url, params=params, headers=self._get_headers())
+
+
+class WXOInferenceBackend:
+    def __init__(self, wxo_client):
+        self.wxo_client = wxo_client
+        self.enable_saas_mode = is_saas_url(wxo_client.service_url)
+
+    def run(self, user_input: Message, agent_name, thread_id=None):
+        agent_id = self.get_agent_id(agent_name)
+        payload = {"message": user_input.model_dump(), "agent_id": agent_id}
+        if thread_id:
+            payload["thread_id"] = thread_id
+
+        if self.enable_saas_mode:
+            path = "/v1/orchestrate/runs"
+        else:
+            path = "/orchestrate/runs"
+
+        response: requests.Response = self.wxo_client.post(payload, path)
+
+        if int(response.status_code) == 200:
+            result = response.json()
+            return result["thread_id"]
+        else:
+            response.raise_for_status()
+
+    def _stream_events(
+        self, user_input: Message, agent_name: str, thread_id=None
+    ) -> Generator[Dict, None, None]:
+        agent_id = self.get_agent_id(agent_name)
+        payload = {"message": user_input.model_dump(), "agent_id": agent_id}
+        if thread_id:
+            payload["thread_id"] = thread_id
+
+        if self.enable_saas_mode:
+            path = "/v1/orchestrate/runs?stream=true"
+        else:
+            path = "/orchestrate/runs?stream=true"
+
+        response: requests.Response = self.wxo_client.post(payload, path, stream=True)
+        import json
+
+        for chunk in self._parse_events(response):
+            chunk = json.loads(chunk.strip())
+            yield chunk
+
+    def parse_conversational_search_response(
+        self,
+        conversational_search: Mapping[str, Any],
+        metadata: ConversationSearchMetadata,
+    ) -> ConversationalSearch:
+        def parse_citations():
+            citations = conversational_search["citations"]
+            parsed_citations = []
+            for citation in citations:
+                c = ConversationalSearchCitations(
+                    url=citation.get("url", ""),
+                    body=citation.get("body", ""),
+                    text=citation.get("text", ""),
+                    title=citation.get("title", ""),
+                    range_start=citation.get("range_start"),
+                    range_end=citation.get("range_end"),
+                    search_result_idx=citation.get("search_result_idx"),
+                )
+                parsed_citations.append(c)
+
+            return parsed_citations
+
+        def parsed_search_results():
+            search_results = conversational_search["search_results"]
+            parsed_search_results = []
+            for result in search_results:
+                result_metadata = result.get("result_metadata", {})
+                result_metadata = ConversationalSearchResultMetadata(
+                    score=result_metadata.get("score"),
+                    document_retrieval_source=result_metadata.get(
+                        "document_retrieval_source"
+                    ),
+                )
+                c = ConversationalSearchResults(
+                    url=result.get("url", ""),
+                    body=result.get("body", ""),
+                    title=result.get("title", ""),
+                    result_metadata=result_metadata,
+                )
+                parsed_search_results.append(c)
+
+            return parsed_search_results
+
+        citations = parse_citations()
+        retrieval_context = parsed_search_results()
+        citations_title = conversational_search.get("citations_title", "")
+        response_length_option = conversational_search.get("response_length_option", "")
+        text = conversational_search.get("text", "")
+
+        confidence_scores = ConversationalConfidenceThresholdScore(
+            **conversational_search.get("confidence_scores")
+        )
+        response_type = conversational_search.get("response_type")
+        # should always be conversational_search
+        assert response_type == ContentType.conversational_search
+
+        conversational_search = ConversationalSearch(
+            metadata=metadata,
+            response_type=response_type,
+            text=text,
+            citations=citations,
+            search_results=retrieval_context,
+            citations_title=citations_title,
+            confidence_scores=confidence_scores,
+            response_length_option=response_length_option,
+        )
+
+        return conversational_search
+
+    def stream_messages(
+        self,
+        user_input: Message,
+        agent_name: str,
+        call_tracker: CallTracker,
+        thread_id=None,
+    ) -> Tuple[List[Message], str, List[ConversationalSearch]]:
+        recover = False
+        messages = list()
+        conversational_search_data = []
+
+        start_time = time.time()
+        for chunk in self._stream_events(user_input, agent_name, thread_id):
+
+            event = chunk.get("event", "")
+            if _thread_id := chunk.get("data", {}).get("thread_id"):
+                thread_id = _thread_id
+                if delta := chunk.get("data", {}).get("delta"):
+                    role = delta["role"]
+                    if step_details := delta.get("step_details"):
+                        if any(
+                            is_transfer_response(step_detail)
+                            for step_detail in step_details
+                        ):
+                            continue
+                        for idx, step_detail in enumerate(step_details):
+                            if step_detail["type"] == "tool_calls":
+                                # in step details, we could have [tool_response, tool_call]
+                                # in this case, we skip since we already capture the tool call
+                                if idx == 1:
+                                    continue
+
+                                content_type = ContentType.tool_call
+                                for tool in step_detail["tool_calls"]:
+                                    # Only add "transfer_to_" calls here. Other tool calls are already
+                                    # captured in the next block, including them here will cause duplication
+                                    # if not tool["name"].startswith("transfer_to_"):
+                                    #     continue
+                                    tool_json = {"type": "tool_call"}
+                                    tool_json.update(tool)
+                                    content = json.dumps(tool_json)
+                                    messages.append(
+                                        Message(
+                                            role=role,
+                                            content=content,
+                                            type=content_type,
+                                            event=event,
+                                        )
+                                    )
+                                    end_time = time.time()
+                                    call_tracker.tool_call.append(end_time - start_time)
+                                    start_time = end_time
+                            elif step_detail["type"] == "tool_call":
+                                # in step details, we could have [tool_response, tool_call]
+                                # in this case, we skip since we already capture the tool call
+                                if idx == 1:
+                                    continue
+                                content_type = ContentType.tool_call
+                                content = json.dumps(step_detail)
+                                messages.append(
+                                    Message(
+                                        role=role,
+                                        content=content,
+                                        type=content_type,
+                                        event=event,
+                                    )
+                                )
+                                end_time = time.time()
+                                call_tracker.tool_call.append(end_time - start_time)
+                                start_time = end_time
+                            elif step_detail["type"] == "tool_response":
+                                content = json.dumps(step_detail)
+                                content_type = ContentType.tool_response
+                                messages.append(
+                                    Message(
+                                        role=role,
+                                        content=content,
+                                        type=content_type,
+                                        event=event,
+                                    )
+                                )
+                                end_time = time.time()
+                                call_tracker.tool_response.append(end_time - start_time)
+                                start_time = end_time
+                    elif content_field := delta.get("content"):
+                        for val in content_field:
+                            response_type = val["response_type"]
+                            # TODO: is this ever hit? the event name is "message.created", and it seems the event should be "message.delta"
+                            if (
+                                response_type == ContentType.text
+                                and chunk["event"] == "message_created"
+                            ):
+                                messages.append(
+                                    Message(
+                                        role=role,
+                                        content=val["text"],
+                                        type=ContentType.text,
+                                    ),
+                                    chunk=event,
+                                )
+                                end_time = time.time()
+                                call_tracker.generic.append(end_time - start_time)
+                                start_time = end_time
+
+                # NOTE: The event here that is parsed is part of the "message.created" event
+                elif message := chunk.get("data", {}).get("message"):
+                    role = message["role"]
+                    for content in message["content"]:
+                        if (
+                            content["response_type"]
+                            == ContentType.conversational_search
+                        ):
+                            end_time = time.time()
+                            call_tracker.generic.append(end_time - start_time)
+                            start_time = end_time
+
+                            """ This is under the assumption the flow is (tool call -> tool response -> response back to user).
+                            In other words, the tool response is not fed back in to the agent.
+                            We get the previous message and extract the `tool_call_id`.
+                            
+                            NOTE: The previous message is a tool call because how we parse the event stream.
+                            NOTE: The conversational search response event does not have a 'tool call id' which can be used to associate with the 'conversational search response'.
+                            """
+
+                            last_message = json.loads(messages[-1].content)
+                            tool_call_id = last_message.get("tool_call_id", None)
+                            assert tool_call_id is not None
+                            conversational_search_metadata = ConversationSearchMetadata(
+                                tool_call_id=tool_call_id
+                            )
+                            conversational_search = (
+                                self.parse_conversational_search_response(
+                                    conversational_search=content,
+                                    metadata=conversational_search_metadata,
+                                )
+                            )
+                            conversational_search_data.append(conversational_search)
+                            messages.append(
+                                Message(
+                                    role=role,
+                                    content=content["text"],
+                                    type=ContentType.conversational_search,
+                                    conversational_search_metadata=conversational_search_metadata,
+                                    event=event,
+                                )
+                            )
+                        if content["response_type"] == ContentType.text:
+                            messages.append(
+                                Message(
+                                    role=role,
+                                    content=content["text"],
+                                    type=ContentType.text,
+                                    event=chunk["event"],
+                                )
+                            )
+                            end_time = time.time()
+                            call_tracker.generic.append(end_time - start_time)
+                            start_time = end_time
+            else:
+                # Exit the loop if we lose the thread_id
+                recover = True
+                break
+
+        if recover and (thread_id is not None):
+            rich.print(
+                "🔬 [bold][magenta]INFO:[/magenta][/bold]",
+                f"Attempting to recover messages from thread_id {thread_id}",
+            )
+            # If we lose the thread_id, we need to wait for a bit to allow the message to come through
+            # before attempting to recover the messages.
+            time.sleep(10)
+            messages = self.recover_messages(thread_id)
+            rich.print(
+                "🔬 [bold][magenta]INFO:[/magenta][/bold]",
+                f"Recovered {len(messages)} messages from thread_id {thread_id}",
+            )
+
+        return messages, thread_id, conversational_search_data
+
+    def _parse_events(
+        self, stream: Generator[bytes, None, None]
+    ) -> Generator[bytes, None, None]:
+        data = b""
+        for chunk in stream:
+            for line in chunk.splitlines(True):
+                data += line
+                if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n", b"\n")):
+                    yield data
+                    data = b""
+        if data:
+            yield data
+
+    def recover_messages(self, thread_id: str) -> List[Message]:
+        messages = self.get_messages(thread_id)
+        return self._get_messages_after_last_user(messages)
+
+    def get_messages(self, thread_id) -> List[Message]:
+        if self.enable_saas_mode:
+            path = f"v1/orchestrate/threads/{thread_id}/messages"
+        else:
+            path = f"threads/{thread_id}/messages"
+        response = self.wxo_client.get(path)
+        if response.status_code == 200:
+            result = response.json()
+
+        else:
+            response.raise_for_status()
+
+        messages = []
+        for entry in result:
+            tool_call_id = None
+            if step_history := entry.get("step_history"):
+                for step_message in step_history:
+                    role = step_message["role"]
+                    if step_details := step_message.get("step_details"):
+                        for step_detail in step_details:
+                            if step_detail["type"] == "tool_calls":
+                                content_type = ContentType.tool_call
+                                for tool in step_detail["tool_calls"]:
+                                    tool_json = {"type": "tool_call"}
+                                    tool_json.update(tool)
+                                    content = json.dumps(tool_json)
+                                    messages.append(
+                                        Message(
+                                            role=role,
+                                            content=content,
+                                            type=content_type,
+                                        )
+                                    )
+                            elif step_detail["type"] == "tool_call":
+                                tool_call_id = step_detail["tool_call_id"]
+                                content_type = ContentType.tool_call
+                                content = json.dumps(step_detail)
+                                messages.append(
+                                    Message(
+                                        role=role, content=content, type=content_type
+                                    )
+                                )
+                            else:
+                                content = json.dumps(step_detail)
+                                content_type = ContentType.tool_response
+                                messages.append(
+                                    Message(
+                                        role=role, content=content, type=content_type
+                                    )
+                                )
+            if content_field := entry.get("content"):
+                role = entry["role"]
+                for val in content_field:
+                    if val["response_type"] == ContentType.text:
+                        messages.append(
+                            Message(
+                                role=role, content=val["text"], type=ContentType.text
+                            )
+                        )
+                    if val["response_type"] == ContentType.conversational_search:
+                        conversational_search_metadata = ConversationSearchMetadata(
+                            tool_call_id=tool_call_id
+                        )
+                        messages.append(
+                            Message(
+                                role=role,
+                                content=val["text"],
+                                type=ContentType.text,
+                                conversational_search_metadata=conversational_search_metadata,
+                            )
+                        )
+
+        return messages
+
+    @staticmethod
+    def _get_messages_after_last_user(messages: List[Message]) -> List[Message]:
+        for i in range(len(messages) - 1, -1, -1):
+            if messages[i].role == "user":
+                return messages[i + 1 :]
+        return messages
+
+    def get_agent_id(self, agent_name: str):
+        if self.enable_saas_mode:
+            path = "v1/orchestrate/agents"
+        else:
+            path = "orchestrate/agents"
+
+        response = self.wxo_client.get(path)
+
+        if response.status_code == 200:
+            result = response.json()
+            for agent in result:
+                if agent.get("name", "") == agent_name:
+                    return agent.get("id")
+
+            raise Exception(f"Agent with name {agent_name} not found.")
+
+        else:
+            response.raise_for_status()
+
+
+class EvaluationController:
+    def __init__(
+        self,
+        wxo_inference_backend: WXOInferenceBackend,
+        llm_user: LLMUser,
+        config: TestConfig,
+    ):
+        self.wxo_inference_backend = wxo_inference_backend
+        self.llm_user = llm_user
+        self.config = config
+
+    def run(
+        self, task_n, story, agent_name: str, starting_user_input: str = None
+    ) -> Tuple[List[Message], List[CallTracker], List[ConversationalSearch]]:
+        step = 0
+        thread_id = None
+        conversation_history: List[Message] = []
+        conversational_search_history_data = []
+        call_tracker = CallTracker()
+        # make this configurable
+        while step < 20:
+
+            if step == 0 and starting_user_input:
+                user_input = Message(
+                    role="user", content=starting_user_input, type=ContentType.text
+                )
+            else:
+                if self.config.enable_manual_user_input == True:
+                    content = input(
+                        "[medium_orchid1]Enter your input[/medium_orchid1] ✍️: "
+                    )
+                    user_input = Message(
+                        role="user", content=content, type=ContentType.text
+                    )
+                else:  # llm
+                    user_input = self.llm_user.generate_user_input(
+                        story, conversation_history
+                    )
+            if self.config.enable_verbose_logging:
+                rich.print(
+                    f"[dark_khaki][Task-{task_n}][/dark_khaki] 👤[bold blue] User:[/bold blue]",
+                    user_input.content,
+                )
+            if is_end(user_input):
+                break
+            conversation_history.append(user_input)
+            messages, thread_id, conversational_search_data = (
+                self.wxo_inference_backend.stream_messages(
+                    user_input,
+                    agent_name=agent_name,
+                    thread_id=thread_id,
+                    call_tracker=call_tracker,
+                )
+            )
+            if self.config.enable_verbose_logging:
+                for message in messages:
+                    rich.print(
+                        f"[orange3][Task-{task_n}][/orange3] 🤖[bold cyan] WXO:[/bold cyan]",
+                        message.content,
+                    )
+            conversation_history.extend(messages)
+            conversational_search_history_data.extend(conversational_search_data)
+            step += 1
+        return conversation_history, call_tracker, conversational_search_history_data
+
+
+def get_wxo_client(service_url: str, token: str):
+    wxo_client = WXOClient(service_url=service_url, api_key=token)
+    return wxo_client
+
+
+def get_wxo_inference_backend(
+    service_url: str, tenant_name: str, token: str = None
+) -> WXOInferenceBackend:
+    if not token:
+        token = tenant_setup(service_url, tenant_name)
+    wxo_client = get_wxo_client(service_url, token)
+    inference_backend = WXOInferenceBackend(wxo_client=wxo_client)
+    return inference_backend
+
+
+if __name__ == "__main__":
+    wai_client = WatsonXProvider(model_id="meta-llama/llama-3-3-70b-instruct")
+    llm_user = LLMUser(
+        wai_client=wai_client,
+        template=LlamaUserTemplateRenderer(
+            "src/wxo_agentic_evaluation/prompt/llama_user_prompt.jinja2"
+        ),
+        user_response_style=None,
+    )
+    auth_config_path = f"{os.path.expanduser('~')}/.cache/orchestrate/credentials.yaml"
+    with open(auth_config_path, "r") as f:
+        auth_config = yaml.safe_load(f)
+    tenant_name = "local"
+    token = auth_config["auth"][tenant_name]["wxo_mcsp_token"]
+
+    wxo_client = WXOClient(service_url="http://localhost:4321", api_key=token)
+    inference_backend = WXOInferenceBackend(wxo_client=wxo_client)
+    config = TestConfig(
+        test_paths=[],
+        output_dir="./wxo_agentic_evaluation/results",
+        auth_config=auth_config,
+        wxo_lite_version="0.1.3",
+    )
+    evaluation_controller = EvaluationController(
+        wxo_inference_backend=inference_backend, llm_user=llm_user, config=config
+    )
+    history, _, _ = evaluation_controller.run(
+        0,
+        "Your username is nken and you want to find out the timeoff schedule of your reports from 20250101 o 202505t",
+        agent_name="hr_agent",
+    )
+    # starting_user_input="my username is nken, i want to know the timeoff schedule for my reports from 20250101 to 202505")
+
+    result = list()
+    for message in history:
+        result.append(message.model_dump())
+
+    os.makedirs("./wxo_agentic_evaluation/results", exist_ok=True)
+    with open("./wxo_agentic_evaluation/results/messages.json", "w") as f:
+        json.dump(result, f)

wxo_agentic_evaluation/llm_matching.py

@@ -0,0 +1,39 @@
+from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.prompt.template_render import (
+    KeywordMatchingTemplateRenderer,
+    SemanticMatchingTemplateRenderer,
+)
+from typing import List
+
+
+class LLMMatcher:
+    def __init__(
+        self,
+        llm_client: WatsonXProvider,
+        keyword_template: KeywordMatchingTemplateRenderer,
+        semantic_template: SemanticMatchingTemplateRenderer,
+    ):
+        self.llm_client = llm_client
+        self.keyword_template = keyword_template
+        self.semantic_template = semantic_template
+
+    def keywords_match(self, response_text: str, keywords: List[str]) -> bool:
+        if len(keywords) == 0:
+            return True
+        # return True if no keywords are provided
+        # This allows for skipping keyword check by providing an empty list
+        keywords_text = "\n".join(keywords)
+        prompt = self.keyword_template.render(
+            keywords_text=keywords_text, response_text=response_text
+        )
+        output = self.llm_client.query(prompt)
+        result = output["generated_text"].strip().lower()
+        return result.startswith("true")
+
+    def semantic_match(self, prediction: str, ground_truth: str) -> bool:
+        prompt = self.semantic_template.render(
+            expected_text=ground_truth, actual_text=prediction
+        )
+        output = self.llm_client.query(prompt)
+        result = output["generated_text"].strip().lower()
+        return result.startswith("true")

wxo_agentic_evaluation/llm_rag_eval.py

@@ -0,0 +1,47 @@
+from typing import List
+import json
+
+from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.prompt.template_render import (
+    FaithfulnessTemplateRenderer,
+    AnswerRelevancyTemplateRenderer,
+)
+from wxo_agentic_evaluation.metrics.llm_as_judge import Faithfulness, AnswerRelevancy
+
+
+class LLMJudge:
+    def __init__(
+        self,
+        llm_client: WatsonXProvider,
+        faithfulness: FaithfulnessTemplateRenderer,
+        answer_relevancy: AnswerRelevancyTemplateRenderer,
+    ):
+        self.llm_client = llm_client
+        self.faithfulness_template = faithfulness
+        self.answer_relevancy_template = answer_relevancy
+
+    # TODO: implement callable, and implement decorator to retry the LLM call
+    def faithfulness(self, claim, retrieval_context: List[str]) -> Faithfulness:
+        retrieval_context = "\n".join(retrieval_context)
+        prompt = self.faithfulness_template.render(
+            claim=claim, retrieval_context=retrieval_context
+        )
+        output = self.llm_client.query(prompt)
+        result = output["generated_text"].strip().lower()
+
+        faithfulness = Faithfulness.model_validate(json.loads(result))
+
+        return faithfulness
+
+    def answer_relevancy(
+        self, question: str, context: str, answer: str
+    ) -> AnswerRelevancy:
+        prompt = self.answer_relevancy_template.render(
+            question=question, context=context, answer=answer
+        )
+        output = self.llm_client.query(prompt)
+        result = output["generated_text"].strip().lower()
+
+        answer_relevancy = AnswerRelevancy(answer_relevancy=json.loads(result))
+
+        return answer_relevancy