ibm-watsonx-orchestrate-evaluation-framework 1.0.0__py3-none-any.whl

wxo_agentic_evaluation/service_instance.py

@@ -0,0 +1,179 @@
+import logging
+import yaml
+import os
+import requests
+from wxo_agentic_evaluation.utils.utils import is_saas_url, is_ibm_cloud_url
+
+logger = logging.getLogger(__name__)
+
+USER = {"username": "wxo.archer@ibm.com", "password": "watsonx"}
+
+
+class ServiceInstance:
+    def __init__(
+        self, service_url, tenant_name, is_saas: bool = None, is_ibm_cloud: bool = None
+    ) -> None:
+        self.service_url = service_url
+        self.tenant_name = tenant_name
+        STAGING_AUTH_ENDPOINT = (
+            "https://iam.platform.test.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
+        )
+        PROD_AUTH_ENDPOINT = (
+            "https://iam.platform.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
+        )
+        PROD_AUTH_ENDPOINT_IBMCLOUD = "https://iam.cloud.ibm.com/identity/token"
+
+        self.is_saas = is_saas_url(service_url) if is_saas is None else is_saas
+        self.is_ibm_cloud = (
+            is_ibm_cloud_url(service_url) if is_ibm_cloud is None else is_ibm_cloud
+        )
+
+        if self.is_saas:
+            if self.is_ibm_cloud:
+                self.auth_endpoint = PROD_AUTH_ENDPOINT_IBMCLOUD
+            else:
+                self.auth_endpoint = (
+                    STAGING_AUTH_ENDPOINT
+                    if "staging" in service_url
+                    else PROD_AUTH_ENDPOINT
+                )
+            self.tenant_url = None  # Not used in SaaS
+            self.tenant_auth_endpoint = None
+        else:
+            self.auth_endpoint = f"{service_url}/api/v1/auth/token"
+            self.tenant_url = f"{service_url}/tenants"
+            self.tenant_auth_endpoint = "{}/api/v1/auth/token?tenant_id={}"
+
+        self.global_token = self.get_user_token()
+
+    def get_user_token(self):
+        try:
+            if self.is_saas:
+                apikey = os.environ.get("WATSONX_IAM_SAAS_APIKEY")
+                if not apikey:
+                    raise RuntimeError(
+                        "WATSONX_IAM_SAAS_APIKEY not set in environment for SaaS mode"
+                    )
+                if self.is_ibm_cloud:
+                    data = {
+                        "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+                        "apikey": apikey,
+                    }
+                    response = requests.post(self.auth_endpoint, data=data)
+                    token_key = "access_token"
+                else:
+                    headers = {
+                        "Accept": "application/json",
+                        "Content-Type": "application/json",
+                    }
+                    payload = {"apikey": apikey}
+                    response = requests.post(
+                        self.auth_endpoint, headers=headers, json=payload
+                    )
+                    token_key = "token"
+            else:
+                response = requests.post(self.auth_endpoint, data=USER)
+                token_key = "access_token"
+
+            return response.json()[token_key]
+
+        except KeyError as e:
+            print(
+                f"[ERROR] Missing key '{e}' in response. SaaS mode: {self.is_saas}. Full response: {response.text}"
+            )
+            raise
+        except requests.RequestException as e:
+            print(f"[ERROR] Request failed: {e}")
+            raise
+
+    def _get_tenant_token(self, tenant_id: str):
+        resp = requests.post(
+            self.tenant_auth_endpoint.format(self.service_url, tenant_id), data=USER
+        )
+        if resp.status_code == 200:
+            return resp.json()["access_token"]
+        else:
+            resp.raise_for_status()
+
+    def get_default_tenant(self, apikey):
+        headers = {
+            "Authorization": f"Bearer {apikey}",
+            "Content-Type": "application/json",
+        }
+        resp = requests.get(self.tenant_url, headers=headers)
+        if resp.status_code == 200:
+            tenant_config = resp.json()
+            for tenant in tenant_config:
+                if tenant["name"] == self.tenant_name:
+                    return tenant
+            return {}
+        else:
+            resp.raise_for_status()
+
+    def create_eval_tenant(self, apikey):
+        headers = {
+            "Authorization": f"Bearer {apikey}",
+            "Content-Type": "application/json",
+        }
+
+        tenant_config = {
+            "name": self.tenant_name,
+            "title": "WatsonX Orchestrate Development",
+            "tags": ["test"],
+        }
+
+        resp = requests.post(self.tenant_url, headers=headers, json=tenant_config)
+        if resp.status_code == 201:
+            return True
+        else:
+            resp.raise_for_status()
+
+    def create_tenant_if_not_exist(self) -> str:
+        if self.is_saas:
+            logger.info(
+                "SaaS mode: running against Remote Service and skipping tenant creation"
+            )
+            return None
+
+        user_auth_token = self.global_token
+        default_tenant = self.get_default_tenant(user_auth_token)
+
+        if not default_tenant:
+            logger.info("no local tenant found. A default tenant is created")
+            self.create_eval_tenant(user_auth_token)
+            default_tenant = self.get_default_tenant(user_auth_token)
+        else:
+            logger.info("local tenant found")
+
+        return default_tenant["id"]
+
+
+def tenant_setup(service_url: str, tenant_name: str):
+    # service_instance = ServiceInstance(
+    #     service_url=service_url,
+    #     tenant_name=tenant_name
+    # )
+    # tenant_id = service_instance.create_tenant_if_not_exist()
+    # if service_instance.is_saas:
+    #     tenant_token = service_instance.global_token
+    # else:
+    #     tenant_token = service_instance._get_tenant_token(tenant_id)
+
+    auth_config_path = f"{os.path.expanduser('~')}/.cache/orchestrate/credentials.yaml"
+    env_config_path = f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+
+    # TO-DO: update SDK and use SDK to manage this
+    with open(auth_config_path, "r") as f:
+        auth_config = yaml.safe_load(f)
+    # auth_config["auth"][tenant_name] = {"wxo_mcsp_token": tenant_token}
+
+    with open(env_config_path, "r") as f:
+        env_config = yaml.safe_load(f)
+    env_config["environments"][tenant_name] = {"wxo_url": service_url}
+    env_config["context"]["active_environment"] = tenant_name
+
+    with open(auth_config_path, "w") as f:
+        yaml.dump(auth_config, f)
+    with open(env_config_path, "w") as f:
+        yaml.dump(env_config, f)
+    return auth_config["auth"][tenant_name]["wxo_mcsp_token"]

wxo_agentic_evaluation/tool_planner.py

@@ -0,0 +1,228 @@
+import json
+import ast
+import csv
+from pathlib import Path
+import importlib.util
+import re
+from jsonargparse import CLI
+import os
+
+from wxo_agentic_evaluation.watsonx_provider import WatsonXProvider
+from wxo_agentic_evaluation.arg_configs import BatchAnnotateConfig
+from wxo_agentic_evaluation.prompt.template_render import ToolPlannerTemplateRenderer, ToolChainAgentTemplateRenderer
+from wxo_agentic_evaluation import __file__
+
+root_dir = os.path.dirname(__file__)
+TOOL_PLANNER_PROMPT_PATH = os.path.join(root_dir, "prompt", "tool_planner.jinja2")
+
+
+def extract_first_json_list(raw: str) -> list:
+    matches = re.findall(r"\[\s*{.*?}\s*]", raw, re.DOTALL)
+    for match in matches:
+        try:
+            parsed = json.loads(match)
+            if isinstance(parsed, list) and all("tool_name" in step for step in parsed):
+                return parsed
+        except Exception:
+            continue
+    print("⚠️ Could not parse tool call plan. Raw output:")
+    print(raw)
+    return []
+
+
+def load_tools_module(tools_path: Path) -> dict:
+    tools_dict = {}
+    files_to_parse = []
+
+    if tools_path.is_file():
+        files_to_parse.append(tools_path)
+    elif tools_path.is_dir():
+        files_to_parse.extend(tools_path.glob("**/*.py"))
+    else:
+        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+
+    for file_path in files_to_parse:
+        try:
+            module_name = file_path.stem
+            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+
+            # Add all module's non-private functions to tools_dict
+            for attr_name in dir(module):
+                attr = getattr(module, attr_name)
+                if callable(attr) and not attr_name.startswith('_'):
+                    tools_dict[attr_name] = attr
+        except Exception as e:
+            print(f"Warning: Failed to load {file_path}: {str(e)}")
+
+    return tools_dict
+
+
+def extract_tool_signatures(tools_path: Path) -> list:
+    tool_data = []
+    files_to_parse = []
+
+    # Handle both single file and directory cases
+    if tools_path.is_file():
+        files_to_parse.append(tools_path)
+    elif tools_path.is_dir():
+        files_to_parse.extend(tools_path.glob("**/*.py"))
+    else:
+        raise ValueError(f"Tools path {tools_path} is neither a file nor directory")
+
+    for file_path in files_to_parse:
+        try:
+            with file_path.open("r", encoding="utf-8") as f:
+                code = f.read()
+            parsed_code = ast.parse(code)
+
+            for node in parsed_code.body:
+                if isinstance(node, ast.FunctionDef):
+                    name = node.name
+                    args = [arg.arg for arg in node.args.args if arg.arg != "self"]
+                    docstring = ast.get_docstring(node)
+                    tool_data.append({
+                        "Function Name": name,
+                        "Arguments": args,
+                        "Docstring": docstring or "No description available"
+                    })
+        except Exception as e:
+            print(f"Warning: Failed to parse {file_path}: {str(e)}")
+            continue
+
+    return tool_data
+
+
+def ensure_data_available(tool_name: str, inputs: dict, snapshot: dict, tools_module: dict) -> dict:
+    cache = snapshot.setdefault("input_output_examples", {}).setdefault(tool_name, [])
+    for entry in cache:
+        if entry["inputs"] == inputs:
+            return entry["output"]
+
+    if tool_name not in tools_module:
+        raise ValueError(f"Tool '{tool_name}' not found")
+
+    output = tools_module[tool_name](**inputs)
+    cache.append({"inputs": inputs, "output": output})
+    if not isinstance(output, dict):
+        print(f" Tool {tool_name} returned non-dict output: {output}")
+    return output
+
+def plan_tool_calls_with_llm(story: str, agent_name: str, tool_signatures_str: str, provider) -> list:
+
+    renderer = ToolPlannerTemplateRenderer(TOOL_PLANNER_PROMPT_PATH)
+
+    prompt = renderer.render(
+        user_story=story,
+        agent_name=agent_name,
+        available_tools=tool_signatures_str,
+    )
+    response = provider.query(prompt)
+    raw = response.get("generated_text", "")
+    parsed = extract_first_json_list(raw)
+    print("\n LLM Tool Plan:")
+    print(json.dumps(parsed, indent=2))
+    return parsed
+
+
+# --- Tool Execution Logic ---
+def run_tool_chain(tool_plan: list, snapshot: dict, tools_module) -> None:
+    memory = {}
+
+    for step in tool_plan:
+        name = step["tool_name"]
+        raw_inputs = step["inputs"]
+        print(f"\n🔧 Tool: {name}")
+        print(f" Raw inputs: {raw_inputs}")
+
+        resolved_inputs = {}
+        list_keys = []
+
+        for k, v in raw_inputs.items():
+            if isinstance(v, str) and v.startswith("$"):
+                expr = v[1:]
+                try:
+                    resolved_value = eval(expr, {}, memory)
+                    resolved_inputs[k] = resolved_value
+                    if isinstance(resolved_value, list):
+                        list_keys.append(k)
+                except Exception as e:
+                    print(f" ❌ Failed to resolve {v} in memory: {memory}")
+                    raise ValueError(f"Failed to resolve placeholder {v}: {e}")
+            else:
+                resolved_inputs[k] = v
+
+        print(f" Resolved inputs: {resolved_inputs}")
+
+        if list_keys:
+            if len(list_keys) > 1:
+                raise ValueError(f"Tool '{name}' received multiple list inputs. Only one supported for now.")
+            list_key = list_keys[0]
+            value_list = resolved_inputs[list_key]
+
+            results = []
+            for idx, val in enumerate(value_list):
+                item_inputs = resolved_inputs.copy()
+                item_inputs[list_key] = val
+                print(f" ⚙️ Running {name} with {list_key} = {val}")
+                output = ensure_data_available(name, item_inputs, snapshot, tools_module)
+                results.append(output)
+                memory[f"{name}_{idx}"] = output
+
+            memory[name] = results
+            print(f"Stored {len(results)} outputs under '{name}' and indexed as '{name}_i'")
+        else:
+            output = ensure_data_available(name, resolved_inputs, snapshot, tools_module)
+            memory[name] = output
+            print(f"Stored output under tool name: {name} = {output}")
+
+
+# --- Main Snapshot Builder ---
+def build_snapshot(agent_name: str, tools_path: Path, stories: list, output_path: Path):
+    agent = {"name": agent_name}
+    tools_module = load_tools_module(tools_path)
+    tool_signatures = extract_tool_signatures(tools_path)
+
+    provider = WatsonXProvider(
+        model_id="meta-llama/llama-3-405b-instruct",
+        llm_decode_parameter={
+            "min_new_tokens": 50,
+            "decoding_method": "greedy",
+            "max_new_tokens": 200
+        }
+    )
+
+    snapshot = {
+        "agent": agent,
+        "tools": tool_signatures,
+        "input_output_examples": {}
+    }
+
+    for story in stories:
+        print(f"\n📘 Planning tool calls for story: {story}")
+        tool_plan = plan_tool_calls_with_llm(story, agent["name"], tool_signatures, provider)
+        run_tool_chain(tool_plan, snapshot, tools_module)
+
+    with output_path.open("w", encoding="utf-8") as f:
+        json.dump(snapshot, f, indent=2)
+    print(f"\n✅ Snapshot saved to {output_path}")
+
+
+if __name__ == "__main__":
+    config = CLI(BatchAnnotateConfig, as_positional=False)
+    tools_path = Path(config.tools_path)
+    stories_path = Path(config.stories_path)
+
+    stories = []
+    agent_name = None
+    with stories_path.open("r", encoding="utf-8", newline='') as f:
+        csv_reader = csv.DictReader(f)
+        for row in csv_reader:
+            stories.append(row["story"])
+            if agent_name is None:
+                agent_name = row["agent"]
+
+    snapshot_path = stories_path.parent / f"{agent_name}_snapshot_llm.json"
+
+    build_snapshot(agent_name, tools_path, stories, snapshot_path)

wxo_agentic_evaluation/type.py

@@ -0,0 +1,176 @@
+from typing import Dict, List, Union, Any, Optional
+from pydantic import BaseModel, computed_field, ConfigDict
+from enum import StrEnum
+
+
+class EventTypes(StrEnum):
+    run_started = "run.started"
+    run_step_delta = "run.step.delta"
+    message_started = "message.started"
+    message_delta = "message.delta"
+    message_created = "message.created"
+    run_completed = "run.completed"
+    done = "done"
+
+
+class ContentType(StrEnum):
+    text = "text"
+    tool_call = "tool_call"
+    tool_response = "tool_response"
+    conversational_search = "conversational_search"
+
+
+class ConversationalSearchCitations(BaseModel):
+    url: str
+    body: str
+    text: str
+    title: str
+    range_end: int
+    range_start: int
+    search_result_idx: int
+
+
+class ConversationalSearchResultMetadata(BaseModel):
+    score: float
+    document_retrieval_source: str
+
+
+class ConversationalSearchResults(BaseModel):
+    url: str
+    body: str
+    title: str
+    result_metadata: ConversationalSearchResultMetadata
+
+
+class ConversationalConfidenceThresholdScore(BaseModel):
+    response_confidence: float
+    response_confidence_threshold: float
+    retrieval_confidence: float
+    retrieval_confidence_threshold: float
+
+    def table(self):
+        return {
+            "response_confidence": str(self.response_confidence),
+            "response_confidence_threshold": str(self.response_confidence_threshold),
+            "retrieval_confidence": str(self.retrieval_confidence),
+            "retrieval_confidence_threshold": str(self.retrieval_confidence_threshold),
+        }
+
+
+class ConversationSearchMetadata(BaseModel):
+    """This class is used to store additional informational about the conversational search response that was not part of the API response.
+
+    For example, the tool call that generated the conversational search response is not part of the API response. However,
+    during evaluation, we want to refer to the tool that generated the conversational search response.
+    """
+
+    tool_call_id: str
+    model_config = ConfigDict(frozen=True)
+
+
+class ConversationalSearch(BaseModel):
+    metadata: ConversationSearchMetadata
+    response_type: str
+    text: str  # same as `content` in Message. This field can be removed if neccesary
+    citations: List[ConversationalSearchCitations]
+    search_results: List[ConversationalSearchResults]
+    citations_title: str
+    confidence_scores: ConversationalConfidenceThresholdScore
+    response_length_option: str
+
+
+class Message(BaseModel):
+    role: str
+    content: Union[str, Dict[str, Any]]
+    type: ContentType
+    # event that produced the message
+    event: Optional[str] = None
+    # used to correlate the Message with the retrieval context (ConversationalSearch)
+    conversational_search_metadata: Optional[ConversationSearchMetadata] = None
+
+    model_config = ConfigDict(frozen=True)
+
+
+class ExtendedMessage(BaseModel):
+    message: Message
+    reason: dict | None = None
+
+
+class KnowledgeBaseGoalDetail(BaseModel):
+    enabled: bool = False
+    metrics: list = []
+
+
+class GoalDetail(BaseModel):
+    name: str
+    tool_name: str = None
+    type: ContentType
+    args: Dict = None
+    response: str = None
+    keywords: List = None
+    knowledge_base: KnowledgeBaseGoalDetail = KnowledgeBaseGoalDetail()
+
+
+class MineField(BaseModel):
+    type: ContentType
+    name: str
+
+
+class EvaluationData(BaseModel):
+    agent: str
+    goals: Dict
+    story: str
+    mine_fields: List[MineField]
+    goal_details: List[GoalDetail]
+    starting_sentence: str = None
+
+
+class ToolCallAndRoutingMetrics(BaseModel):
+    total_tool_calls: int
+    expected_tool_calls: int
+    relevant_tool_calls: int
+    correct_tool_calls: int
+    total_routing_calls: int
+    expected_routing_calls: int
+
+    @computed_field
+    @property
+    def non_transfer_tool_calls(self) -> int:
+        return self.total_tool_calls - self.total_routing_calls
+
+    @computed_field
+    @property
+    def tool_call_accuracy(self) -> float:
+        return round(
+            (
+                self.correct_tool_calls / self.non_transfer_tool_calls
+                if self.non_transfer_tool_calls > 0
+                else 0.0
+            ),
+            2,
+        )
+
+    @computed_field
+    @property
+    def tool_call_relevancy(self) -> float:
+        return round(
+            (
+                (self.relevant_tool_calls - self.expected_routing_calls)
+                / self.non_transfer_tool_calls
+                if self.non_transfer_tool_calls > 0
+                else 0.0
+            ),
+            2,
+        )
+
+    @computed_field
+    @property
+    def agent_routing_accuracy(self) -> float:
+        return round(
+            (
+                self.expected_routing_calls / self.total_routing_calls
+                if self.total_routing_calls > 0
+                else 0.0
+            ),
+            2,
+        )

wxo_agentic_evaluation/utils/__init__.py

@@ -0,0 +1,6 @@
+import json
+
+
+def json_dump(output_path, object):
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(object, f, indent=4)