ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/referenceless_eval/prompt/runner.py

@@ -0,0 +1,158 @@
+import asyncio
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+)
+
+from pydantic import BaseModel
+
+Prompt = Union[str, List[Dict[str, Any]]]
+PromptAndSchema = Tuple[
+    Union[str, List[Dict[str, Any]]], Optional[Dict[str, Any]]
+]
+SyncGen = Callable[[Prompt], Union[str, Any]]
+BatchGen = Callable[[List[Prompt]], List[Union[str, Any]]]
+AsyncGen = Callable[[Prompt], Awaitable[Union[str, Any]]]
+AsyncBatchGen = Callable[[List[Prompt]], Awaitable[List[Union[str, Any]]]]
+
+T = TypeVar("T")
+
+
+class PromptResult(BaseModel):
+    """
+    Holds the prompt sent and the response (or error).
+    """
+
+    prompt: Prompt
+    response: Optional[Any] = None
+    error: Optional[str] = None
+
+
+class PromptRunner:
+    """
+    Runs a collection of prompts through various generation strategies.
+
+    Attributes:
+        prompts: the list of prompts to run.
+    """
+
+    def __init__(
+        self, prompts: Optional[List[Union[Prompt, PromptAndSchema]]] = None
+    ) -> None:
+        """
+        Args:
+            prompts: initial list of prompts (strings or chat messages).
+        """
+        self.prompts: List[Union[Prompt, PromptAndSchema]] = prompts or []
+
+    def add_prompt(self, prompt: Union[Prompt, PromptAndSchema]) -> None:
+        """Append a prompt to the runner."""
+        self.prompts.append(prompt)
+
+    def remove_prompt(self, prompt: Union[Prompt, PromptAndSchema]) -> None:
+        """Remove a prompt (first occurrence)."""
+        self.prompts.remove(prompt)
+
+    def clear_prompts(self) -> None:
+        """Remove all prompts."""
+        self.prompts.clear()
+
+    def get_prompt_and_schema(
+        self, prompt: Union[Prompt, PromptAndSchema]
+    ) -> Tuple[Prompt, Optional[Dict[str, Any]]]:
+        """
+        Extract the prompt and schema from a Prompt object.
+
+        Args:
+            prompt: The prompt to extract from.
+
+        Returns:
+            Tuple of (prompt, schema).
+        """
+        if isinstance(prompt, tuple):
+            return prompt[0], prompt[1]
+        return prompt, None
+
+    def run_all(
+        self,
+        gen_fn: SyncGen,
+        prompt_param_name: str = "prompt",
+        schema_param_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> List[PromptResult]:
+        """
+        Run each prompt through a synchronous single-prompt generator.
+
+        Args:
+            gen_fn: Callable taking one Prompt, returning str or Any.
+            prompt_param_name: Name of the parameter for the prompt.
+            schema_param_name: Name of the parameter for the schema.
+            kwargs: Additional arguments to pass to the function.
+
+        Returns:
+            List of PromptResult.
+        """
+        results: List[PromptResult] = []
+        for p in self.prompts:
+            try:
+                prompt, schema = self.get_prompt_and_schema(p)
+                args = {prompt_param_name: prompt, **kwargs}
+                if schema_param_name and schema:
+                    args[schema_param_name] = schema
+                resp = gen_fn(**args)
+                results.append(PromptResult(prompt=prompt, response=resp))
+            except Exception as e:
+                results.append(PromptResult(prompt=prompt, error=str(e)))
+        return results
+
+    async def run_async(
+        self,
+        async_fn: AsyncGen,
+        max_parallel: int = 10,
+        prompt_param_name: str = "prompt",
+        schema_param_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> List[PromptResult]:
+        """
+        Run each prompt through an async single-prompt generator with concurrency limit.
+        Results are returned in the same order as self.prompts.
+
+        Args:
+            async_fn: Async callable taking one Prompt, returning str or Any.
+            max_parallel: Max concurrent tasks.
+            prompt_param_name: Name of the parameter for the prompt.
+            schema_param_name: Name of the parameter for the schema.
+            kwargs: Additional arguments to pass to the async function.
+
+        Returns:
+            List of PromptResult.
+        """
+        semaphore = asyncio.Semaphore(max_parallel)
+
+        async def _run_one(index: int, p: Prompt) -> Tuple[int, PromptResult]:
+            async with semaphore:
+                try:
+                    prompt, schema = self.get_prompt_and_schema(p)
+                    args = {prompt_param_name: prompt, **kwargs}
+                    if schema_param_name and schema:
+                        args[schema_param_name] = schema
+                    resp = await async_fn(**args)
+                    return index, PromptResult(prompt=prompt, response=resp)
+                except Exception as e:
+                    return index, PromptResult(prompt=prompt, error=str(e))
+
+        tasks = [
+            asyncio.create_task(_run_one(i, p))
+            for i, p in enumerate(self.prompts)
+        ]
+        indexed_results = await asyncio.gather(*tasks)
+        # Sort results to match original order
+        indexed_results.sort(key=lambda x: x[0])
+        return [res for _, res in indexed_results]

wxo_agentic_evaluation/referenceless_eval/referenceless_eval.py

@@ -0,0 +1,191 @@
+import json
+from typing import Any, List, Mapping, Optional
+
+import rich
+
+from wxo_agentic_evaluation.referenceless_eval.function_calling.consts import (
+    METRIC_FUNCTION_SELECTION_APPROPRIATENESS,
+    METRIC_GENERAL_HALLUCINATION_CHECK,
+)
+from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.pipeline import (
+    ReflectionPipeline,
+)
+from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.types import (
+    ToolCall,
+    ToolSpec,
+)
+from wxo_agentic_evaluation.runtime_adapter.wxo_runtime_adapter import (
+    WXORuntimeAdapter,
+)
+from wxo_agentic_evaluation.service_provider import get_provider
+from wxo_agentic_evaluation.type import Message
+from wxo_agentic_evaluation.utils.gateway_provider_utils import (
+    get_provider_kwargs,
+)
+
+DEFAULT_GENERATION_PARAMS = {
+    "min_new_tokens": 0,
+    "decoding_method": "greedy",
+    "max_new_tokens": 4096,
+}
+
+
+class ReferencelessEvaluation:
+    """
+    Note: static.final_decison, if `True` -> then all static metrics were valid. If false, atleast one of the static metrics failed. Look at explanation for reasoning
+    Note: if static.final_decision == True, check semantic metrics. Semantic metrics **not** run if static.final_decision is False.
+    ---
+    Note: For semantic metrics, check agentic constraints. If agent-constraints == False, no point in checking others. If true, check others.
+    Note: METRIC_FUNCTION_SELECTION_APPROPRIATENESS == False, implies that the LLM should have called some other function/tool before *OR* it is a redundant call.
+    Note: When parsing the semantic metrics, check for `is_correct` field.  if `false` there is some mistake that the LLMaJ found in that tool call.
+    """
+
+    def __init__(
+        self,
+        api_spec: List[Mapping[str, Any]],
+        model_id: str,
+        task_n: str,
+        dataset_name: str,
+        runtime_pipeline: bool = True,
+        generation_params=DEFAULT_GENERATION_PARAMS,
+        inference_backend: Optional[WXORuntimeAdapter] = None,
+    ):
+
+        extra_kwargs = {}
+        if inference_backend is not None:
+            wxo_client = getattr(inference_backend, "wxo_client")
+            instance_url = getattr(wxo_client, "service_url", None)
+            token = getattr(wxo_client, "api_key", None)
+            if instance_url:
+                extra_kwargs["instance_url"] = instance_url
+            if token:
+                extra_kwargs["token"] = token
+
+        self.metrics_client = ReferencelessEvaluation.get_metrics_client(
+            model_id=model_id,
+            params=generation_params,
+            referenceless_eval=True,
+            **extra_kwargs,
+        )
+
+        self.pipeline = ReflectionPipeline(
+            metrics_client=self.metrics_client,
+            general_metrics=[METRIC_GENERAL_HALLUCINATION_CHECK],
+            function_metrics=[METRIC_FUNCTION_SELECTION_APPROPRIATENESS],
+            parameter_metrics=None,
+            runtime_pipeline=runtime_pipeline,
+        )
+
+        self.task_n = task_n
+        self.dataset_name = dataset_name
+
+        self.apis_specs = [ToolSpec.model_validate(spec) for spec in api_spec]
+
+    @staticmethod
+    def get_metrics_client(**kwargs):
+
+        provider_kwargs = get_provider_kwargs(**kwargs)
+
+        return get_provider(
+            **provider_kwargs,
+        )
+
+    @staticmethod
+    def fmt_tool_call(tool_id, tool_call_name, arguments, context):
+        call = {
+            "call": {
+                "id": tool_id,
+                "type": "function",
+                "function": {
+                    "name": tool_call_name,
+                    "arguments": arguments,
+                },
+            },
+            "context": context,
+        }
+
+        return call
+
+    @staticmethod
+    def fmt_msgs_referenceless(
+        messages: List[Message],
+    ) -> List[Mapping[str, Any]]:
+        """Assume that the last item in the `messages` array is the tool call, and preceding items
+        in the messages array is the context.
+        """
+        examples = []
+        processed_data = [
+            {
+                k: msg.model_dump().get(k)
+                for k in ["role", "content", "type"]
+                if k in msg.model_dump()
+            }
+            for msg in messages
+        ]
+
+        for idx, message in enumerate(processed_data):
+            role = message["role"]
+            content = message["content"]
+            context = processed_data[:idx]
+
+            if role == "assistant" and message["type"] == "tool_call":
+                tool_call_msg = json.loads(content)
+                if tool_call_msg["name"].startswith("transfer_to"):
+                    continue
+
+                call = ReferencelessEvaluation.fmt_tool_call(
+                    tool_id=tool_call_msg.get("id", "1"),
+                    tool_call_name=tool_call_msg["name"],
+                    arguments=json.dumps(tool_call_msg["args"]),
+                    context=context,
+                )
+                examples.append(call)
+
+        return examples
+
+    def _run_pipeline(self, examples: List[Mapping[str, Any]]):
+        results = []
+        for example in examples:
+            result = self.pipeline.run_sync(
+                conversation=example["context"],
+                inventory=self.apis_specs,
+                call=example["call"],
+                continue_on_static=False,
+                retries=2,
+            )
+            result_dict = result.model_dump()
+            results.append(result_dict)
+
+        return results
+
+    def run(self, examples: List[Mapping[str, str]], verbose=False):
+        """`examples` should be an array where each element is formatted:
+
+        call = {
+            "call": {
+                "id": tool_call_msg.get("id", "1"),
+                "type": "function",
+                "function": {
+                    "name": tool_call_msg["name"],
+                    "arguments": json.dumps(tool_call_msg["args"]),
+                },
+            },
+            "context": context,
+        }
+        """
+
+        examples = [
+            {
+                "call": ToolCall.model_validate(ex["call"]),
+                "context": ex["context"],
+            }
+            for ex in examples
+        ]
+
+        if verbose:
+            rich.print(
+                f"[yellow][b][Task-{self.task_n}] There are {len(examples)} examples to analyze"
+            )
+        results = self._run_pipeline(examples)
+
+        return results

wxo_agentic_evaluation/resource_map.py

@@ -1,5 +1,7 @@
 from collections import defaultdict
-from wxo_agentic_evaluation.inference_backend import WXOClient, is_saas_url
+
+from wxo_agentic_evaluation.utils.utils import is_saas_url
+from wxo_agentic_evaluation.wxo_client import WXOClient
 
 
 class ResourceMap:
@@ -14,7 +16,7 @@ class ResourceMap:
         if is_saas_url(self.wxo_client.service_url):
             # TO-DO: this is not validated after the v1 prefix change
             # need additional validation
-            tools_path = "v1/orchestrate/tools/"
+            tools_path = "v1/orchestrate/tools"
             agents_path = "v1/orchestrate/agents"
         else:
             tools_path = "v1/tools/"
@@ -33,6 +35,7 @@ class ResourceMap:
 
         if resp.status_code == 200:
             agents = resp.json()
+            self.all_agent_objs = agents
             for agent in agents:
                 agent_name = agent["name"]
                 tools = [tool_map[id] for id in agent["tools"]]
@@ -44,4 +47,4 @@ class ResourceMap:
 
         agent2tools = dict(agent2tools)
         tools2agents = dict(tools2agents)
-        return agent2tools, tools2agents
+        return agent2tools, tools2agents

wxo_agentic_evaluation/runner.py

@@ -0,0 +1,329 @@
+import json
+import os
+from pathlib import Path
+from typing import List, Tuple
+
+import rich
+
+from wxo_agentic_evaluation.arg_configs import TestConfig
+from wxo_agentic_evaluation.evaluation_controller.evaluation_controller import (
+    EvaluationController,
+)
+from wxo_agentic_evaluation.evaluation_package import EvaluationPackage
+from wxo_agentic_evaluation.llm_user import LLMUser
+from wxo_agentic_evaluation.metrics.metrics import (
+    CustomEvalMetrics,
+    KnowledgeBaseMetricSummary,
+    ToolCallAndRoutingMetrics,
+)
+from wxo_agentic_evaluation.resource_map import ResourceMap
+from wxo_agentic_evaluation.runtime_adapter.wxo_runtime_adapter import (
+    WXORuntimeAdapter,
+)
+from wxo_agentic_evaluation.service_provider.provider import Provider
+from wxo_agentic_evaluation.type import OrchestrateDataset
+from wxo_agentic_evaluation.utils import json_dump
+from wxo_agentic_evaluation.utils.evaluation_discovery import (
+    find_evaluation_subclasses,
+)
+
+
+def _save_data(
+    config: TestConfig,
+    test_case_name: str,
+    run_tag: str,
+    data,
+    file_path: str | None = None,
+    file_suffix: str | None = None,
+) -> None:
+    """
+    Save data to a JSON file.
+
+    Args:
+        config: Test configuration
+        test_case_name: Test case name
+        run_tag: Run tag
+        data: Data to save
+        file_path: Complete file path (optional)
+        file_suffix: File suffix for messages directory (optional)
+    """
+    if file_path:
+        json_dump(str(file_path), data)
+    elif file_suffix:
+        json_dump(
+            os.path.join(
+                config.output_dir,
+                "messages",
+                f"{test_case_name}{run_tag}{file_suffix}",
+            ),
+            data,
+        )
+
+    # Handle conversational search data
+    if (
+        isinstance(data, list)
+        and data
+        and hasattr(data[0], "model_dump")
+        and file_suffix == ".retrieval_context.json"
+    ):
+        out_folder = Path(config.output_dir) / "knowledge_base_metrics"
+        out_folder.mkdir(exist_ok=True)
+        retrieval_context = [context.model_dump() for context in data]
+        json_dump(
+            str(out_folder / f"{test_case_name}{run_tag}{file_suffix}"),
+            retrieval_context,
+        )
+
+
+def _process_tool_calls(
+    history: List,
+    evaluation_data: OrchestrateDataset,
+    resource_map: ResourceMap,
+) -> Tuple[List[str], List[str], List[str]]:
+    """
+    Process tool calls from history and evaluation data.
+
+    Args:
+        history: Message history
+        evaluation_data: evaluation data
+        resource_map: Resource map
+
+    Returns:
+        Tuple of (expected tool calls, actual tool calls, missed tool calls)
+    """
+    expected_tools = [
+        goal_detail.tool_name
+        for goal_detail in evaluation_data.goal_details
+        if getattr(goal_detail, "type", None) == "tool_call"
+    ]
+
+    raw_actual = []
+    for message in history:
+        try:
+            if getattr(message, "type", None) == "tool_call":
+                payload = (
+                    json.loads(message.content)
+                    if isinstance(message.content, str)
+                    else message.content
+                )
+                name = (payload or {}).get("name")
+                if name:
+                    raw_actual.append(str(name).strip())
+        except Exception:
+            pass
+
+    expected_set = set(expected_tools)
+    agent_names = (
+        set(getattr(resource_map, "agent2tools", {}).keys())
+        if resource_map
+        else set()
+    )
+
+    filtered_actual_tool_calls = [
+        name for name in raw_actual if name not in agent_names
+    ]
+    missed_tool_calls = sorted(expected_set - set(filtered_actual_tool_calls))
+
+    return expected_tools, filtered_actual_tool_calls, missed_tool_calls
+
+
+def process_test_case(
+    task_n: int,
+    test_case: str,
+    config: TestConfig,
+    runtime_adapter: WXORuntimeAdapter,
+    resource_map: ResourceMap,
+    llm_user: LLMUser,
+    llmaaj_provider: Provider,
+    run_idx: int = 0,
+) -> List[
+    Tuple[
+        ToolCallAndRoutingMetrics, KnowledgeBaseMetricSummary, CustomEvalMetrics
+    ]
+]:
+    """
+    Process a single test case.
+
+    Args:
+        task_n: Task number
+        test_case: Path to the test case file
+        config: Test configuration
+        inference_backend: Inference backend
+        resource_map: Resource map
+        llm_user: LLM user
+        llmaaj_provider: Provider for custom metrics
+        run_idx: Run index
+
+    Returns:
+        List of tuples (metrics, knowledge_base_metrics, custom_metrics)
+    """
+    summary_results_for_path = []
+    test_case_name = os.path.basename(test_case).replace(".json", "")
+    run_tag = f".run{run_idx+1}" if config.n_runs > 1 else ""
+
+    with open(test_case, "r") as f:
+        evaluation_data = OrchestrateDataset.model_validate(json.load(f))
+
+    # Set up evaluation controller and run test
+    evaluation_controller = EvaluationController(
+        runtime=runtime_adapter,
+        llm_user=llm_user,
+        config=config,
+    )
+
+    rich.print(
+        f"[bold magenta]Running test case: {test_case_name}[/bold magenta]"
+    )
+
+    # Run the evaluation
+    history, call_tracker, conversational_search_data, _ = (
+        evaluation_controller.run(
+            task_n,
+            story=evaluation_data.story,
+            agent_name=evaluation_data.agent,
+            starting_user_input=evaluation_data.starting_sentence,
+            max_user_turns=evaluation_data.max_user_turns,
+        )
+    )
+    
+    # Save metadata (that contains thread_id)
+    json_dump(
+        os.path.join(
+            config.output_dir,
+            f"{test_case_name}{run_tag}.metadata.json",
+        ),
+        call_tracker.metadata,
+    )
+
+    if config.skip_legacy_evaluation:
+        return summary_results_for_path  # empty result set, skip evaluation
+
+    # Save message history
+    result = [message.model_dump() for message in history]
+    _save_data(
+        config, test_case_name, run_tag, result, file_suffix=".messages.json"
+    )
+
+    # Save conversational search data if available
+    if conversational_search_data:
+        retrieval_context = [
+            context.model_dump() for context in conversational_search_data
+        ]
+        out_folder = Path(config.output_dir) / "knowledge_base_metrics"
+        out_folder.mkdir(exist_ok=True)
+        file_path = str(
+            out_folder / f"{test_case_name}{run_tag}.retrieval_context.json"
+        )
+        _save_data(
+            config,
+            test_case_name,
+            run_tag,
+            retrieval_context,
+            file_path=file_path,
+        )
+
+    # If data annotation run, skip summary generation
+    if config.data_annotation_run:
+        return summary_results_for_path  # empty result set, skip summary
+
+    # Load custom extractors and evaluations
+    all_extractors = []
+    all_custom_evals = []
+
+    # Load custom extractors
+    if config.extractors_config.paths is not None:
+        for path in config.extrators_config.paths:
+            extractors = find_evaluation_subclasses(
+                directory=path, base_class_name="Extractor"
+            )
+            for extractor_class in extractors:
+                all_extractors.append(extractor_class())
+
+    # Load custom evaluations
+    if config.custom_metrics_config.paths is not None:
+        for path in config.custom_metrics_config.paths:
+            custom_eval_classes = find_evaluation_subclasses(path)
+            for _class in custom_eval_classes:
+                all_custom_evals.append(_class(llm_client=llmaaj_provider))
+
+    # Create evaluation package and generate summary
+    evaluation_package = EvaluationPackage(
+        test_case_name=test_case_name,
+        messages=history,
+        ground_truth=evaluation_data,
+        conversational_search_data=conversational_search_data,
+        resource_map=resource_map,
+        config=config,
+        custom_evals=all_custom_evals,
+        extractors=all_extractors,
+        similarity_threshold=config.similarity_threshold,
+        enable_fuzzy_matching=config.enable_fuzzy_matching,
+        strict_topological_matching=config.strict_topological_matching,
+    )
+
+    # Generate summary
+    (
+        _keyword_semantic_matches,
+        knowledge_base_metrics,
+        messages_with_reason,
+        metrics,
+        custom_metrics,
+    ) = evaluation_package.generate_summary()
+
+    # Process messages with reason
+    temp = [message.model_dump() for message in messages_with_reason]
+
+    # Process tool calls
+    expected_tools, filtered_actual_tool_calls, missed_tool_calls = (
+        _process_tool_calls(history, evaluation_data, resource_map)
+    )
+
+    # Add meta information
+    temp.append(
+        {
+            "meta": {
+                "expected_tool_calls": expected_tools,
+                "actual_tool_calls": filtered_actual_tool_calls,
+                "missed_tool_calls": missed_tool_calls,
+            }
+        }
+    )
+
+    # Save analysis results
+    _save_data(
+        config,
+        test_case_name,
+        run_tag,
+        temp,
+        file_suffix=".messages.analyze.json",
+    )
+    _save_data(
+        config,
+        test_case_name,
+        run_tag,
+        metrics.model_dump(),
+        file_suffix=".metrics.json",
+    )
+
+    # Update metrics
+    metrics.dataset_name = test_case_name
+
+    # Calculate average response time
+    metrics.avg_resp_time = 0.0
+    if hasattr(call_tracker, "generic") and hasattr(call_tracker, "tool_call"):
+        generic_calls = getattr(call_tracker, "generic", [])
+        tool_calls = getattr(call_tracker, "tool_call", [])
+
+        if generic_calls or tool_calls:
+            total_time = sum(generic_calls) + sum(tool_calls)
+            total_calls = len(generic_calls) + len(tool_calls)
+            if total_calls > 0:
+                metrics.avg_resp_time = round(total_time / total_calls, 2)
+    metrics.avg_resp_time = round(metrics.avg_resp_time, 2)
+
+    # Add results to summary
+    summary_results_for_path.append(
+        (metrics, knowledge_base_metrics, custom_metrics)
+    )
+
+    return summary_results_for_path