ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/referenceless_eval/function_calling/pipeline/types.py

@@ -390,18 +390,18 @@ class PipelineResult(BaseModel):
     )
 
     @model_validator(mode="after")
-    def compute_overall(cls, values: PipelineResult) -> PipelineResult:
+    def compute_overall(self) -> Self:
         """
         After validation, compute overall_valid as AND of:
           • all semantic is_correct flags
           • if transform exists: all execution_success flags
         """
-        static: StaticResult = values.static
+        static: StaticResult = self.static
         if static:
             # static checks
             ok = static.final_decision
 
-        sem: SemanticResult = values.semantic
+        sem: SemanticResult = self.semantic
         if sem:
             # semantic checks
             if sem.general and sem.general.metrics:
@@ -441,11 +441,11 @@ class PipelineResult(BaseModel):
             if param_avgs:
                 cat_avgs.append(sum(param_avgs) / len(param_avgs))
 
-        values.overall_avg_score = (
+        self.overall_avg_score = (
             sum(cat_avgs) / len(cat_avgs) if cat_avgs else None
         )
-        values.overall_valid = ok
-        return values
+        self.overall_valid = ok
+        return self
 
 
 # ----------------------------------------------------------------------
@@ -531,17 +531,17 @@ class ToolFunctionCall(BaseModel):
     )
 
     @model_validator(mode="after")
-    def _parse_arguments(cls, values: ToolFunctionCall) -> ToolFunctionCall:
+    def _parse_arguments(self) -> Self:
         """
         After model construction, parse the `arguments` JSON string
         into `parsed_arguments`, or raise a ValidationError.
         """
         try:
-            raw = values.arguments
-            values.parsed_arguments = json.loads(raw)
+            raw = self.arguments
+            self.parsed_arguments = json.loads(raw)
         except json.JSONDecodeError as e:
             raise ValidationError(f"Invalid JSON in arguments: {e}") from e
-        return values
+        return self
 
 
 class ToolCall(BaseModel):

wxo_agentic_evaluation/referenceless_eval/referenceless_eval.py

@@ -1,5 +1,5 @@
 import json
-from typing import Any, List, Mapping
+from typing import Any, List, Mapping, Optional
 
 import rich
 
@@ -14,8 +14,20 @@ from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.types i
     ToolCall,
     ToolSpec,
 )
+from wxo_agentic_evaluation.runtime_adapter.wxo_runtime_adapter import (
+    WXORuntimeAdapter,
+)
 from wxo_agentic_evaluation.service_provider import get_provider
 from wxo_agentic_evaluation.type import Message
+from wxo_agentic_evaluation.utils.gateway_provider_utils import (
+    get_provider_kwargs,
+)
+
+DEFAULT_GENERATION_PARAMS = {
+    "min_new_tokens": 0,
+    "decoding_method": "greedy",
+    "max_new_tokens": 4096,
+}
 
 
 class ReferencelessEvaluation:
@@ -31,20 +43,29 @@ class ReferencelessEvaluation:
     def __init__(
         self,
         api_spec: List[Mapping[str, Any]],
-        messages: List[Message],
         model_id: str,
         task_n: str,
         dataset_name: str,
+        runtime_pipeline: bool = True,
+        generation_params=DEFAULT_GENERATION_PARAMS,
+        inference_backend: Optional[WXORuntimeAdapter] = None,
     ):
 
-        self.metrics_client = get_provider(
+        extra_kwargs = {}
+        if inference_backend is not None:
+            wxo_client = getattr(inference_backend, "wxo_client")
+            instance_url = getattr(wxo_client, "service_url", None)
+            token = getattr(wxo_client, "api_key", None)
+            if instance_url:
+                extra_kwargs["instance_url"] = instance_url
+            if token:
+                extra_kwargs["token"] = token
+
+        self.metrics_client = ReferencelessEvaluation.get_metrics_client(
             model_id=model_id,
-            params={
-                "min_new_tokens": 0,
-                "decoding_method": "greedy",
-                "max_new_tokens": 4096,
-            },
+            params=generation_params,
             referenceless_eval=True,
+            **extra_kwargs,
         )
 
         self.pipeline = ReflectionPipeline(
@@ -52,39 +73,54 @@ class ReferencelessEvaluation:
             general_metrics=[METRIC_GENERAL_HALLUCINATION_CHECK],
             function_metrics=[METRIC_FUNCTION_SELECTION_APPROPRIATENESS],
             parameter_metrics=None,
+            runtime_pipeline=runtime_pipeline,
         )
 
         self.task_n = task_n
         self.dataset_name = dataset_name
 
         self.apis_specs = [ToolSpec.model_validate(spec) for spec in api_spec]
-        self.messages = messages
 
-    def _run_pipeline(self, examples: List[Mapping[str, Any]]):
-        results = []
-        for example in examples:
-            result = self.pipeline.run_sync(
-                conversation=example["context"],
-                inventory=self.apis_specs,
-                call=example["call"],
-                continue_on_static=False,
-                retries=2,
-            )
-            result_dict = result.model_dump()
-            results.append(result_dict)
+    @staticmethod
+    def get_metrics_client(**kwargs):
 
-        return results
+        provider_kwargs = get_provider_kwargs(**kwargs)
 
-    def run(self):
-        examples = []
+        return get_provider(
+            **provider_kwargs,
+        )
+
+    @staticmethod
+    def fmt_tool_call(tool_id, tool_call_name, arguments, context):
+        call = {
+            "call": {
+                "id": tool_id,
+                "type": "function",
+                "function": {
+                    "name": tool_call_name,
+                    "arguments": arguments,
+                },
+            },
+            "context": context,
+        }
+
+        return call
 
+    @staticmethod
+    def fmt_msgs_referenceless(
+        messages: List[Message],
+    ) -> List[Mapping[str, Any]]:
+        """Assume that the last item in the `messages` array is the tool call, and preceding items
+        in the messages array is the context.
+        """
+        examples = []
         processed_data = [
             {
                 k: msg.model_dump().get(k)
                 for k in ["role", "content", "type"]
                 if k in msg.model_dump()
             }
-            for msg in self.messages
+            for msg in messages
         ]
 
         for idx, message in enumerate(processed_data):
@@ -97,22 +133,47 @@ class ReferencelessEvaluation:
                 if tool_call_msg["name"].startswith("transfer_to"):
                     continue
 
-                call = {
-                    "call": {
-                        "id": tool_call_msg.get("id", "1"),
-                        "type": "function",
-                        "function": {
-                            "name": tool_call_msg["name"],
-                            "arguments": json.dumps(tool_call_msg["args"]),
-                        },
-                    },
-                    "context": context,
-                }
+                call = ReferencelessEvaluation.fmt_tool_call(
+                    tool_id=tool_call_msg.get("id", "1"),
+                    tool_call_name=tool_call_msg["name"],
+                    arguments=json.dumps(tool_call_msg["args"]),
+                    context=context,
+                )
                 examples.append(call)
 
-        rich.print(
-            f"[yellow][b][Task-{self.task_n}] There are {len(examples)} examples to analyze"
-        )
+        return examples
+
+    def _run_pipeline(self, examples: List[Mapping[str, Any]]):
+        results = []
+        for example in examples:
+            result = self.pipeline.run_sync(
+                conversation=example["context"],
+                inventory=self.apis_specs,
+                call=example["call"],
+                continue_on_static=False,
+                retries=2,
+            )
+            result_dict = result.model_dump()
+            results.append(result_dict)
+
+        return results
+
+    def run(self, examples: List[Mapping[str, str]], verbose=False):
+        """`examples` should be an array where each element is formatted:
+
+        call = {
+            "call": {
+                "id": tool_call_msg.get("id", "1"),
+                "type": "function",
+                "function": {
+                    "name": tool_call_msg["name"],
+                    "arguments": json.dumps(tool_call_msg["args"]),
+                },
+            },
+            "context": context,
+        }
+        """
+
         examples = [
             {
                 "call": ToolCall.model_validate(ex["call"]),
@@ -120,6 +181,11 @@ class ReferencelessEvaluation:
             }
             for ex in examples
         ]
+
+        if verbose:
+            rich.print(
+                f"[yellow][b][Task-{self.task_n}] There are {len(examples)} examples to analyze"
+            )
         results = self._run_pipeline(examples)
 
         return results

wxo_agentic_evaluation/resource_map.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 
-from wxo_agentic_evaluation.inference_backend import WXOClient, is_saas_url
+from wxo_agentic_evaluation.utils.utils import is_saas_url
+from wxo_agentic_evaluation.wxo_client import WXOClient
 
 
 class ResourceMap:
@@ -34,6 +35,7 @@ class ResourceMap:
 
         if resp.status_code == 200:
             agents = resp.json()
+            self.all_agent_objs = agents
             for agent in agents:
                 agent_name = agent["name"]
                 tools = [tool_map[id] for id in agent["tools"]]

wxo_agentic_evaluation/runner.py

@@ -0,0 +1,329 @@
+import json
+import os
+from pathlib import Path
+from typing import List, Tuple
+
+import rich
+
+from wxo_agentic_evaluation.arg_configs import TestConfig
+from wxo_agentic_evaluation.evaluation_controller.evaluation_controller import (
+    EvaluationController,
+)
+from wxo_agentic_evaluation.evaluation_package import EvaluationPackage
+from wxo_agentic_evaluation.llm_user import LLMUser
+from wxo_agentic_evaluation.metrics.metrics import (
+    CustomEvalMetrics,
+    KnowledgeBaseMetricSummary,
+    ToolCallAndRoutingMetrics,
+)
+from wxo_agentic_evaluation.resource_map import ResourceMap
+from wxo_agentic_evaluation.runtime_adapter.wxo_runtime_adapter import (
+    WXORuntimeAdapter,
+)
+from wxo_agentic_evaluation.service_provider.provider import Provider
+from wxo_agentic_evaluation.type import OrchestrateDataset
+from wxo_agentic_evaluation.utils import json_dump
+from wxo_agentic_evaluation.utils.evaluation_discovery import (
+    find_evaluation_subclasses,
+)
+
+
+def _save_data(
+    config: TestConfig,
+    test_case_name: str,
+    run_tag: str,
+    data,
+    file_path: str | None = None,
+    file_suffix: str | None = None,
+) -> None:
+    """
+    Save data to a JSON file.
+
+    Args:
+        config: Test configuration
+        test_case_name: Test case name
+        run_tag: Run tag
+        data: Data to save
+        file_path: Complete file path (optional)
+        file_suffix: File suffix for messages directory (optional)
+    """
+    if file_path:
+        json_dump(str(file_path), data)
+    elif file_suffix:
+        json_dump(
+            os.path.join(
+                config.output_dir,
+                "messages",
+                f"{test_case_name}{run_tag}{file_suffix}",
+            ),
+            data,
+        )
+
+    # Handle conversational search data
+    if (
+        isinstance(data, list)
+        and data
+        and hasattr(data[0], "model_dump")
+        and file_suffix == ".retrieval_context.json"
+    ):
+        out_folder = Path(config.output_dir) / "knowledge_base_metrics"
+        out_folder.mkdir(exist_ok=True)
+        retrieval_context = [context.model_dump() for context in data]
+        json_dump(
+            str(out_folder / f"{test_case_name}{run_tag}{file_suffix}"),
+            retrieval_context,
+        )
+
+
+def _process_tool_calls(
+    history: List,
+    evaluation_data: OrchestrateDataset,
+    resource_map: ResourceMap,
+) -> Tuple[List[str], List[str], List[str]]:
+    """
+    Process tool calls from history and evaluation data.
+
+    Args:
+        history: Message history
+        evaluation_data: evaluation data
+        resource_map: Resource map
+
+    Returns:
+        Tuple of (expected tool calls, actual tool calls, missed tool calls)
+    """
+    expected_tools = [
+        goal_detail.tool_name
+        for goal_detail in evaluation_data.goal_details
+        if getattr(goal_detail, "type", None) == "tool_call"
+    ]
+
+    raw_actual = []
+    for message in history:
+        try:
+            if getattr(message, "type", None) == "tool_call":
+                payload = (
+                    json.loads(message.content)
+                    if isinstance(message.content, str)
+                    else message.content
+                )
+                name = (payload or {}).get("name")
+                if name:
+                    raw_actual.append(str(name).strip())
+        except Exception:
+            pass
+
+    expected_set = set(expected_tools)
+    agent_names = (
+        set(getattr(resource_map, "agent2tools", {}).keys())
+        if resource_map
+        else set()
+    )
+
+    filtered_actual_tool_calls = [
+        name for name in raw_actual if name not in agent_names
+    ]
+    missed_tool_calls = sorted(expected_set - set(filtered_actual_tool_calls))
+
+    return expected_tools, filtered_actual_tool_calls, missed_tool_calls
+
+
+def process_test_case(
+    task_n: int,
+    test_case: str,
+    config: TestConfig,
+    runtime_adapter: WXORuntimeAdapter,
+    resource_map: ResourceMap,
+    llm_user: LLMUser,
+    llmaaj_provider: Provider,
+    run_idx: int = 0,
+) -> List[
+    Tuple[
+        ToolCallAndRoutingMetrics, KnowledgeBaseMetricSummary, CustomEvalMetrics
+    ]
+]:
+    """
+    Process a single test case.
+
+    Args:
+        task_n: Task number
+        test_case: Path to the test case file
+        config: Test configuration
+        inference_backend: Inference backend
+        resource_map: Resource map
+        llm_user: LLM user
+        llmaaj_provider: Provider for custom metrics
+        run_idx: Run index
+
+    Returns:
+        List of tuples (metrics, knowledge_base_metrics, custom_metrics)
+    """
+    summary_results_for_path = []
+    test_case_name = os.path.basename(test_case).replace(".json", "")
+    run_tag = f".run{run_idx+1}" if config.n_runs > 1 else ""
+
+    with open(test_case, "r") as f:
+        evaluation_data = OrchestrateDataset.model_validate(json.load(f))
+
+    # Set up evaluation controller and run test
+    evaluation_controller = EvaluationController(
+        runtime=runtime_adapter,
+        llm_user=llm_user,
+        config=config,
+    )
+
+    rich.print(
+        f"[bold magenta]Running test case: {test_case_name}[/bold magenta]"
+    )
+
+    # Run the evaluation
+    history, call_tracker, conversational_search_data, _ = (
+        evaluation_controller.run(
+            task_n,
+            story=evaluation_data.story,
+            agent_name=evaluation_data.agent,
+            starting_user_input=evaluation_data.starting_sentence,
+            max_user_turns=evaluation_data.max_user_turns,
+        )
+    )
+    
+    # Save metadata (that contains thread_id)
+    json_dump(
+        os.path.join(
+            config.output_dir,
+            f"{test_case_name}{run_tag}.metadata.json",
+        ),
+        call_tracker.metadata,
+    )
+
+    if config.skip_legacy_evaluation:
+        return summary_results_for_path  # empty result set, skip evaluation
+
+    # Save message history
+    result = [message.model_dump() for message in history]
+    _save_data(
+        config, test_case_name, run_tag, result, file_suffix=".messages.json"
+    )
+
+    # Save conversational search data if available
+    if conversational_search_data:
+        retrieval_context = [
+            context.model_dump() for context in conversational_search_data
+        ]
+        out_folder = Path(config.output_dir) / "knowledge_base_metrics"
+        out_folder.mkdir(exist_ok=True)
+        file_path = str(
+            out_folder / f"{test_case_name}{run_tag}.retrieval_context.json"
+        )
+        _save_data(
+            config,
+            test_case_name,
+            run_tag,
+            retrieval_context,
+            file_path=file_path,
+        )
+
+    # If data annotation run, skip summary generation
+    if config.data_annotation_run:
+        return summary_results_for_path  # empty result set, skip summary
+
+    # Load custom extractors and evaluations
+    all_extractors = []
+    all_custom_evals = []
+
+    # Load custom extractors
+    if config.extractors_config.paths is not None:
+        for path in config.extrators_config.paths:
+            extractors = find_evaluation_subclasses(
+                directory=path, base_class_name="Extractor"
+            )
+            for extractor_class in extractors:
+                all_extractors.append(extractor_class())
+
+    # Load custom evaluations
+    if config.custom_metrics_config.paths is not None:
+        for path in config.custom_metrics_config.paths:
+            custom_eval_classes = find_evaluation_subclasses(path)
+            for _class in custom_eval_classes:
+                all_custom_evals.append(_class(llm_client=llmaaj_provider))
+
+    # Create evaluation package and generate summary
+    evaluation_package = EvaluationPackage(
+        test_case_name=test_case_name,
+        messages=history,
+        ground_truth=evaluation_data,
+        conversational_search_data=conversational_search_data,
+        resource_map=resource_map,
+        config=config,
+        custom_evals=all_custom_evals,
+        extractors=all_extractors,
+        similarity_threshold=config.similarity_threshold,
+        enable_fuzzy_matching=config.enable_fuzzy_matching,
+        strict_topological_matching=config.strict_topological_matching,
+    )
+
+    # Generate summary
+    (
+        _keyword_semantic_matches,
+        knowledge_base_metrics,
+        messages_with_reason,
+        metrics,
+        custom_metrics,
+    ) = evaluation_package.generate_summary()
+
+    # Process messages with reason
+    temp = [message.model_dump() for message in messages_with_reason]
+
+    # Process tool calls
+    expected_tools, filtered_actual_tool_calls, missed_tool_calls = (
+        _process_tool_calls(history, evaluation_data, resource_map)
+    )
+
+    # Add meta information
+    temp.append(
+        {
+            "meta": {
+                "expected_tool_calls": expected_tools,
+                "actual_tool_calls": filtered_actual_tool_calls,
+                "missed_tool_calls": missed_tool_calls,
+            }
+        }
+    )
+
+    # Save analysis results
+    _save_data(
+        config,
+        test_case_name,
+        run_tag,
+        temp,
+        file_suffix=".messages.analyze.json",
+    )
+    _save_data(
+        config,
+        test_case_name,
+        run_tag,
+        metrics.model_dump(),
+        file_suffix=".metrics.json",
+    )
+
+    # Update metrics
+    metrics.dataset_name = test_case_name
+
+    # Calculate average response time
+    metrics.avg_resp_time = 0.0
+    if hasattr(call_tracker, "generic") and hasattr(call_tracker, "tool_call"):
+        generic_calls = getattr(call_tracker, "generic", [])
+        tool_calls = getattr(call_tracker, "tool_call", [])
+
+        if generic_calls or tool_calls:
+            total_time = sum(generic_calls) + sum(tool_calls)
+            total_calls = len(generic_calls) + len(tool_calls)
+            if total_calls > 0:
+                metrics.avg_resp_time = round(total_time / total_calls, 2)
+    metrics.avg_resp_time = round(metrics.avg_resp_time, 2)
+
+    # Add results to summary
+    summary_results_for_path.append(
+        (metrics, knowledge_base_metrics, custom_metrics)
+    )
+
+    return summary_results_for_path

wxo_agentic_evaluation/runtime_adapter/runtime_adapter.py

@@ -0,0 +1,14 @@
+from wxo_agentic_evaluation.type import CallTracker, Message, RuntimeResponse
+from abc import abstractmethod
+
+
+class RuntimeAdapter:
+
+    @abstractmethod
+    def run(
+        self,
+        user_message: Message,
+        context: dict,
+        thread_id=None,
+    ) -> RuntimeResponse:
+        pass