ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/referenceless_eval/function_calling/metrics/loader.py

@@ -0,0 +1,245 @@
+import json
+from enum import Enum
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Tuple, Union
+
+from pydantic import ValidationError
+
+from wxo_agentic_evaluation.referenceless_eval.function_calling.metrics.function_call.general import (
+    GeneralMetricsPrompt,
+)
+from wxo_agentic_evaluation.referenceless_eval.function_calling.metrics.function_selection.function_selection import (
+    FunctionSelectionPrompt,
+)
+from wxo_agentic_evaluation.referenceless_eval.metrics import (
+    Metric,
+    MetricPrompt,
+)
+
+PromptType = Union[
+    GeneralMetricsPrompt,
+    FunctionSelectionPrompt,
+]
+
+
+# Enum for prompt kinds
+class PromptKind(str, Enum):
+    GENERAL = "general"
+    FUNCTION_SELECTION = "function_selection"
+    PARAMETER = "parameter"
+
+
+# Map enum → Prompt class
+_PROMPT_CLASS_MAP: Dict[PromptKind, Any] = {
+    PromptKind.GENERAL: GeneralMetricsPrompt,
+    PromptKind.FUNCTION_SELECTION: FunctionSelectionPrompt,
+}
+
+
+class LoaderError(Exception):
+    """Raised when prompt loading fails."""
+
+
+def load_prompts_from_jsonl(
+    path: Union[str, Path],
+    kind: PromptKind,
+) -> List[PromptType]:
+    """
+    Load prompts from a JSONL file.
+
+    Args:
+        path: .jsonl file path.
+        kind: PromptKind value.
+
+    Returns:
+        List of PromptType, each with its examples loaded.
+
+    Raises:
+        LoaderError on I/O, parse, or validation errors.
+    """
+    PromptCls = _PROMPT_CLASS_MAP.get(kind)
+    if PromptCls is None:
+        raise LoaderError(f"Unknown PromptKind: {kind}")
+
+    p = Path(path)
+    if not p.is_file():
+        raise LoaderError(f"File not found: {path}")
+
+    prompts: List[PromptType] = []
+    for lineno, raw in enumerate(
+        p.read_text(encoding="utf-8").splitlines(), start=1
+    ):
+        if not raw.strip():
+            continue
+        try:
+            rec = json.loads(raw)
+        except json.JSONDecodeError as e:
+            raise LoaderError(f"{path}:{lineno} invalid JSON: {e}") from e
+
+        # Extract
+        try:
+            schema = rec["jsonschema"]
+            examples = rec.get("examples", [])
+            description = rec.get("description", schema.get("description", ""))
+        except KeyError as e:
+            raise LoaderError(f"{path}:{lineno} missing key {e}") from e
+
+        # Build metric
+        try:
+            metric = Metric.from_jsonschema(schema)
+            metric.description = description
+        except Exception as e:
+            raise LoaderError(f"{path}:{lineno} invalid schema: {e}") from e
+
+        # Instantiate prompt
+        prompt: MetricPrompt
+        try:
+            prompt = PromptCls(
+                metric=metric, task_description=metric.description
+            )
+        except TypeError:
+            prompt = PromptCls(metric=metric)
+
+        # Load examples
+        for ex_idx, ex in enumerate(examples, start=1):
+            try:
+                user_kwargs = ex["user_kwargs"]
+                output = ex["output"]
+            except KeyError as e:
+                raise LoaderError(
+                    f"{path}:{lineno}, example {ex_idx} missing {e}"
+                ) from e
+            try:
+                prompt.add_example(user_kwargs, output)
+            except (ValidationError, ValueError) as e:
+                raise LoaderError(
+                    f"{path}:{lineno}, example {ex_idx} invalid: {e}"
+                ) from e
+
+        prompts.append(prompt)
+
+    return prompts
+
+
+def load_prompts_from_list(
+    records: Iterable[Dict[str, Any]], kind: PromptKind
+) -> List[PromptType]:
+    """
+    Load prompts from an in-memory list of dicts, same structure as JSONL.
+
+    Args:
+        records: Iterable of dicts with keys {schema, thresholds, examples, description}.
+        kind: PromptKind value.
+
+    Returns:
+        List of PromptType.
+
+    Raises:
+        LoaderError on missing data or validation failures.
+    """
+    PromptCls = _PROMPT_CLASS_MAP.get(kind)
+    if PromptCls is None:
+        raise LoaderError(f"Unknown PromptKind: {kind}")
+
+    prompts: List[PromptType] = []
+    for idx, rec in enumerate(records, start=1):
+        # same logic as JSONL loader
+        try:
+            schema = rec["jsonschema"]
+            examples = rec.get("examples", [])
+            description = schema.get("description", rec.get("name", ""))
+        except KeyError as e:
+            raise LoaderError(f"Record {idx} missing key {e}") from e
+
+        try:
+            metric = Metric.from_jsonschema(schema)
+            metric.description = description
+        except Exception as e:
+            raise LoaderError(f"Record {idx} invalid schema: {e}") from e
+
+        try:
+            prompt = PromptCls(
+                metric=metric, task_description=rec["task_description"]
+            )
+        except TypeError:
+            prompt = PromptCls(metric=metric)
+
+        for ex_idx, ex in enumerate(examples, start=1):
+            try:
+                user_kwargs = ex["user_kwargs"]
+                output = ex["output"]
+            except KeyError as e:
+                raise LoaderError(
+                    f"Record {idx}, example {ex_idx} missing {e}"
+                ) from e
+            try:
+                prompt.add_example(user_kwargs, output)
+            except (ValidationError, ValueError) as e:
+                raise LoaderError(
+                    f"Record {idx}, example {ex_idx} invalid: {e}"
+                ) from e
+
+        prompts.append(prompt)
+
+    return prompts
+
+
+def load_prompts_from_metrics(
+    metrics_with_examples: Iterable[Tuple[Metric, List[Dict[str, Any]]]],
+    kind: PromptKind,
+) -> List[PromptType]:
+    """
+    Instantiate prompts directly from Metric objects and example data.
+
+    Args:
+        metrics_with_examples: An iterable of (Metric instance, examples) tuples.
+            Each examples list item must be a dict with:
+              - "user_kwargs": Dict[str, Any]
+              - "output": Dict[str, Any]
+        kind: Which PromptKind to use (GENERAL, FUNCTION_SELECTION, PARAMETER).
+
+    Returns:
+        A list of PromptType, each with its few-shot examples loaded.
+
+    Raises:
+        LoaderError: on missing data or validation errors.
+    """
+    PromptCls = _PROMPT_CLASS_MAP.get(kind)
+    if PromptCls is None:
+        raise LoaderError(f"Unknown PromptKind: {kind}")
+
+    prompts: List[PromptType] = []
+    for idx, (metric, examples) in enumerate(metrics_with_examples, start=1):
+        if not isinstance(metric, Metric):
+            raise LoaderError(
+                f"Item {idx}: expected a Metric instance, got {type(metric)}"
+            )
+
+        # Instantiate prompt with the metric's description as task_description
+        try:
+            prompt = PromptCls(
+                metric=metric, task_description=metric.description
+            )
+        except TypeError:
+            # Fallback if constructor signature differs
+            prompt = PromptCls(metric=metric)
+
+        # Add each provided example
+        for ex_idx, ex in enumerate(examples or [], start=1):
+            if "user_kwargs" not in ex or "output" not in ex:
+                raise LoaderError(
+                    f"Metric {metric.name}, example {ex_idx}: "
+                    "each example must include 'user_kwargs' and 'output'."
+                )
+            user_kwargs = ex["user_kwargs"]
+            output = ex["output"]
+            try:
+                prompt.add_example(user_kwargs, output)
+            except (ValidationError, ValueError) as e:
+                raise LoaderError(
+                    f"Metric {metric.name}, example {ex_idx} invalid: {e}"
+                ) from e
+
+        prompts.append(prompt)
+
+    return prompts

wxo_agentic_evaluation/referenceless_eval/function_calling/pipeline/adapters.py

@@ -0,0 +1,106 @@
+from typing import Any, Dict, List
+
+from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.types import (
+    ToolCall,
+    ToolSpec,
+)
+
+# ────────────────────────────────────────────────────────────────────────────────
+# Adapter definitions
+# ────────────────────────────────────────────────────────────────────────────────
+
+
+class BaseAdapter:
+    """Abstract adapter to unify different API spec and call representations."""
+
+    def get_tools_inventory(self) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def get_tools_inventory_summary(self) -> List[Dict[str, Any]]:
+        raise NotImplementedError
+
+    def get_tool_spec(self, tool_name: str) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def get_call_dict(self) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def get_function_name(self) -> str:
+        raise NotImplementedError
+
+    def get_parameters(self) -> Dict[str, Any]:
+        raise NotImplementedError
+
+    def get_param_spec_snippet(self, param_name: str) -> Dict[str, Any]:
+        raise NotImplementedError
+
+
+class OpenAIAdapter(BaseAdapter):
+    """Adapter for ToolSpec + ToolCall inputs."""
+
+    def __init__(self, specs: List[ToolSpec], call: ToolCall):
+        self.specs = specs
+        self.call = call
+
+    def get_tools_inventory(self) -> List[Dict[str, Any]]:
+        return [spec.model_dump() for spec in self.specs]
+
+    def get_tools_inventory_summary(self) -> List[Dict[str, Any]]:
+        return [
+            {
+                "tool_name": spec.function.name,
+                "tool_description": spec.function.description,
+                "tool_parameters": {
+                    prop_name: prop_d["type"]
+                    for prop_name, prop_d in spec.function.parameters.get(
+                        "properties", {}
+                    ).items()
+                },
+            }
+            for spec in self.specs
+        ]
+
+    def get_tool_spec(self, tool_name: str) -> Dict[str, Any]:
+        tool = next(
+            (t for t in self.specs if t.function.name == tool_name), None
+        )
+        return tool.function.model_dump() if tool else {}
+
+    def get_call_dict(self) -> Dict[str, Any]:
+        call_dict = {
+            "id": self.call.id,
+            "type": "function",
+            "function": {
+                "name": self.call.function.name,
+                "arguments": self.call.function.arguments,
+            },
+        }
+        return call_dict
+
+    def get_function_name(self) -> str:
+        return self.call.function.name
+
+    def get_parameters(self) -> Dict[str, Any]:
+        return self.call.function.parsed_arguments
+
+    def get_param_spec_snippet(self, param_name: str) -> Dict[str, Any]:
+        spec = next(
+            (
+                s
+                for s in self.specs
+                if s.function.name == self.get_function_name()
+            ),
+            None,
+        )
+        if not spec:
+            return {"type": "object", "properties": {}, "required": []}
+        props = spec.function.parameters.get(
+            "properties", spec.function.parameters
+        )
+        if param_name not in props:
+            return {"type": "object", "properties": {}, "required": []}
+        return {
+            "type": "object",
+            "properties": {param_name: props[param_name]},
+            "required": [param_name],
+        }

wxo_agentic_evaluation/referenceless_eval/function_calling/pipeline/pipeline.py

@@ -0,0 +1,291 @@
+import importlib.resources
+import json
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+from wxo_agentic_evaluation.referenceless_eval.function_calling import metrics
+from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.semantic_checker import (
+    SemanticChecker,
+)
+from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.static_checker import (
+    evaluate_static,
+)
+from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.types import (
+    FunctionCallInput,
+    FunctionCallMetric,
+    PipelineResult,
+    SemanticResult,
+    StaticMetricResult,
+    StaticResult,
+    ToolCall,
+    ToolSpec,
+)
+from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import (
+    LLMKitWrapper,
+)
+
+
+def metrics_dir():
+    path = importlib.resources.files(metrics)
+    return path
+
+
+# Default metric JSON paths
+_METRICS_DIR = metrics_dir()
+_DEFAULT_GENERAL = _METRICS_DIR / "function_call" / "general_metrics.json"
+_DEFAULT_GENERAL_RUNTIME = (
+    _METRICS_DIR / "function_call" / "general_metrics_runtime.json"
+)
+_DEFAULT_FUNCSEL = (
+    _METRICS_DIR / "function_selection" / "function_selection_metrics.json"
+)
+_DEFAULT_FUNCSEL_RUNTIME = (
+    _METRICS_DIR
+    / "function_selection"
+    / "function_selection_metrics_runtime.json"
+)
+_DEFAULT_PARAM = _METRICS_DIR / "parameter" / "parameter_metrics.json"
+_DEFAULT_PARAM_RUNTIME = (
+    _METRICS_DIR / "parameter" / "parameter_metrics_runtime.json"
+)
+
+
+class ReflectionPipeline:
+    """
+    High-level orchestration for function-call reflection.
+
+    Modes:
+      • static_only: schema checks
+      • semantic_only: LLM metrics + transforms
+      • run: full static -> semantic -> assemble -> PipelineResult
+
+    Supports sync, custom JSON overrides, and any registered LLM.
+    runtime_pipeline: if set to true, use faster prompts (no actionable recommendations, shorter explanations)
+    """
+
+    def __init__(
+        self,
+        metrics_client: LLMKitWrapper,
+        codegen_client: Optional[LLMKitWrapper] = None,
+        general_metrics: Optional[
+            Union[Path, List[FunctionCallMetric], List[str]]
+        ] = _DEFAULT_GENERAL_RUNTIME,
+        function_metrics: Optional[
+            Union[Path, List[FunctionCallMetric], List[str]]
+        ] = _DEFAULT_FUNCSEL_RUNTIME,
+        parameter_metrics: Optional[
+            Union[Path, List[FunctionCallMetric], List[str]]
+        ] = _DEFAULT_PARAM_RUNTIME,
+        transform_enabled: Optional[bool] = False,
+        runtime_pipeline: Optional[bool] = True,
+        use_examples: Optional[bool] = True,
+    ):
+
+        self.metrics_client = metrics_client
+        if codegen_client is None:
+            self.codegen_client = metrics_client
+        else:
+            self.codegen_client = codegen_client
+
+        self.general_metrics = general_metrics
+        self.function_metrics = function_metrics
+        self.parameter_metrics = parameter_metrics
+
+        metrics_definitions = []
+
+        for metrics, default_path in [
+            (
+                self.general_metrics,
+                (
+                    _DEFAULT_GENERAL_RUNTIME
+                    if runtime_pipeline
+                    else _DEFAULT_GENERAL
+                ),
+            ),
+            (
+                self.function_metrics,
+                (
+                    _DEFAULT_FUNCSEL_RUNTIME
+                    if runtime_pipeline
+                    else _DEFAULT_FUNCSEL
+                ),
+            ),
+            (
+                self.parameter_metrics,
+                _DEFAULT_PARAM_RUNTIME if runtime_pipeline else _DEFAULT_PARAM,
+            ),
+        ]:
+            if not metrics:
+                metrics_definitions.append(None)
+                continue
+
+            # Handle metric names list
+            if isinstance(metrics, list) and all(
+                isinstance(x, str) for x in metrics
+            ):
+                # Load the default JSON file
+                if not default_path.is_file():
+                    raise FileNotFoundError(
+                        f"Default metrics file not found: {default_path}"
+                    )
+
+                with default_path.open("r") as f_in:
+                    all_metrics = json.load(f_in)
+
+                # Filter metrics by name
+                filtered_metrics = [
+                    metric
+                    for metric in all_metrics
+                    if metric.get("name") in metrics
+                ]
+
+                # Remove examples from prompts if requested
+                if not use_examples:
+                    for metric in filtered_metrics:
+                        metric.pop("examples", None)
+
+                if len(filtered_metrics) != len(metrics):
+                    found_names = {
+                        metric.get("name") for metric in filtered_metrics
+                    }
+                    missing = set(metrics) - found_names
+                    raise ValueError(f"Metrics not found: {missing}")
+
+                metrics_definitions.append(filtered_metrics)
+                continue
+
+            # Handle Path or List[FunctionCallMetric] (existing logic)
+            if not isinstance(metrics, (Path, list)):
+                raise TypeError(
+                    "metrics must be Path, List[FunctionCallMetric], List[str], or None"
+                )
+            if isinstance(metrics, list) and all(
+                isinstance(x, FunctionCallMetric) for x in metrics
+            ):
+                metrics_definitions.append(
+                    [metric.model_dump() for metric in metrics]
+                )
+            else:
+                if not metrics.is_file():
+                    raise FileNotFoundError(
+                        f"Metrics file not found: {metrics}"
+                    )
+                metrics_definitions.append(
+                    [
+                        json.loads(json_obj)
+                        for json_obj in metrics.read_text(
+                            encoding="utf8"
+                        ).splitlines()
+                        if json_obj.strip()
+                    ]
+                )
+
+        gen_defs, fun_defs, par_defs = None, None, None
+
+        if metrics_definitions:
+            gen_defs = metrics_definitions[0]
+            if len(metrics_definitions) >= 2:
+                fun_defs = metrics_definitions[1]
+                if len(metrics_definitions) >= 3:
+                    par_defs = metrics_definitions[2]
+
+        # 3) Initialize semantic checker
+        self.semantic_checker = SemanticChecker(
+            general_metrics=gen_defs,
+            function_metrics=fun_defs,
+            parameter_metrics=par_defs,
+            metrics_client=self.metrics_client,
+            codegen_client=self.codegen_client,
+            transform_enabled=transform_enabled,
+        )
+
+    @staticmethod
+    def static_only(
+        inventory: List[ToolSpec],
+        call: ToolCall,
+    ) -> StaticResult:
+        """
+        Run schema-based static checks.
+
+        Returns:
+            StaticResult with per-check results and final_decision.
+        """
+        try:
+            return evaluate_static(inventory, call)
+        except Exception as e:
+            return StaticResult(
+                metrics={
+                    "json_schema_validation": StaticMetricResult(
+                        description="Invalid JSON schema",
+                        valid=False,
+                        explanation=f"error parsing JSON schema: {str(e)}",
+                    )
+                },
+                final_decision=False,
+            )
+
+    def semantic_sync(
+        self,
+        conversation: Union[str, List[Dict[str, str]]],
+        inventory: List[ToolSpec],
+        call: ToolCall,
+        retries: Optional[int] = 2,
+        transform_enabled: Optional[bool] = None,
+    ) -> SemanticResult:
+        """
+        Synchronous LLM-based semantic metrics (+ optional transforms).
+        """
+        # delegate to SemanticChecker
+        return self.semantic_checker.run_sync(
+            inventory,
+            call,
+            conversation,
+            retries=retries,
+            transform_enabled=transform_enabled,
+        )
+
+    def run_sync(
+        self,
+        conversation: Union[str, List[Dict[str, str]]],
+        inventory: List[ToolSpec],
+        call: ToolCall,
+        continue_on_static: Optional[bool] = False,
+        retries: Optional[int] = 1,
+        transform_enabled: Optional[bool] = None,
+    ) -> PipelineResult:
+        """
+        Full sync pipeline: static -> semantic -> assemble PipelineResult.
+        """
+        static_res = self.static_only(inventory, call)
+
+        if not static_res.final_decision and not continue_on_static:
+            inputs = FunctionCallInput(
+                conversation_context=conversation,
+                tools_inventory=inventory,
+                tool_call=call,
+            )
+            return PipelineResult(
+                inputs=inputs,
+                static=static_res,
+                semantic=SemanticResult(
+                    general=None,
+                    function_selection=None,
+                    parameter=None,
+                    transform=None,
+                ),
+                overall_valid=False,
+            )
+
+        semantic_res = self.semantic_sync(
+            conversation, inventory, call, retries, transform_enabled
+        )
+        return PipelineResult(
+            inputs=FunctionCallInput(
+                conversation_context=conversation,
+                tools_inventory=inventory,
+                tool_call=call,
+            ),
+            static=static_res,
+            semantic=semantic_res,
+            overall_valid=True,
+        )