ibm-watsonx-orchestrate-evaluation-framework 1.1.5__py3-none-any.whl → 1.1.7__py3-none-any.whl

wxo_agentic_evaluation/referenceless_eval/function_calling/pipeline/types.py

@@ -390,18 +390,18 @@ class PipelineResult(BaseModel):
     )
 
     @model_validator(mode="after")
-    def compute_overall(cls, values: PipelineResult) -> PipelineResult:
+    def compute_overall(self) -> Self:
         """
         After validation, compute overall_valid as AND of:
           • all semantic is_correct flags
           • if transform exists: all execution_success flags
         """
-        static: StaticResult = values.static
+        static: StaticResult = self.static
         if static:
             # static checks
             ok = static.final_decision
 
-        sem: SemanticResult = values.semantic
+        sem: SemanticResult = self.semantic
         if sem:
             # semantic checks
             if sem.general and sem.general.metrics:
@@ -441,11 +441,11 @@ class PipelineResult(BaseModel):
             if param_avgs:
                 cat_avgs.append(sum(param_avgs) / len(param_avgs))
 
-        values.overall_avg_score = (
+        self.overall_avg_score = (
             sum(cat_avgs) / len(cat_avgs) if cat_avgs else None
         )
-        values.overall_valid = ok
-        return values
+        self.overall_valid = ok
+        return self
 
 
 # ----------------------------------------------------------------------
@@ -531,17 +531,17 @@ class ToolFunctionCall(BaseModel):
     )
 
     @model_validator(mode="after")
-    def _parse_arguments(cls, values: ToolFunctionCall) -> ToolFunctionCall:
+    def _parse_arguments(self) -> Self:
         """
         After model construction, parse the `arguments` JSON string
         into `parsed_arguments`, or raise a ValidationError.
         """
         try:
-            raw = values.arguments
-            values.parsed_arguments = json.loads(raw)
+            raw = self.arguments
+            self.parsed_arguments = json.loads(raw)
         except json.JSONDecodeError as e:
             raise ValidationError(f"Invalid JSON in arguments: {e}") from e
-        return values
+        return self
 
 
 class ToolCall(BaseModel):

wxo_agentic_evaluation/referenceless_eval/referenceless_eval.py

@@ -1,8 +1,9 @@
 import json
-from typing import Any, List, Mapping
+from typing import Any, List, Mapping, Optional
 
 import rich
 
+from wxo_agentic_evaluation.inference_backend import WXOInferenceBackend
 from wxo_agentic_evaluation.referenceless_eval.function_calling.consts import (
     METRIC_FUNCTION_SELECTION_APPROPRIATENESS,
     METRIC_GENERAL_HALLUCINATION_CHECK,
@@ -16,6 +17,15 @@ from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.types i
 )
 from wxo_agentic_evaluation.service_provider import get_provider
 from wxo_agentic_evaluation.type import Message
+from wxo_agentic_evaluation.utils.gateway_provider_utils import (
+    get_provider_kwargs,
+)
+
+DEFAULT_GENERATION_PARAMS = {
+    "min_new_tokens": 0,
+    "decoding_method": "greedy",
+    "max_new_tokens": 4096,
+}
 
 
 class ReferencelessEvaluation:
@@ -31,20 +41,29 @@ class ReferencelessEvaluation:
     def __init__(
         self,
         api_spec: List[Mapping[str, Any]],
-        messages: List[Message],
         model_id: str,
         task_n: str,
         dataset_name: str,
+        runtime_pipeline: bool = True,
+        generation_params=DEFAULT_GENERATION_PARAMS,
+        inference_backend: Optional[WXOInferenceBackend] = None,
     ):
 
-        self.metrics_client = get_provider(
+        extra_kwargs = {}
+        if inference_backend is not None:
+            wxo_client = getattr(inference_backend, "wxo_client")
+            instance_url = getattr(wxo_client, "service_url", None)
+            token = getattr(wxo_client, "api_key", None)
+            if instance_url:
+                extra_kwargs["instance_url"] = instance_url
+            if token:
+                extra_kwargs["token"] = token
+
+        self.metrics_client = ReferencelessEvaluation.get_metrics_client(
             model_id=model_id,
-            params={
-                "min_new_tokens": 0,
-                "decoding_method": "greedy",
-                "max_new_tokens": 4096,
-            },
+            params=generation_params,
             referenceless_eval=True,
+            **extra_kwargs,
         )
 
         self.pipeline = ReflectionPipeline(
@@ -52,39 +71,54 @@ class ReferencelessEvaluation:
             general_metrics=[METRIC_GENERAL_HALLUCINATION_CHECK],
             function_metrics=[METRIC_FUNCTION_SELECTION_APPROPRIATENESS],
             parameter_metrics=None,
+            runtime_pipeline=runtime_pipeline,
         )
 
         self.task_n = task_n
         self.dataset_name = dataset_name
 
         self.apis_specs = [ToolSpec.model_validate(spec) for spec in api_spec]
-        self.messages = messages
 
-    def _run_pipeline(self, examples: List[Mapping[str, Any]]):
-        results = []
-        for example in examples:
-            result = self.pipeline.run_sync(
-                conversation=example["context"],
-                inventory=self.apis_specs,
-                call=example["call"],
-                continue_on_static=False,
-                retries=2,
-            )
-            result_dict = result.model_dump()
-            results.append(result_dict)
+    @staticmethod
+    def get_metrics_client(**kwargs):
 
-        return results
+        provider_kwargs = get_provider_kwargs(**kwargs)
 
-    def run(self):
-        examples = []
+        return get_provider(
+            **provider_kwargs,
+        )
+
+    @staticmethod
+    def fmt_tool_call(tool_id, tool_call_name, arguments, context):
+        call = {
+            "call": {
+                "id": tool_id,
+                "type": "function",
+                "function": {
+                    "name": tool_call_name,
+                    "arguments": arguments,
+                },
+            },
+            "context": context,
+        }
 
+        return call
+
+    @staticmethod
+    def fmt_msgs_referenceless(
+        messages: List[Message],
+    ) -> List[Mapping[str, Any]]:
+        """Assume that the last item in the `messages` array is the tool call, and preceding items
+        in the messages array is the context.
+        """
+        examples = []
         processed_data = [
             {
                 k: msg.model_dump().get(k)
                 for k in ["role", "content", "type"]
                 if k in msg.model_dump()
             }
-            for msg in self.messages
+            for msg in messages
         ]
 
         for idx, message in enumerate(processed_data):
@@ -97,22 +131,47 @@ class ReferencelessEvaluation:
                 if tool_call_msg["name"].startswith("transfer_to"):
                     continue
 
-                call = {
-                    "call": {
-                        "id": tool_call_msg.get("id", "1"),
-                        "type": "function",
-                        "function": {
-                            "name": tool_call_msg["name"],
-                            "arguments": json.dumps(tool_call_msg["args"]),
-                        },
-                    },
-                    "context": context,
-                }
+                call = ReferencelessEvaluation.fmt_tool_call(
+                    tool_id=tool_call_msg.get("id", "1"),
+                    tool_call_name=tool_call_msg["name"],
+                    arguments=json.dumps(tool_call_msg["args"]),
+                    context=context,
+                )
                 examples.append(call)
 
-        rich.print(
-            f"[yellow][b][Task-{self.task_n}] There are {len(examples)} examples to analyze"
-        )
+        return examples
+
+    def _run_pipeline(self, examples: List[Mapping[str, Any]]):
+        results = []
+        for example in examples:
+            result = self.pipeline.run_sync(
+                conversation=example["context"],
+                inventory=self.apis_specs,
+                call=example["call"],
+                continue_on_static=False,
+                retries=2,
+            )
+            result_dict = result.model_dump()
+            results.append(result_dict)
+
+        return results
+
+    def run(self, examples: List[Mapping[str, str]], verbose=False):
+        """`examples` should be an array where each element is formatted:
+
+        call = {
+            "call": {
+                "id": tool_call_msg.get("id", "1"),
+                "type": "function",
+                "function": {
+                    "name": tool_call_msg["name"],
+                    "arguments": json.dumps(tool_call_msg["args"]),
+                },
+            },
+            "context": context,
+        }
+        """
+
         examples = [
             {
                 "call": ToolCall.model_validate(ex["call"]),
@@ -120,6 +179,11 @@ class ReferencelessEvaluation:
             }
             for ex in examples
         ]
+
+        if verbose:
+            rich.print(
+                f"[yellow][b][Task-{self.task_n}] There are {len(examples)} examples to analyze"
+            )
         results = self._run_pipeline(examples)
 
         return results

wxo_agentic_evaluation/resource_map.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 
-from wxo_agentic_evaluation.inference_backend import WXOClient, is_saas_url
+from wxo_agentic_evaluation.utils.utils import is_saas_url
+from wxo_agentic_evaluation.wxo_client import WXOClient
 
 
 class ResourceMap:
@@ -34,6 +35,7 @@ class ResourceMap:
 
         if resp.status_code == 200:
             agents = resp.json()
+            self.all_agent_objs = agents
             for agent in agents:
                 agent_name = agent["name"]
                 tools = [tool_map[id] for id in agent["tools"]]

wxo_agentic_evaluation/service_instance.py

@@ -143,7 +143,7 @@ class ServiceInstance:
     def create_tenant_if_not_exist(self) -> str:
         if self.is_saas:
             logger.info(
-                "SaaS mode: running against Remote Service and skipping tenant creation"
+                "[d b]SaaS mode: running against Remote Service and skipping tenant creation"
             )
             return None
 
@@ -151,11 +151,13 @@ class ServiceInstance:
         default_tenant = self.get_default_tenant(user_auth_token)
 
         if not default_tenant:
-            logger.info("no local tenant found. A default tenant is created")
+            logger.info(
+                "[d b]no local tenant found. A default tenant is created"
+            )
             self.create_eval_tenant(user_auth_token)
             default_tenant = self.get_default_tenant(user_auth_token)
         else:
-            logger.info("local tenant found")
+            logger.info("[d b]local tenant found")
 
         return default_tenant["id"]
 
@@ -247,6 +249,13 @@ def tenant_setup(
 
     context["active_environment"] = tenant_name
 
+    # Ensure parent directories exist so tests (which may run in clean envs)
+    # can write these files without raising FileNotFoundError.
+    auth_dir = os.path.dirname(auth_config_path)
+    env_dir = os.path.dirname(env_config_path)
+    os.makedirs(auth_dir, exist_ok=True)
+    os.makedirs(env_dir, exist_ok=True)
+
     with open(auth_config_path, "w") as f:
         yaml.dump(auth_config, f)
     with open(env_config_path, "w") as f:

wxo_agentic_evaluation/service_provider/__init__.py

@@ -1,6 +1,13 @@
+import logging
 import os
 
+from rich.console import Console
+from rich.logging import RichHandler
+
 from wxo_agentic_evaluation.arg_configs import ProviderConfig
+from wxo_agentic_evaluation.service_provider.gateway_provider import (
+    GatewayProvider,
+)
 from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
     ModelProxyProvider,
 )
@@ -8,6 +15,7 @@ from wxo_agentic_evaluation.service_provider.ollama_provider import (
     OllamaProvider,
 )
 from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import (
+    GatewayProviderLLMKitWrapper,
     ModelProxyProviderLLMKitWrapper,
     WatsonXLLMKitWrapper,
 )
@@ -15,24 +23,123 @@ from wxo_agentic_evaluation.service_provider.watsonx_provider import (
     WatsonXProvider,
 )
 
+USE_GATEWAY_MODEL_PROVIDER: bool = (
+    os.environ.get("USE_GATEWAY_MODEL_PROVIDER", "FALSE").upper() == "TRUE"
+)
+
+
+_logging_console = Console(stderr=True)
+
+logger = logging.getLogger(__name__)
+
+
+def get_log_level_from_env():
+
+    level_env = os.getenv("WXO_EVALUATION_LOGLEVEL")
+    return level_env
+
+
+LOGGING_ENABLED = get_log_level_from_env() is not None
+
+
+def configure_logging_for_package_from_env(
+    package_name: str = "wxo_agentic_evaluation",
+    ensure_output: bool = True,
+) -> None:
+    """
+    Configure logging using the env var WXO_EVALUATION_LOGLEVEL - no logging if that's not set
+    """
+    try:
+        level_env = get_log_level_from_env()
+        if not level_env:
+            return
+
+        level = None
+        upper = level_env.strip().upper()
+        if hasattr(logging, upper):
+            level = getattr(logging, upper, None)
+
+        pkg_logger = logging.getLogger(package_name)
+        pkg_logger.setLevel(level)
+
+        if ensure_output:
+            if not pkg_logger.handlers:
+                handler = RichHandler(
+                    console=_logging_console,
+                    rich_tracebacks=True,
+                    show_time=False,
+                    show_level=False,
+                    show_path=False,
+                    markup=True,
+                    enable_link_path=True,
+                    omit_repeated_times=True,
+                    tracebacks_theme="github-dark",
+                )
+                handler.setFormatter(
+                    logging.Formatter("%(levelname)s %(message)s")
+                )
+                handler.setLevel(logging.NOTSET)
+                pkg_logger.addHandler(handler)
+            pkg_logger.propagate = False
+
+        # Quiet common noisy debug libs
+        for name in (
+            "urllib3",
+            "urllib3.connectionpool",
+            "requests.packages.urllib3",
+        ):
+            logging.getLogger(name).setLevel(logging.WARNING)
+    except:
+        logger.warning("Input log level %s not valid", level_env)
+
+
+configure_logging_for_package_from_env()
+
 
 def _instantiate_provider(
     config: ProviderConfig, is_referenceless_eval: bool = False, **kwargs
 ):
+
     if config.provider == "watsonx":
+        logger.info("Instantiate watsonx provider")
         if is_referenceless_eval:
             provider = WatsonXLLMKitWrapper
         else:
             provider = WatsonXProvider
-        return provider(model_id=config.model_id, **kwargs)
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
     elif config.provider == "ollama":
+        logger.info("Instantiate Ollama")
         return OllamaProvider(model_id=config.model_id, **kwargs)
+
+    elif config.provider == "gateway":
+        logger.info("Instantiate gateway inference provider")
+        if is_referenceless_eval:
+            provider = GatewayProviderLLMKitWrapper
+        else:
+            provider = GatewayProvider
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     elif config.provider == "model_proxy":
+        logger.info("Instantiate model proxy provider")
         if is_referenceless_eval:
             provider = ModelProxyProviderLLMKitWrapper
         else:
             provider = ModelProxyProvider
-        return provider(model_id=config.model_id, **kwargs)
+
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     else:
         raise RuntimeError(
             f"target provider is not supported {config.provider}"
@@ -42,23 +149,36 @@ def _instantiate_provider(
 def get_provider(
     config: ProviderConfig = None,
     model_id: str = None,
+    embedding_model_id: str = None,
     referenceless_eval: bool = False,
     **kwargs,
 ):
+
+    if config:
+        return _instantiate_provider(config, **kwargs)
+
     if not model_id:
         raise ValueError("model_id must be provided if config is not supplied")
 
+    if USE_GATEWAY_MODEL_PROVIDER:
+        logger.info("[d b]Using gateway inference provider override")
+        config = ProviderConfig(provider="gateway", model_id=model_id)
+        return _instantiate_provider(config, referenceless_eval, **kwargs)
+
     if "WATSONX_APIKEY" in os.environ and "WATSONX_SPACE_ID" in os.environ:
-        config = ProviderConfig(provider="watsonx", model_id=model_id)
+        logger.info("[d b]Using watsonx inference provider")
+        config = ProviderConfig(
+            provider="watsonx",
+            model_id=model_id,
+            embedding_model_id=embedding_model_id,
+        )
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
     if "WO_INSTANCE" in os.environ:
+        logger.info("[d b]Using model_proxy inference provider")
         config = ProviderConfig(provider="model_proxy", model_id=model_id)
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
-    if config:
-        return _instantiate_provider(config, **kwargs)
-
-    raise RuntimeError(
-        "No provider found. Please either provide a config or set the required environment variables."
-    )
+    logger.info("[d b]Using gateway inference provider default")
+    config = ProviderConfig(provider="gateway", model_id=model_id)
+    return _instantiate_provider(config, referenceless_eval, **kwargs)