ibm-watsonx-orchestrate-evaluation-framework 1.1.6__py3-none-any.whl → 1.1.7__py3-none-any.whl

wxo_agentic_evaluation/referenceless_eval/referenceless_eval.py

@@ -1,8 +1,9 @@
 import json
-from typing import Any, List, Mapping
+from typing import Any, List, Mapping, Optional
 
 import rich
 
+from wxo_agentic_evaluation.inference_backend import WXOInferenceBackend
 from wxo_agentic_evaluation.referenceless_eval.function_calling.consts import (
     METRIC_FUNCTION_SELECTION_APPROPRIATENESS,
     METRIC_GENERAL_HALLUCINATION_CHECK,
@@ -16,13 +17,17 @@ from wxo_agentic_evaluation.referenceless_eval.function_calling.pipeline.types i
 )
 from wxo_agentic_evaluation.service_provider import get_provider
 from wxo_agentic_evaluation.type import Message
+from wxo_agentic_evaluation.utils.gateway_provider_utils import (
+    get_provider_kwargs,
+)
 
-DEFAULT_GENERATION_PARAMS= {
+DEFAULT_GENERATION_PARAMS = {
     "min_new_tokens": 0,
     "decoding_method": "greedy",
     "max_new_tokens": 4096,
 }
 
+
 class ReferencelessEvaluation:
     """
     Note: static.final_decison, if `True` -> then all static metrics were valid. If false, atleast one of the static metrics failed. Look at explanation for reasoning
@@ -40,13 +45,25 @@ class ReferencelessEvaluation:
         task_n: str,
         dataset_name: str,
         runtime_pipeline: bool = True,
-        generation_params = DEFAULT_GENERATION_PARAMS
+        generation_params=DEFAULT_GENERATION_PARAMS,
+        inference_backend: Optional[WXOInferenceBackend] = None,
     ):
 
-        self.metrics_client = get_provider(
+        extra_kwargs = {}
+        if inference_backend is not None:
+            wxo_client = getattr(inference_backend, "wxo_client")
+            instance_url = getattr(wxo_client, "service_url", None)
+            token = getattr(wxo_client, "api_key", None)
+            if instance_url:
+                extra_kwargs["instance_url"] = instance_url
+            if token:
+                extra_kwargs["token"] = token
+
+        self.metrics_client = ReferencelessEvaluation.get_metrics_client(
             model_id=model_id,
             params=generation_params,
             referenceless_eval=True,
+            **extra_kwargs,
         )
 
         self.pipeline = ReflectionPipeline(
@@ -62,20 +79,29 @@ class ReferencelessEvaluation:
 
         self.apis_specs = [ToolSpec.model_validate(spec) for spec in api_spec]
 
+    @staticmethod
+    def get_metrics_client(**kwargs):
+
+        provider_kwargs = get_provider_kwargs(**kwargs)
+
+        return get_provider(
+            **provider_kwargs,
+        )
+
     @staticmethod
     def fmt_tool_call(tool_id, tool_call_name, arguments, context):
         call = {
-                "call": {
-                    "id": tool_id,
-                    "type": "function",
-                    "function": {
-                        "name": tool_call_name,
-                        "arguments": arguments,
-                    },
+            "call": {
+                "id": tool_id,
+                "type": "function",
+                "function": {
+                    "name": tool_call_name,
+                    "arguments": arguments,
                 },
-                "context": context,
-            }
-        
+            },
+            "context": context,
+        }
+
         return call
 
     @staticmethod
@@ -104,12 +130,12 @@ class ReferencelessEvaluation:
                 tool_call_msg = json.loads(content)
                 if tool_call_msg["name"].startswith("transfer_to"):
                     continue
-                
+
                 call = ReferencelessEvaluation.fmt_tool_call(
                     tool_id=tool_call_msg.get("id", "1"),
                     tool_call_name=tool_call_msg["name"],
                     arguments=json.dumps(tool_call_msg["args"]),
-                    context=context
+                    context=context,
                 )
                 examples.append(call)
 

wxo_agentic_evaluation/service_instance.py

@@ -143,7 +143,7 @@ class ServiceInstance:
     def create_tenant_if_not_exist(self) -> str:
         if self.is_saas:
             logger.info(
-                "SaaS mode: running against Remote Service and skipping tenant creation"
+                "[d b]SaaS mode: running against Remote Service and skipping tenant creation"
             )
             return None
 
@@ -151,11 +151,13 @@ class ServiceInstance:
         default_tenant = self.get_default_tenant(user_auth_token)
 
         if not default_tenant:
-            logger.info("no local tenant found. A default tenant is created")
+            logger.info(
+                "[d b]no local tenant found. A default tenant is created"
+            )
             self.create_eval_tenant(user_auth_token)
             default_tenant = self.get_default_tenant(user_auth_token)
         else:
-            logger.info("local tenant found")
+            logger.info("[d b]local tenant found")
 
         return default_tenant["id"]
 

wxo_agentic_evaluation/service_provider/__init__.py

@@ -1,6 +1,13 @@
+import logging
 import os
 
+from rich.console import Console
+from rich.logging import RichHandler
+
 from wxo_agentic_evaluation.arg_configs import ProviderConfig
+from wxo_agentic_evaluation.service_provider.gateway_provider import (
+    GatewayProvider,
+)
 from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
     ModelProxyProvider,
 )
@@ -8,6 +15,7 @@ from wxo_agentic_evaluation.service_provider.ollama_provider import (
     OllamaProvider,
 )
 from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import (
+    GatewayProviderLLMKitWrapper,
     ModelProxyProviderLLMKitWrapper,
     WatsonXLLMKitWrapper,
 )
@@ -15,24 +23,123 @@ from wxo_agentic_evaluation.service_provider.watsonx_provider import (
     WatsonXProvider,
 )
 
+USE_GATEWAY_MODEL_PROVIDER: bool = (
+    os.environ.get("USE_GATEWAY_MODEL_PROVIDER", "FALSE").upper() == "TRUE"
+)
+
+
+_logging_console = Console(stderr=True)
+
+logger = logging.getLogger(__name__)
+
+
+def get_log_level_from_env():
+
+    level_env = os.getenv("WXO_EVALUATION_LOGLEVEL")
+    return level_env
+
+
+LOGGING_ENABLED = get_log_level_from_env() is not None
+
+
+def configure_logging_for_package_from_env(
+    package_name: str = "wxo_agentic_evaluation",
+    ensure_output: bool = True,
+) -> None:
+    """
+    Configure logging using the env var WXO_EVALUATION_LOGLEVEL - no logging if that's not set
+    """
+    try:
+        level_env = get_log_level_from_env()
+        if not level_env:
+            return
+
+        level = None
+        upper = level_env.strip().upper()
+        if hasattr(logging, upper):
+            level = getattr(logging, upper, None)
+
+        pkg_logger = logging.getLogger(package_name)
+        pkg_logger.setLevel(level)
+
+        if ensure_output:
+            if not pkg_logger.handlers:
+                handler = RichHandler(
+                    console=_logging_console,
+                    rich_tracebacks=True,
+                    show_time=False,
+                    show_level=False,
+                    show_path=False,
+                    markup=True,
+                    enable_link_path=True,
+                    omit_repeated_times=True,
+                    tracebacks_theme="github-dark",
+                )
+                handler.setFormatter(
+                    logging.Formatter("%(levelname)s %(message)s")
+                )
+                handler.setLevel(logging.NOTSET)
+                pkg_logger.addHandler(handler)
+            pkg_logger.propagate = False
+
+        # Quiet common noisy debug libs
+        for name in (
+            "urllib3",
+            "urllib3.connectionpool",
+            "requests.packages.urllib3",
+        ):
+            logging.getLogger(name).setLevel(logging.WARNING)
+    except:
+        logger.warning("Input log level %s not valid", level_env)
+
+
+configure_logging_for_package_from_env()
+
 
 def _instantiate_provider(
     config: ProviderConfig, is_referenceless_eval: bool = False, **kwargs
 ):
+
     if config.provider == "watsonx":
+        logger.info("Instantiate watsonx provider")
         if is_referenceless_eval:
             provider = WatsonXLLMKitWrapper
         else:
             provider = WatsonXProvider
-        return provider(model_id=config.model_id, **kwargs)
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
     elif config.provider == "ollama":
+        logger.info("Instantiate Ollama")
         return OllamaProvider(model_id=config.model_id, **kwargs)
+
+    elif config.provider == "gateway":
+        logger.info("Instantiate gateway inference provider")
+        if is_referenceless_eval:
+            provider = GatewayProviderLLMKitWrapper
+        else:
+            provider = GatewayProvider
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     elif config.provider == "model_proxy":
+        logger.info("Instantiate model proxy provider")
         if is_referenceless_eval:
             provider = ModelProxyProviderLLMKitWrapper
         else:
             provider = ModelProxyProvider
-        return provider(model_id=config.model_id, **kwargs)
+
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     else:
         raise RuntimeError(
             f"target provider is not supported {config.provider}"
@@ -42,23 +149,36 @@ def _instantiate_provider(
 def get_provider(
     config: ProviderConfig = None,
     model_id: str = None,
+    embedding_model_id: str = None,
     referenceless_eval: bool = False,
     **kwargs,
 ):
+
+    if config:
+        return _instantiate_provider(config, **kwargs)
+
     if not model_id:
         raise ValueError("model_id must be provided if config is not supplied")
 
+    if USE_GATEWAY_MODEL_PROVIDER:
+        logger.info("[d b]Using gateway inference provider override")
+        config = ProviderConfig(provider="gateway", model_id=model_id)
+        return _instantiate_provider(config, referenceless_eval, **kwargs)
+
     if "WATSONX_APIKEY" in os.environ and "WATSONX_SPACE_ID" in os.environ:
-        config = ProviderConfig(provider="watsonx", model_id=model_id)
+        logger.info("[d b]Using watsonx inference provider")
+        config = ProviderConfig(
+            provider="watsonx",
+            model_id=model_id,
+            embedding_model_id=embedding_model_id,
+        )
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
     if "WO_INSTANCE" in os.environ:
+        logger.info("[d b]Using model_proxy inference provider")
         config = ProviderConfig(provider="model_proxy", model_id=model_id)
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
-    if config:
-        return _instantiate_provider(config, **kwargs)
-
-    raise RuntimeError(
-        "No provider found. Please either provide a config or set the required environment variables."
-    )
+    logger.info("[d b]Using gateway inference provider default")
+    config = ProviderConfig(provider="gateway", model_id=model_id)
+    return _instantiate_provider(config, referenceless_eval, **kwargs)