ibm-watsonx-orchestrate-evaluation-framework 1.1.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/service_instance.py

@@ -1,10 +1,10 @@
 import logging
 import os
-
-import requests
 import shutil
 from pathlib import Path
-from typing import Optional, Any, Dict, Iterable, Tuple
+from typing import Any, Dict, Iterable, Optional, Tuple
+
+import requests
 import yaml
 
 from wxo_agentic_evaluation.utils.utils import is_ibm_cloud_url, is_saas_url
@@ -143,7 +143,7 @@ class ServiceInstance:
     def create_tenant_if_not_exist(self) -> str:
         if self.is_saas:
             logger.info(
-                "SaaS mode: running against Remote Service and skipping tenant creation"
+                "[d b]SaaS mode: running against Remote Service and skipping tenant creation"
             )
             return None
 
@@ -151,20 +151,24 @@ class ServiceInstance:
         default_tenant = self.get_default_tenant(user_auth_token)
 
         if not default_tenant:
-            logger.info("no local tenant found. A default tenant is created")
+            logger.info(
+                "[d b]no local tenant found. A default tenant is created"
+            )
             self.create_eval_tenant(user_auth_token)
             default_tenant = self.get_default_tenant(user_auth_token)
         else:
-            logger.info("local tenant found")
+            logger.info("[d b]local tenant found")
 
         return default_tenant["id"]
 
+
 def get_env_settings(
-    tenant_name: str,
-    env_config_path: Optional[str] = None
+    tenant_name: str, env_config_path: Optional[str] = None
 ) -> Dict[str, Any]:
     if env_config_path is None:
-        env_config_path = f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+        env_config_path = (
+            f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+        )
 
     try:
         with open(env_config_path, "r", encoding="utf-8") as f:
@@ -180,12 +184,11 @@ def get_env_settings(
     return dict(merged)
 
 
-
 def apply_env_overrides(
     base: Dict[str, Any],
     tenant_name: str,
     keys: Optional[Iterable[str]] = None,
-    env_config_path: Optional[str] = None
+    env_config_path: Optional[str] = None,
 ) -> Dict[str, Any]:
     """
     Returns a new dict where base is overridden by tenant-defined values.
@@ -202,8 +205,9 @@ def apply_env_overrides(
     return merged
 
 
-
-def tenant_setup(service_url: Optional[str], tenant_name: str) -> Tuple[Optional[str], Optional[str], Dict[str, Any]]:
+def tenant_setup(
+    service_url: Optional[str], tenant_name: str
+) -> Tuple[Optional[str], Optional[str], Dict[str, Any]]:
     # service_instance = ServiceInstance(
     #     service_url=service_url,
     #     tenant_name=tenant_name
@@ -245,17 +249,22 @@ def tenant_setup(service_url: Optional[str], tenant_name: str) -> Tuple[Optional
 
     context["active_environment"] = tenant_name
 
+    # Ensure parent directories exist so tests (which may run in clean envs)
+    # can write these files without raising FileNotFoundError.
+    auth_dir = os.path.dirname(auth_config_path)
+    env_dir = os.path.dirname(env_config_path)
+    os.makedirs(auth_dir, exist_ok=True)
+    os.makedirs(env_dir, exist_ok=True)
+
     with open(auth_config_path, "w") as f:
         yaml.dump(auth_config, f)
     with open(env_config_path, "w") as f:
         yaml.dump(env_config, f)
 
     token = (
-        auth_config.get("auth", {})
-        .get(tenant_name, {})
-        .get("wxo_mcsp_token")
+        auth_config.get("auth", {}).get(tenant_name, {}).get("wxo_mcsp_token")
     )
 
     env_merged = get_env_settings(tenant_name, env_config_path=env_config_path)
 
-    return token, resolved_service_url, env_merged
+    return token, resolved_service_url, env_merged

wxo_agentic_evaluation/service_provider/__init__.py

@@ -1,6 +1,13 @@
+import logging
 import os
 
+from rich.console import Console
+from rich.logging import RichHandler
+
 from wxo_agentic_evaluation.arg_configs import ProviderConfig
+from wxo_agentic_evaluation.service_provider.gateway_provider import (
+    GatewayProvider,
+)
 from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
     ModelProxyProvider,
 )
@@ -8,6 +15,7 @@ from wxo_agentic_evaluation.service_provider.ollama_provider import (
     OllamaProvider,
 )
 from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import (
+    GatewayProviderLLMKitWrapper,
     ModelProxyProviderLLMKitWrapper,
     WatsonXLLMKitWrapper,
 )
@@ -15,24 +23,131 @@ from wxo_agentic_evaluation.service_provider.watsonx_provider import (
     WatsonXProvider,
 )
 
+try:
+    from wxo_agentic_evaluation.service_provider.portkey_provider import (
+        PortkeyProvider,
+    )
+except:
+    pass
+
+
+USE_GATEWAY_MODEL_PROVIDER: bool = (
+    os.environ.get("USE_GATEWAY_MODEL_PROVIDER", "FALSE").upper() == "TRUE"
+)
+
+
+_logging_console = Console(stderr=True)
+
+logger = logging.getLogger(__name__)
+
+
+def get_log_level_from_env():
+
+    level_env = os.getenv("WXO_EVALUATION_LOGLEVEL")
+    return level_env
+
+
+LOGGING_ENABLED = get_log_level_from_env() is not None
+
+
+def configure_logging_for_package_from_env(
+    package_name: str = "wxo_agentic_evaluation",
+    ensure_output: bool = True,
+) -> None:
+    """
+    Configure logging using the env var WXO_EVALUATION_LOGLEVEL - no logging if that's not set
+    """
+    try:
+        level_env = get_log_level_from_env()
+        if not level_env:
+            return
+
+        level = None
+        upper = level_env.strip().upper()
+        if hasattr(logging, upper):
+            level = getattr(logging, upper, None)
+
+        pkg_logger = logging.getLogger(package_name)
+        pkg_logger.setLevel(level)
+
+        if ensure_output:
+            if not pkg_logger.handlers:
+                handler = RichHandler(
+                    console=_logging_console,
+                    rich_tracebacks=True,
+                    show_time=False,
+                    show_level=False,
+                    show_path=False,
+                    markup=True,
+                    enable_link_path=True,
+                    omit_repeated_times=True,
+                    tracebacks_theme="github-dark",
+                )
+                handler.setFormatter(
+                    logging.Formatter("%(levelname)s %(message)s")
+                )
+                handler.setLevel(logging.NOTSET)
+                pkg_logger.addHandler(handler)
+            pkg_logger.propagate = False
+
+        # Quiet common noisy debug libs
+        for name in (
+            "urllib3",
+            "urllib3.connectionpool",
+            "requests.packages.urllib3",
+        ):
+            logging.getLogger(name).setLevel(logging.WARNING)
+    except:
+        logger.warning("Input log level %s not valid", level_env)
+
+
+configure_logging_for_package_from_env()
+
 
 def _instantiate_provider(
     config: ProviderConfig, is_referenceless_eval: bool = False, **kwargs
 ):
+
     if config.provider == "watsonx":
+        logger.info("Instantiate watsonx provider")
         if is_referenceless_eval:
             provider = WatsonXLLMKitWrapper
         else:
             provider = WatsonXProvider
-        return provider(model_id=config.model_id, **kwargs)
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
     elif config.provider == "ollama":
+        logger.info("Instantiate Ollama")
         return OllamaProvider(model_id=config.model_id, **kwargs)
+
+    elif config.provider == "gateway":
+        logger.info("Instantiate gateway inference provider")
+        if is_referenceless_eval:
+            provider = GatewayProviderLLMKitWrapper
+        else:
+            provider = GatewayProvider
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     elif config.provider == "model_proxy":
+        logger.info("Instantiate model proxy provider")
         if is_referenceless_eval:
             provider = ModelProxyProviderLLMKitWrapper
         else:
             provider = ModelProxyProvider
-        return provider(model_id=config.model_id, **kwargs)
+
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     else:
         raise RuntimeError(
             f"target provider is not supported {config.provider}"
@@ -42,23 +157,44 @@ def _instantiate_provider(
 def get_provider(
     config: ProviderConfig = None,
     model_id: str = None,
+    embedding_model_id: str = None,
     referenceless_eval: bool = False,
+    provider: str = None,
+    api_key: str = None,
+    use_portkey_provider: bool = False,
     **kwargs,
 ):
+
+    if use_portkey_provider:
+        return PortkeyProvider(
+            provider=provider, model_id=model_id, api_key=api_key, **kwargs
+        )
+
+    if config:
+        return _instantiate_provider(config, **kwargs)
+
     if not model_id:
         raise ValueError("model_id must be provided if config is not supplied")
 
+    if USE_GATEWAY_MODEL_PROVIDER:
+        logger.info("[d b]Using gateway inference provider override")
+        config = ProviderConfig(provider="gateway", model_id=model_id)
+        return _instantiate_provider(config, referenceless_eval, **kwargs)
+
     if "WATSONX_APIKEY" in os.environ and "WATSONX_SPACE_ID" in os.environ:
-        config = ProviderConfig(provider="watsonx", model_id=model_id)
+        logger.info("[d b]Using watsonx inference provider")
+        config = ProviderConfig(
+            provider="watsonx",
+            model_id=model_id,
+            embedding_model_id=embedding_model_id,
+        )
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
     if "WO_INSTANCE" in os.environ:
+        logger.info("[d b]Using model_proxy inference provider")
         config = ProviderConfig(provider="model_proxy", model_id=model_id)
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
-    if config:
-        return _instantiate_provider(config, **kwargs)
-
-    raise RuntimeError(
-        "No provider found. Please either provide a config or set the required environment variables."
-    )
+    logger.info("[d b]Using gateway inference provider default")
+    config = ProviderConfig(provider="gateway", model_id=model_id)
+    return _instantiate_provider(config, referenceless_eval, **kwargs)