ibm-watsonx-orchestrate-evaluation-framework 1.0.3__py3-none-any.whl → 1.1.8b0__py3-none-any.whl

wxo_agentic_evaluation/service_instance.py

@@ -1,8 +1,13 @@
 import logging
-import yaml
 import os
+import shutil
+from pathlib import Path
+from typing import Any, Dict, Iterable, Optional, Tuple
+
 import requests
-from wxo_agentic_evaluation.utils.utils import is_saas_url, is_ibm_cloud_url
+import yaml
+
+from wxo_agentic_evaluation.utils.utils import is_ibm_cloud_url, is_saas_url
 
 logger = logging.getLogger(__name__)
 
@@ -11,13 +16,15 @@ USER = {"username": "wxo.archer@ibm.com", "password": "watsonx"}
 
 class ServiceInstance:
     def __init__(
-        self, service_url, tenant_name, is_saas: bool = None, is_ibm_cloud: bool = None
+        self,
+        service_url,
+        tenant_name,
+        is_saas: bool = None,
+        is_ibm_cloud: bool = None,
     ) -> None:
         self.service_url = service_url
         self.tenant_name = tenant_name
-        STAGING_AUTH_ENDPOINT = (
-            "https://iam.platform.test.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
-        )
+        STAGING_AUTH_ENDPOINT = "https://iam.platform.test.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
         PROD_AUTH_ENDPOINT = (
             "https://iam.platform.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
         )
@@ -25,7 +32,9 @@ class ServiceInstance:
 
         self.is_saas = is_saas_url(service_url) if is_saas is None else is_saas
         self.is_ibm_cloud = (
-            is_ibm_cloud_url(service_url) if is_ibm_cloud is None else is_ibm_cloud
+            is_ibm_cloud_url(service_url)
+            if is_ibm_cloud is None
+            else is_ibm_cloud
         )
 
         if self.is_saas:
@@ -49,10 +58,10 @@ class ServiceInstance:
     def get_user_token(self):
         try:
             if self.is_saas:
-                apikey = os.environ.get("WATSONX_IAM_SAAS_APIKEY")
+                apikey = os.environ.get("WO_API_KEY")
                 if not apikey:
                     raise RuntimeError(
-                        "WATSONX_IAM_SAAS_APIKEY not set in environment for SaaS mode"
+                        "WO_API_KEY not set in environment for SaaS mode"
                     )
                 if self.is_ibm_cloud:
                     data = {
@@ -88,7 +97,8 @@ class ServiceInstance:
 
     def _get_tenant_token(self, tenant_id: str):
         resp = requests.post(
-            self.tenant_auth_endpoint.format(self.service_url, tenant_id), data=USER
+            self.tenant_auth_endpoint.format(self.service_url, tenant_id),
+            data=USER,
         )
         if resp.status_code == 200:
             return resp.json()["access_token"]
@@ -122,7 +132,9 @@ class ServiceInstance:
             "tags": ["test"],
         }
 
-        resp = requests.post(self.tenant_url, headers=headers, json=tenant_config)
+        resp = requests.post(
+            self.tenant_url, headers=headers, json=tenant_config
+        )
         if resp.status_code == 201:
             return True
         else:
@@ -131,7 +143,7 @@ class ServiceInstance:
     def create_tenant_if_not_exist(self) -> str:
         if self.is_saas:
             logger.info(
-                "SaaS mode: running against Remote Service and skipping tenant creation"
+                "[d b]SaaS mode: running against Remote Service and skipping tenant creation"
             )
             return None
 
@@ -139,16 +151,63 @@ class ServiceInstance:
         default_tenant = self.get_default_tenant(user_auth_token)
 
         if not default_tenant:
-            logger.info("no local tenant found. A default tenant is created")
+            logger.info(
+                "[d b]no local tenant found. A default tenant is created"
+            )
             self.create_eval_tenant(user_auth_token)
             default_tenant = self.get_default_tenant(user_auth_token)
         else:
-            logger.info("local tenant found")
+            logger.info("[d b]local tenant found")
 
         return default_tenant["id"]
 
 
-def tenant_setup(service_url: str, tenant_name: str):
+def get_env_settings(
+    tenant_name: str, env_config_path: Optional[str] = None
+) -> Dict[str, Any]:
+    if env_config_path is None:
+        env_config_path = (
+            f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+        )
+
+    try:
+        with open(env_config_path, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        return {}
+
+    tenant_env = (cfg.get("environments") or {}).get(tenant_name) or {}
+    cached_user_env = cfg.get("cached_user_env") or {}
+
+    merged = cached_user_env | tenant_env
+
+    return dict(merged)
+
+
+def apply_env_overrides(
+    base: Dict[str, Any],
+    tenant_name: str,
+    keys: Optional[Iterable[str]] = None,
+    env_config_path: Optional[str] = None,
+) -> Dict[str, Any]:
+    """
+    Returns a new dict where base is overridden by tenant-defined values.
+    - If keys is None, tries to override any keys present in tenant env.
+    - Only overrides when the tenant value is present and not None.
+    """
+    env = get_env_settings(tenant_name, env_config_path=env_config_path)
+    merged = dict(base)
+    keys_to_consider = keys if keys is not None else env.keys()
+
+    for k in keys_to_consider:
+        if k in env and env[k] is not None:
+            merged[k] = env[k]
+    return merged
+
+
+def tenant_setup(
+    service_url: Optional[str], tenant_name: str
+) -> Tuple[Optional[str], Optional[str], Dict[str, Any]]:
     # service_instance = ServiceInstance(
     #     service_url=service_url,
     #     tenant_name=tenant_name
@@ -159,21 +218,53 @@ def tenant_setup(service_url: str, tenant_name: str):
     # else:
     #     tenant_token = service_instance._get_tenant_token(tenant_id)
 
-    auth_config_path = f"{os.path.expanduser('~')}/.cache/orchestrate/credentials.yaml"
-    env_config_path = f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+    auth_config_path = (
+        f"{os.path.expanduser('~')}/.cache/orchestrate/credentials.yaml"
+    )
+    env_config_path = (
+        f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+    )
+
+    try:
+        with open(auth_config_path, "r", encoding="utf-8") as f:
+            auth_config = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        auth_config = {}
+
+    try:
+        with open(env_config_path, "r", encoding="utf-8") as f:
+            env_config = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        env_config = {}
 
-    # TO-DO: update SDK and use SDK to manage this
-    with open(auth_config_path, "r") as f:
-        auth_config = yaml.safe_load(f)
-    # auth_config["auth"][tenant_name] = {"wxo_mcsp_token": tenant_token}
+    environments = env_config.setdefault("environments", {})
+    context = env_config.setdefault("context", {})
 
-    with open(env_config_path, "r") as f:
-        env_config = yaml.safe_load(f)
-    env_config["environments"][tenant_name] = {"wxo_url": service_url}
-    env_config["context"]["active_environment"] = tenant_name
+    tenant_env = environments.setdefault(tenant_name, {})
+
+    if service_url and str(service_url).strip():
+        tenant_env["wxo_url"] = service_url
+
+    resolved_service_url = tenant_env.get("wxo_url")
+
+    context["active_environment"] = tenant_name
+
+    # Ensure parent directories exist so tests (which may run in clean envs)
+    # can write these files without raising FileNotFoundError.
+    auth_dir = os.path.dirname(auth_config_path)
+    env_dir = os.path.dirname(env_config_path)
+    os.makedirs(auth_dir, exist_ok=True)
+    os.makedirs(env_dir, exist_ok=True)
 
     with open(auth_config_path, "w") as f:
         yaml.dump(auth_config, f)
     with open(env_config_path, "w") as f:
         yaml.dump(env_config, f)
-    return auth_config["auth"][tenant_name]["wxo_mcsp_token"]
+
+    token = (
+        auth_config.get("auth", {}).get(tenant_name, {}).get("wxo_mcsp_token")
+    )
+
+    env_merged = get_env_settings(tenant_name, env_config_path=env_config_path)
+
+    return token, resolved_service_url, env_merged

wxo_agentic_evaluation/service_provider/__init__.py

@@ -1,35 +1,200 @@
-from wxo_agentic_evaluation.service_provider.ollama_provider import OllamaProvider
-from wxo_agentic_evaluation.service_provider.watsonx_provider import WatsonXProvider
-from wxo_agentic_evaluation.service_provider.model_proxy_provider import ModelProxyProvider
+import logging
+import os
+
+from rich.console import Console
+from rich.logging import RichHandler
+
 from wxo_agentic_evaluation.arg_configs import ProviderConfig
+from wxo_agentic_evaluation.service_provider.gateway_provider import (
+    GatewayProvider,
+)
+from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
+    ModelProxyProvider,
+)
+from wxo_agentic_evaluation.service_provider.ollama_provider import (
+    OllamaProvider,
+)
+from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import (
+    GatewayProviderLLMKitWrapper,
+    ModelProxyProviderLLMKitWrapper,
+    WatsonXLLMKitWrapper,
+)
+from wxo_agentic_evaluation.service_provider.watsonx_provider import (
+    WatsonXProvider,
+)
 
-import os
+try:
+    from wxo_agentic_evaluation.service_provider.portkey_provider import (
+        PortkeyProvider,
+    )
+except:
+    pass
+
+
+USE_GATEWAY_MODEL_PROVIDER: bool = (
+    os.environ.get("USE_GATEWAY_MODEL_PROVIDER", "FALSE").upper() == "TRUE"
+)
+
+
+_logging_console = Console(stderr=True)
+
+logger = logging.getLogger(__name__)
+
+
+def get_log_level_from_env():
+
+    level_env = os.getenv("WXO_EVALUATION_LOGLEVEL")
+    return level_env
+
+
+LOGGING_ENABLED = get_log_level_from_env() is not None
+
+
+def configure_logging_for_package_from_env(
+    package_name: str = "wxo_agentic_evaluation",
+    ensure_output: bool = True,
+) -> None:
+    """
+    Configure logging using the env var WXO_EVALUATION_LOGLEVEL - no logging if that's not set
+    """
+    try:
+        level_env = get_log_level_from_env()
+        if not level_env:
+            return
+
+        level = None
+        upper = level_env.strip().upper()
+        if hasattr(logging, upper):
+            level = getattr(logging, upper, None)
+
+        pkg_logger = logging.getLogger(package_name)
+        pkg_logger.setLevel(level)
+
+        if ensure_output:
+            if not pkg_logger.handlers:
+                handler = RichHandler(
+                    console=_logging_console,
+                    rich_tracebacks=True,
+                    show_time=False,
+                    show_level=False,
+                    show_path=False,
+                    markup=True,
+                    enable_link_path=True,
+                    omit_repeated_times=True,
+                    tracebacks_theme="github-dark",
+                )
+                handler.setFormatter(
+                    logging.Formatter("%(levelname)s %(message)s")
+                )
+                handler.setLevel(logging.NOTSET)
+                pkg_logger.addHandler(handler)
+            pkg_logger.propagate = False
+
+        # Quiet common noisy debug libs
+        for name in (
+            "urllib3",
+            "urllib3.connectionpool",
+            "requests.packages.urllib3",
+        ):
+            logging.getLogger(name).setLevel(logging.WARNING)
+    except:
+        logger.warning("Input log level %s not valid", level_env)
+
+
+configure_logging_for_package_from_env()
+
+
+def _instantiate_provider(
+    config: ProviderConfig, is_referenceless_eval: bool = False, **kwargs
+):
 
-def _instantiate_provider(config: ProviderConfig, **kwargs):
     if config.provider == "watsonx":
-        return WatsonXProvider(model_id=config.model_id, **kwargs)
+        logger.info("Instantiate watsonx provider")
+        if is_referenceless_eval:
+            provider = WatsonXLLMKitWrapper
+        else:
+            provider = WatsonXProvider
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
     elif config.provider == "ollama":
+        logger.info("Instantiate Ollama")
         return OllamaProvider(model_id=config.model_id, **kwargs)
+
+    elif config.provider == "gateway":
+        logger.info("Instantiate gateway inference provider")
+        if is_referenceless_eval:
+            provider = GatewayProviderLLMKitWrapper
+        else:
+            provider = GatewayProvider
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     elif config.provider == "model_proxy":
-        return ModelProxyProvider(model_id=config.model_id, **kwargs)
+        logger.info("Instantiate model proxy provider")
+        if is_referenceless_eval:
+            provider = ModelProxyProviderLLMKitWrapper
+        else:
+            provider = ModelProxyProvider
+
+        return provider(
+            model_id=config.model_id,
+            embedding_model_id=config.embedding_model_id,
+            **kwargs,
+        )
+
     else:
-        raise RuntimeError(f"target provider is not supported {config.provider}")
-    
-def get_provider(config: ProviderConfig = None, model_id: str = None, **kwargs):
+        raise RuntimeError(
+            f"target provider is not supported {config.provider}"
+        )
+
+
+def get_provider(
+    config: ProviderConfig = None,
+    model_id: str = None,
+    embedding_model_id: str = None,
+    referenceless_eval: bool = False,
+    provider: str = None,
+    api_key: str = None,
+    use_portkey_provider: bool = False,
+    **kwargs,
+):
+
+    if use_portkey_provider:
+        return PortkeyProvider(
+            provider=provider, model_id=model_id, api_key=api_key, **kwargs
+        )
+
     if config:
         return _instantiate_provider(config, **kwargs)
 
     if not model_id:
         raise ValueError("model_id must be provided if config is not supplied")
 
+    if USE_GATEWAY_MODEL_PROVIDER:
+        logger.info("[d b]Using gateway inference provider override")
+        config = ProviderConfig(provider="gateway", model_id=model_id)
+        return _instantiate_provider(config, referenceless_eval, **kwargs)
+
     if "WATSONX_APIKEY" in os.environ and "WATSONX_SPACE_ID" in os.environ:
-        config = ProviderConfig(provider="watsonx", model_id=model_id)
-        return _instantiate_provider(config, **kwargs)
+        logger.info("[d b]Using watsonx inference provider")
+        config = ProviderConfig(
+            provider="watsonx",
+            model_id=model_id,
+            embedding_model_id=embedding_model_id,
+        )
+        return _instantiate_provider(config, referenceless_eval, **kwargs)
 
-    if "WO_API_KEY" in os.environ and "WO_INSTANCE" in os.environ:
+    if "WO_INSTANCE" in os.environ:
+        logger.info("[d b]Using model_proxy inference provider")
         config = ProviderConfig(provider="model_proxy", model_id=model_id)
-        return _instantiate_provider(config, **kwargs)
+        return _instantiate_provider(config, referenceless_eval, **kwargs)
 
-    raise RuntimeError(
-        "No provider found. Please either provide a config or set the required environment variables."
-    )
+    logger.info("[d b]Using gateway inference provider default")
+    config = ProviderConfig(provider="gateway", model_id=model_id)
+    return _instantiate_provider(config, referenceless_eval, **kwargs)