ibm-watsonx-orchestrate-evaluation-framework 1.1.1__py3-none-any.whl → 1.1.3__py3-none-any.whl

wxo_agentic_evaluation/service_instance.py

@@ -1,8 +1,13 @@
 import logging
-import yaml
 import os
+
 import requests
-from wxo_agentic_evaluation.utils.utils import is_saas_url, is_ibm_cloud_url
+import shutil
+from pathlib import Path
+from typing import Optional, Any, Dict, Iterable, Tuple
+import yaml
+
+from wxo_agentic_evaluation.utils.utils import is_ibm_cloud_url, is_saas_url
 
 logger = logging.getLogger(__name__)
 
@@ -11,13 +16,15 @@ USER = {"username": "wxo.archer@ibm.com", "password": "watsonx"}
 
 class ServiceInstance:
     def __init__(
-        self, service_url, tenant_name, is_saas: bool = None, is_ibm_cloud: bool = None
+        self,
+        service_url,
+        tenant_name,
+        is_saas: bool = None,
+        is_ibm_cloud: bool = None,
     ) -> None:
         self.service_url = service_url
         self.tenant_name = tenant_name
-        STAGING_AUTH_ENDPOINT = (
-            "https://iam.platform.test.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
-        )
+        STAGING_AUTH_ENDPOINT = "https://iam.platform.test.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
         PROD_AUTH_ENDPOINT = (
             "https://iam.platform.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
         )
@@ -25,7 +32,9 @@ class ServiceInstance:
 
         self.is_saas = is_saas_url(service_url) if is_saas is None else is_saas
         self.is_ibm_cloud = (
-            is_ibm_cloud_url(service_url) if is_ibm_cloud is None else is_ibm_cloud
+            is_ibm_cloud_url(service_url)
+            if is_ibm_cloud is None
+            else is_ibm_cloud
         )
 
         if self.is_saas:
@@ -88,7 +97,8 @@ class ServiceInstance:
 
     def _get_tenant_token(self, tenant_id: str):
         resp = requests.post(
-            self.tenant_auth_endpoint.format(self.service_url, tenant_id), data=USER
+            self.tenant_auth_endpoint.format(self.service_url, tenant_id),
+            data=USER,
         )
         if resp.status_code == 200:
             return resp.json()["access_token"]
@@ -122,7 +132,9 @@ class ServiceInstance:
             "tags": ["test"],
         }
 
-        resp = requests.post(self.tenant_url, headers=headers, json=tenant_config)
+        resp = requests.post(
+            self.tenant_url, headers=headers, json=tenant_config
+        )
         if resp.status_code == 201:
             return True
         else:
@@ -147,8 +159,51 @@ class ServiceInstance:
 
         return default_tenant["id"]
 
+def get_env_settings(
+    tenant_name: str,
+    env_config_path: Optional[str] = None
+) -> Dict[str, Any]:
+    if env_config_path is None:
+        env_config_path = f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+
+    try:
+        with open(env_config_path, "r", encoding="utf-8") as f:
+            cfg = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        return {}
+
+    tenant_env = (cfg.get("environments") or {}).get(tenant_name) or {}
+    cached_user_env = cfg.get("cached_user_env") or {}
+
+    merged = cached_user_env | tenant_env
+
+    return dict(merged)
+
+
 
-def tenant_setup(service_url: str, tenant_name: str):
+def apply_env_overrides(
+    base: Dict[str, Any],
+    tenant_name: str,
+    keys: Optional[Iterable[str]] = None,
+    env_config_path: Optional[str] = None
+) -> Dict[str, Any]:
+    """
+    Returns a new dict where base is overridden by tenant-defined values.
+    - If keys is None, tries to override any keys present in tenant env.
+    - Only overrides when the tenant value is present and not None.
+    """
+    env = get_env_settings(tenant_name, env_config_path=env_config_path)
+    merged = dict(base)
+    keys_to_consider = keys if keys is not None else env.keys()
+
+    for k in keys_to_consider:
+        if k in env and env[k] is not None:
+            merged[k] = env[k]
+    return merged
+
+
+
+def tenant_setup(service_url: Optional[str], tenant_name: str) -> Tuple[Optional[str], Optional[str], Dict[str, Any]]:
     # service_instance = ServiceInstance(
     #     service_url=service_url,
     #     tenant_name=tenant_name
@@ -159,21 +214,48 @@ def tenant_setup(service_url: str, tenant_name: str):
     # else:
     #     tenant_token = service_instance._get_tenant_token(tenant_id)
 
-    auth_config_path = f"{os.path.expanduser('~')}/.cache/orchestrate/credentials.yaml"
-    env_config_path = f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+    auth_config_path = (
+        f"{os.path.expanduser('~')}/.cache/orchestrate/credentials.yaml"
+    )
+    env_config_path = (
+        f"{os.path.expanduser('~')}/.config/orchestrate/config.yaml"
+    )
+
+    try:
+        with open(auth_config_path, "r", encoding="utf-8") as f:
+            auth_config = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        auth_config = {}
+
+    try:
+        with open(env_config_path, "r", encoding="utf-8") as f:
+            env_config = yaml.safe_load(f) or {}
+    except FileNotFoundError:
+        env_config = {}
 
-    # TO-DO: update SDK and use SDK to manage this
-    with open(auth_config_path, "r") as f:
-        auth_config = yaml.safe_load(f)
-    # auth_config["auth"][tenant_name] = {"wxo_mcsp_token": tenant_token}
+    environments = env_config.setdefault("environments", {})
+    context = env_config.setdefault("context", {})
 
-    with open(env_config_path, "r") as f:
-        env_config = yaml.safe_load(f)
-    env_config["environments"][tenant_name] = {"wxo_url": service_url}
-    env_config["context"]["active_environment"] = tenant_name
+    tenant_env = environments.setdefault(tenant_name, {})
+
+    if service_url and str(service_url).strip():
+        tenant_env["wxo_url"] = service_url
+
+    resolved_service_url = tenant_env.get("wxo_url")
+
+    context["active_environment"] = tenant_name
 
     with open(auth_config_path, "w") as f:
         yaml.dump(auth_config, f)
     with open(env_config_path, "w") as f:
         yaml.dump(env_config, f)
-    return auth_config["auth"][tenant_name]["wxo_mcsp_token"]
+
+    token = (
+        auth_config.get("auth", {})
+        .get(tenant_name, {})
+        .get("wxo_mcsp_token")
+    )
+
+    env_merged = get_env_settings(tenant_name, env_config_path=env_config_path)
+
+    return token, resolved_service_url, env_merged

wxo_agentic_evaluation/service_provider/__init__.py

@@ -1,12 +1,24 @@
-from wxo_agentic_evaluation.service_provider.ollama_provider import OllamaProvider
-from wxo_agentic_evaluation.service_provider.watsonx_provider import WatsonXProvider
-from wxo_agentic_evaluation.service_provider.model_proxy_provider import ModelProxyProvider
-from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import ModelProxyProviderLLMKitWrapper, WatsonXLLMKitWrapper
+import os
+
 from wxo_agentic_evaluation.arg_configs import ProviderConfig
+from wxo_agentic_evaluation.service_provider.model_proxy_provider import (
+    ModelProxyProvider,
+)
+from wxo_agentic_evaluation.service_provider.ollama_provider import (
+    OllamaProvider,
+)
+from wxo_agentic_evaluation.service_provider.referenceless_provider_wrapper import (
+    ModelProxyProviderLLMKitWrapper,
+    WatsonXLLMKitWrapper,
+)
+from wxo_agentic_evaluation.service_provider.watsonx_provider import (
+    WatsonXProvider,
+)
 
-import os
 
-def _instantiate_provider(config: ProviderConfig, is_referenceless_eval: bool = False, **kwargs):
+def _instantiate_provider(
+    config: ProviderConfig, is_referenceless_eval: bool = False, **kwargs
+):
     if config.provider == "watsonx":
         if is_referenceless_eval:
             provider = WatsonXLLMKitWrapper
@@ -22,12 +34,17 @@ def _instantiate_provider(config: ProviderConfig, is_referenceless_eval: bool =
             provider = ModelProxyProvider
         return provider(model_id=config.model_id, **kwargs)
     else:
-        raise RuntimeError(f"target provider is not supported {config.provider}")
-    
-def get_provider(config: ProviderConfig = None, model_id: str = None, referenceless_eval: bool = False, **kwargs):
-    if config:
-        return _instantiate_provider(config, **kwargs)
+        raise RuntimeError(
+            f"target provider is not supported {config.provider}"
+        )
 
+
+def get_provider(
+    config: ProviderConfig = None,
+    model_id: str = None,
+    referenceless_eval: bool = False,
+    **kwargs,
+):
     if not model_id:
         raise ValueError("model_id must be provided if config is not supplied")
 
@@ -35,10 +52,13 @@ def get_provider(config: ProviderConfig = None, model_id: str = None, referencel
         config = ProviderConfig(provider="watsonx", model_id=model_id)
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
-    if "WO_API_KEY" in os.environ and "WO_INSTANCE" in os.environ:
+    if "WO_INSTANCE" in os.environ:
         config = ProviderConfig(provider="model_proxy", model_id=model_id)
         return _instantiate_provider(config, referenceless_eval, **kwargs)
 
+    if config:
+        return _instantiate_provider(config, **kwargs)
+
     raise RuntimeError(
         "No provider found. Please either provide a config or set the required environment variables."
-    )
+    )

wxo_agentic_evaluation/service_provider/model_proxy_provider.py

@@ -1,16 +1,39 @@
 import os
-import requests
 import time
-
-from typing import List
 from threading import Lock
+from typing import List, Tuple
+
+import requests
 
 from wxo_agentic_evaluation.service_provider.provider import Provider
 from wxo_agentic_evaluation.utils.utils import is_ibm_cloud_url
 
-AUTH_ENDPOINT_AWS = "https://iam.platform.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
+AUTH_ENDPOINT_AWS = (
+    "https://iam.platform.saas.ibm.com/siusermgr/api/1.0/apikeys/token"
+)
 AUTH_ENDPOINT_IBM_CLOUD = "https://iam.cloud.ibm.com/identity/token"
-DEFAULT_PARAM = {"min_new_tokens": 1, "decoding_method": "greedy", "max_new_tokens": 400}
+DEFAULT_PARAM = {
+    "min_new_tokens": 1,
+    "decoding_method": "greedy",
+    "max_new_tokens": 400,
+}
+
+
+def _infer_cpd_auth_url(instance_url: str) -> str:
+    inst = (instance_url or "").rstrip("/")
+    if not inst:
+        return "/icp4d-api/v1/authorize"
+    if "/orchestrate" in inst:
+        base = inst.split("/orchestrate", 1)[0].rstrip("/")
+        return base + "/icp4d-api/v1/authorize"
+    return inst + "/icp4d-api/v1/authorize"
+
+
+def _normalize_cpd_auth_url(url: str) -> str:
+    u = (url or "").rstrip("/")
+    if u.endswith("/icp4d-api"):
+        return u + "/v1/authorize"
+    return url
 
 
 class ModelProxyProvider(Provider):
@@ -21,46 +44,184 @@ class ModelProxyProvider(Provider):
         instance_url=None,
         timeout=300,
         embedding_model_id=None,
-        params=None
+        params=None,
     ):
         super().__init__()
 
         instance_url = os.environ.get("WO_INSTANCE", instance_url)
-        api_key = os.environ.get("WO_API_KEY", api_key)
-        if not instance_url or not api_key:
-            raise RuntimeError("instance url and WO apikey must be specified to use WO model proxy")
+        if not instance_url:
+            raise RuntimeError(
+                "instance url must be specified to use WO model proxy"
+            )
 
         self.timeout = timeout
-        self.model_id = model_id
-
+        self.model_id = os.environ.get("MODEL_OVERRIDE", model_id)
         self.embedding_model_id = embedding_model_id
 
-        self.api_key = api_key
+        self.api_key = os.environ.get("WO_API_KEY", api_key)
+        self.username = os.environ.get("WO_USERNAME", None)
+        self.password = os.environ.get("WO_PASSWORD", None)
+        self.auth_type = os.environ.get(
+            "WO_AUTH_TYPE", ""
+        ).lower()  # explicit override if set, otherwise inferred- match ADK values
+        explicit_auth_url = os.environ.get("AUTHORIZATION_URL", None)
+
         self.is_ibm_cloud = is_ibm_cloud_url(instance_url)
-        self.auth_url = AUTH_ENDPOINT_IBM_CLOUD if self.is_ibm_cloud else AUTH_ENDPOINT_AWS
-    
-        self.instance_url = instance_url
-        self.url = self.instance_url + "/ml/v1/text/generation?version=2024-05-01"
+        self.instance_url = instance_url.rstrip("/")
+
+        self.auth_mode, self.auth_url = self._resolve_auth_mode_and_url(
+            explicit_auth_url=explicit_auth_url
+        )
+        self._wo_ssl_verify = (
+            os.environ.get("WO_SSL_VERIFY", "true").lower() != "false"
+        )
+        env_space_id = os.environ.get("WATSONX_SPACE_ID", None)
+        if self.auth_mode == "cpd":
+            if not env_space_id or not env_space_id.strip():
+                raise RuntimeError(
+                    "CPD mode requires WATSONX_SPACE_ID environment variable to be set"
+                )
+            self.space_id = env_space_id.strip()
+        else:
+            self.space_id = (
+                env_space_id.strip()
+                if env_space_id and env_space_id.strip()
+                else "1"
+            )
+
+        if self.auth_mode == "cpd":
+            if "/orchestrate" in self.instance_url:
+                self.instance_url = self.instance_url.split("/orchestrate", 1)[
+                    0
+                ].rstrip("/")
+            if not self.username:
+                raise RuntimeError("CPD auth requires WO_USERNAME to be set")
+            if not (self.password or self.api_key):
+                raise RuntimeError(
+                    "CPD auth requires either WO_PASSWORD or WO_API_KEY to be set (with WO_USERNAME)"
+                )
+        else:
+            if not self.api_key:
+                raise RuntimeError(
+                    "WO_API_KEY must be specified for SaaS or IBM IAM auth"
+                )
+
+        self.url = (
+            self.instance_url + "/ml/v1/text/generation?version=2024-05-01"
+        )
         self.embedding_url = self.instance_url + "/ml/v1/text/embeddings"
 
         self.lock = Lock()
         self.token, self.refresh_time = self.get_token()
         self.params = params if params else DEFAULT_PARAM
 
-    def get_token(self):
+    def _resolve_auth_mode_and_url(
+        self, explicit_auth_url: str | None
+    ) -> Tuple[str, str]:
+        """
+        Returns (auth_mode, auth_url)
+        - auth_mode: "cpd" | "ibm_iam" | "saas"
+        """
+        if explicit_auth_url:
+            if "/icp4d-api" in explicit_auth_url:
+                return "cpd", _normalize_cpd_auth_url(explicit_auth_url)
+            if self.auth_type == "ibm_iam":
+                return "ibm_iam", explicit_auth_url
+            elif self.auth_type == "saas":
+                return "saas", explicit_auth_url
+            else:
+                mode = "ibm_iam" if self.is_ibm_cloud else "saas"
+                return mode, explicit_auth_url
+
+        if self.auth_type == "cpd":
+            inferred_cpd_url = _infer_cpd_auth_url(self.instance_url)
+            return "cpd", inferred_cpd_url
+        if self.auth_type == "ibm_iam":
+            return "ibm_iam", AUTH_ENDPOINT_IBM_CLOUD
+        if self.auth_type == "saas":
+            return "saas", AUTH_ENDPOINT_AWS
+
+        if "/orchestrate" in self.instance_url:
+            inferred_cpd_url = _infer_cpd_auth_url(self.instance_url)
+            return "cpd", inferred_cpd_url
+
         if self.is_ibm_cloud:
-            payload = {"grant_type": "urn:ibm:params:oauth:grant-type:apikey", "apikey": self.api_key}
-            resp = requests.post(self.auth_url, data=payload)
-            token_key = "access_token"
+            return "ibm_iam", AUTH_ENDPOINT_IBM_CLOUD
         else:
-            payload = {"apikey": self.api_key}
-            resp = requests.post(self.auth_url, json=payload)
-            token_key = "token"
+            return "saas", AUTH_ENDPOINT_AWS
+
+    def get_token(self):
+        headers = {}
+        post_args = {}
+        timeout = 10
+        exchange_url = self.auth_url
+
+        if self.auth_mode == "ibm_iam":
+            headers = {
+                "Accept": "application/json",
+                "Content-Type": "application/x-www-form-urlencoded",
+            }
+            form_data = {
+                "grant_type": "urn:ibm:params:oauth:grant-type:apikey",
+                "apikey": self.api_key,
+            }
+            post_args = {"data": form_data}
+            resp = requests.post(
+                exchange_url,
+                headers=headers,
+                timeout=timeout,
+                verify=self._wo_ssl_verify,
+                **post_args,
+            )
+        elif self.auth_mode == "cpd":
+            headers = {
+                "Accept": "application/json",
+                "Content-Type": "application/json",
+            }
+            body = {"username": self.username}
+            if self.password:
+                body["password"] = self.password
+            else:
+                body["api_key"] = self.api_key
+            timeout = self.timeout
+            resp = requests.post(
+                exchange_url,
+                headers=headers,
+                json=body,
+                timeout=timeout,
+                verify=self._wo_ssl_verify,
+            )
+        else:
+            headers = {
+                "Accept": "application/json",
+                "Content-Type": "application/json",
+            }
+            post_args = {"json": {"apikey": self.api_key}}
+            resp = requests.post(
+                exchange_url,
+                headers=headers,
+                timeout=timeout,
+                verify=self._wo_ssl_verify,
+                **post_args,
+            )
+
         if resp.status_code == 200:
             json_obj = resp.json()
-            token = json_obj[token_key]
-            expires_in = json_obj["expires_in"]
-            refresh_time = time.time() + int(0.8*expires_in)
+            token = json_obj.get("access_token") or json_obj.get("token")
+            if not token:
+                raise RuntimeError(
+                    f"No token field found in response: {json_obj!r}"
+                )
+
+            expires_in = json_obj.get("expires_in")
+            try:
+                expires_in = int(expires_in) if expires_in is not None else None
+            except Exception:
+                expires_in = None
+            if not expires_in or expires_in <= 0:
+                expires_in = int(os.environ.get("TOKEN_DEFAULT_EXPIRES_IN", 1))
+
+            refresh_time = time.time() + int(0.8 * expires_in)
             return token, refresh_time
 
         resp.raise_for_status()
@@ -76,13 +237,24 @@ class ModelProxyProvider(Provider):
 
     def encode(self, sentences: List[str]) -> List[list]:
         if self.embedding_model_id is None:
-            raise Exception("embedding model id must be specified for text generation")
+            raise Exception(
+                "embedding model id must be specified for text generation"
+            )
 
         self.refresh_token_if_expires()
         headers = self.get_header()
-        payload = {"inputs": sentences, "model_id": self.embedding_model_id, "space_id": "1"}
-                   #"timeout": self.timeout}
-        resp = requests.post(self.embedding_url, json=payload, headers=headers)
+        payload = {
+            "inputs": sentences,
+            "model_id": self.embedding_model_id,
+            "space_id": self.space_id,
+        }
+        # "timeout": self.timeout}
+        resp = requests.post(
+            self.embedding_url,
+            json=payload,
+            headers=headers,
+            verify=self._wo_ssl_verify,
+        )
 
         if resp.status_code == 200:
             json_obj = resp.json()
@@ -95,9 +267,16 @@ class ModelProxyProvider(Provider):
             raise Exception("model id must be specified for text generation")
         self.refresh_token_if_expires()
         headers = self.get_header()
-        payload = {"input": sentence, "model_id": self.model_id, "space_id": "1",
-                   "timeout": self.timeout, "parameters": self.params}
-        resp = requests.post(self.url, json=payload, headers=headers)
+        payload = {
+            "input": sentence,
+            "model_id": self.model_id,
+            "space_id": self.space_id,
+            "timeout": self.timeout,
+            "parameters": self.params,
+        }
+        resp = requests.post(
+            self.url, json=payload, headers=headers, verify=self._wo_ssl_verify
+        )
         if resp.status_code == 200:
             return resp.json()["results"][0]["generated_text"]
 
@@ -105,5 +284,8 @@ class ModelProxyProvider(Provider):
 
 
 if __name__ == "__main__":
-    provider = ModelProxyProvider(model_id="meta-llama/llama-3-3-70b-instruct", embedding_model_id="ibm/slate-30m-english-rtrvr")
+    provider = ModelProxyProvider(
+        model_id="meta-llama/llama-3-3-70b-instruct",
+        embedding_model_id="ibm/slate-30m-english-rtrvr",
+    )
     print(provider.query("ok"))

wxo_agentic_evaluation/service_provider/ollama_provider.py

@@ -1,17 +1,16 @@
-import requests
 import json
-from wxo_agentic_evaluation.service_provider.provider import Provider
-from typing import List
 import os
+from typing import List
+
+import requests
+
+from wxo_agentic_evaluation.service_provider.provider import Provider
 
 OLLAMA_URL = os.environ.get("OLLAMA_HOST", "http://localhost:11434")
 
 
 class OllamaProvider(Provider):
-    def __init__(
-        self,
-        model_id=None
-    ):
+    def __init__(self, model_id=None):
         self.url = OLLAMA_URL + "/api/generate"
         self.model_id = model_id
         super().__init__()
@@ -20,14 +19,14 @@ class OllamaProvider(Provider):
         payload = {"model": self.model_id, "prompt": sentence}
         resp = requests.post(self.url, json=payload, stream=True)
         final_text = ""
-        data = b''
+        data = b""
         for chunk in resp:
             data += chunk
-            if data.endswith(b'\n'):
+            if data.endswith(b"\n"):
                 json_obj = json.loads(data)
                 if not json_obj["done"] and json_obj["response"]:
                     final_text += json_obj["response"]
-                data = b''
+                data = b""
 
         return final_text
 
@@ -37,4 +36,4 @@ class OllamaProvider(Provider):
 
 if __name__ == "__main__":
     provider = OllamaProvider(model_id="llama3.1:8b")
-    print(provider.query("ok"))
+    print(provider.query("ok"))

wxo_agentic_evaluation/service_provider/provider.py

@@ -16,4 +16,3 @@ class Provider(ABC):
     @abstractmethod
     def encode(self, sentences: List[str]) -> List[list]:
         pass
-