ibm-watsonx-orchestrate 1.12.0b0__py3-none-any.whl → 1.13.0b0__py3-none-any.whl

ibm_watsonx_orchestrate/utils/environment.py

@@ -6,16 +6,32 @@ import subprocess
 import sys
 import tempfile
 from pathlib import Path
+from typing import Tuple, OrderedDict
 from urllib.parse import urlparse
+from enum import Enum
 
 from dotenv import dotenv_values
 
+from ibm_watsonx_orchestrate.cli.commands.environment.types import EnvironmentAuthType
 from ibm_watsonx_orchestrate.cli.commands.server.types import WatsonXAIEnvConfig, ModelGatewayEnvConfig
 from ibm_watsonx_orchestrate.cli.config import USER_ENV_CACHE_HEADER, Config
 from ibm_watsonx_orchestrate.client.utils import is_arm_architecture
+from ibm_watsonx_orchestrate.utils.utils import parse_bool_safe, parse_int_safe, parse_bool_safe_and_get_raw_val
+
 
 logger = logging.getLogger(__name__)
 
+class DeveloperEditionSources(str, Enum):
+    MYIBM = "myibm"
+    ORCHESTRATE = "orchestrate"
+    INTERNAL = "internal"
+    CUSTOM = "custom"
+
+    def __str__(self):
+        return self.value
+    
+    def __repr__(self):
+        return self.value
 
 class EnvService:
 
@@ -65,8 +81,8 @@ class EnvService:
             return env_file
 
     @staticmethod
-    def read_env_file (env_path: Path | str) -> dict:
-        return dotenv_values(str(env_path))
+    def read_env_file (env_path: Path | str) -> OrderedDict:
+        return dotenv_values(env_path)
 
     def get_user_env (self, user_env_file: Path | str, fallback_to_persisted_env: bool = True) -> dict:
         if user_env_file is not None and isinstance(user_env_file, str):
@@ -80,19 +96,19 @@ class EnvService:
         return user_env
 
     @staticmethod
-    def get_dev_edition_source_core(env_dict: dict | None) -> str:
+    def get_dev_edition_source_core(env_dict: dict | None) -> DeveloperEditionSources | str:
         if not env_dict:
-            return "myibm"
+            return DeveloperEditionSources.MYIBM
 
         source = env_dict.get("WO_DEVELOPER_EDITION_SOURCE")
 
         if source:
             return source
         if env_dict.get("WO_INSTANCE"):
-            return "orchestrate"
-        return "myibm"
+            return DeveloperEditionSources.ORCHESTRATE
+        return DeveloperEditionSources.MYIBM
 
-    def get_dev_edition_source(self, user_env_file: str):
+    def get_dev_edition_source(self, user_env_file: str) -> DeveloperEditionSources | str:
         return self.get_dev_edition_source_core(self.get_user_env(user_env_file))
 
     @staticmethod
@@ -105,17 +121,35 @@ class EnvService:
 
         return merged
 
+    @staticmethod
+    def resolve_auth_type (env_dict: dict) -> str | None:
+        auth_type = env_dict.get("WO_AUTH_TYPE")
+
+        # Try infer the auth type if not provided
+        if not auth_type:
+            instance_url = env_dict.get("WO_INSTANCE")
+            if instance_url:
+                if ".cloud.ibm.com" in instance_url:
+                    auth_type = EnvironmentAuthType.IBM_CLOUD_IAM.value
+                elif ".ibm.com" in instance_url:
+                    auth_type = EnvironmentAuthType.MCSP.value
+                elif "https://cpd" in instance_url:
+                    auth_type = EnvironmentAuthType.CPD.value
+
+        return auth_type
+
     @staticmethod
     def __get_default_registry_env_vars_by_dev_edition_source (default_env: dict, user_env: dict, source: str) -> dict[str, str]:
         component_registry_var_names = {key for key in default_env if key.endswith("_REGISTRY")} | {'REGISTRY_URL'}
 
         registry_url = user_env.get("REGISTRY_URL", None)
+        user_env["HAS_USER_PROVIDED_REGISTRY_URL"] = registry_url is not None
         if not registry_url:
-            if source == "internal":
+            if source == DeveloperEditionSources.INTERNAL:
                 registry_url = "us.icr.io/watson-orchestrate-private"
-            elif source == "myibm":
+            elif source == DeveloperEditionSources.MYIBM:
                 registry_url = "cp.icr.io/cp/wxo-lite"
-            elif source == "orchestrate":
+            elif source == DeveloperEditionSources.ORCHESTRATE:
                 # extract the hostname from the WO_INSTANCE URL, and replace the "api." prefix with "registry." to construct the registry URL per region
                 wo_url = user_env.get("WO_INSTANCE")
 
@@ -123,15 +157,34 @@ class EnvService:
                     raise ValueError(
                         "WO_INSTANCE is required in the environment file if the developer edition source is set to 'orchestrate'.")
 
-                parsed = urlparse(wo_url)
-                hostname = parsed.hostname
+                wo_auth_type = EnvService.resolve_auth_type(user_env)
+
+                if wo_auth_type == EnvironmentAuthType.CPD.value:
+                    registry_url = "cpd/cp/wxo-lite"
 
-                registry_url = f"registry.{hostname[4:]}/cp/wxo-lite"
+                else:
+                    parsed = urlparse(wo_url)
+                    hostname = parsed.hostname
+
+                    registry_url = f"registry.{hostname[4:]}/cp/wxo-lite"
+            elif source == DeveloperEditionSources.CUSTOM:
+                registry_url = user_env.get("CUSTOM_REGISTRY_URL")
+                if not registry_url:
+                    raise ValueError(
+                        f"REGISTRY_URL is required in the environment file when the developer edition source is set to 'custom'."
+                    )
             else:
                 raise ValueError(
-                    f"Unknown value for developer edition source: {source}. Must be one of ['internal', 'myibm', 'orchestrate']."
+                    f"Unknown value for developer edition source: {source}. Must be one of {list(map(str, DeveloperEditionSources))}."
                 )
-
+        
+        # For non-custom use cases remove etcd and elastic search as they have different default registries
+        if source != DeveloperEditionSources.CUSTOM:
+            component_registry_var_names -= {"ETCD_REGISTRY", "ELASTICSEARCH_REGISTRY"}
+        # In the custom case default the OPENSOURCE_REGISTRY_PROXY to also be the REGISTRY_URL
+        else:
+            component_registry_var_names.add("OPENSOURCE_REGISTRY_PROXY")
+        
         result = {name: registry_url for name in component_registry_var_names}
         return result
 
@@ -145,21 +198,28 @@ class EnvService:
 
     @staticmethod
     def write_merged_env_file (merged_env: dict, target_path: str = None) -> Path:
+        target_file = None
         if target_path:
-            file = open(target_path, "w")
+            target_file = target_path
+
         else:
-            file = tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".env")
+            with tempfile.NamedTemporaryFile(mode="w", delete=False, suffix=".env") as ntf:
+                target_file = ntf.name
 
-        with file:
+        with open(target_file, "w") as file:
             for key, val in merged_env.items():
                 file.write(f"{key}={val}\n")
-        return Path(file.name)
 
-    def persist_user_env (self, env: dict, include_secrets: bool = False) -> None:
+        return Path(target_file)
+
+    def persist_user_env (self, env: dict, include_secrets: bool = False, source: DeveloperEditionSources| str | None = None) -> None:
         if include_secrets:
             persistable_env = env
         else:
             persistable_env = {k: env[k] for k in EnvService.__NON_SECRET_ENV_ITEMS if k in env}
+        
+        if source == DeveloperEditionSources.CUSTOM and "REGISTRY_URL" in env:
+            persistable_env["CUSTOM_REGISTRY_URL"] = env.get("REGISTRY_URL")
 
         self.__config.save(
             {
@@ -367,3 +427,88 @@ class EnvService:
 
     def define_saas_wdu_runtime (self, value: str = "none") -> None:
         self.__config.write(USER_ENV_CACHE_HEADER, "SAAS_WDU_RUNTIME", value)
+
+    @staticmethod
+    def did_user_provide_registry_url (env_dict: dict) -> bool:
+        has_user_provided_registry_url = parse_bool_safe(env_dict.get("HAS_USER_PROVIDED_REGISTRY_URL"), fallback=None)
+
+        if has_user_provided_registry_url is None:
+            raise Exception("Unable to determine if user has provided REGISTRY_URL.")
+
+        return has_user_provided_registry_url
+
+
+class EnvSettingsService:
+
+    @staticmethod
+    def __get_string_safe(value) -> str | None:
+        if value is not None and isinstance(value, str):
+            value = value.strip()
+
+            if value == "":
+                value = None
+
+            return value
+
+        return None
+
+    def __init__(self, env_file: Path | str) -> None:
+        self.__env_dict = dotenv_values(str(env_file))
+
+    def get_env(self) -> dict:
+        return self.__env_dict
+
+    def get_user_provided_docker_os_type(self) -> str | None:
+        return self.__get_string_safe(self.__env_dict.get("DOCKER_IMAGE_OS_TYPE"))
+
+    def get_user_provided_docker_arch_type(self) -> str | None:
+        return self.__get_string_safe(self.__env_dict.get("DOCKER_IMAGE_ARCH_TYPE"))
+
+    def get_wo_instance_url(self) -> str:
+        return self.__env_dict["WO_INSTANCE"]
+
+    def get_parsed_wo_instance_details(self) -> Tuple[str, str, str, str]:
+        instance_url = self.get_wo_instance_url()
+        parsed = urlparse(instance_url)
+        route_parts = parsed.path.split("/")
+
+        orchestrate_namespace = route_parts[2]      # this is usually "cpd-instance-1"
+        wxo_tenant_id = route_parts[4]
+
+        return parsed.scheme, parsed.netloc, orchestrate_namespace, wxo_tenant_id
+
+    def get_wo_username(self) -> str:
+        return self.__env_dict.get("WO_USERNAME")
+
+    def get_wo_password(self) -> str:
+        return self.__env_dict.get("WO_PASSWORD")
+
+    def get_wo_api_key(self) -> str:
+        return self.__env_dict.get("WO_API_KEY")
+
+    def use_parallel_docker_image_layer_pulls(self):
+        return parse_bool_safe(value=self.__env_dict.get("DOCKER_IMAGE_PULL_LAYERS_PARALLELISM"), fallback=True)
+
+    def get_docker_pull_parallel_worker_count(self):
+        fallback = 7
+        parsed = parse_int_safe(value=self.__env_dict.get("DOCKER_IMAGE_PULL_PARALLEL_WORKERS_COUNT"),
+                                fallback=fallback)
+
+        if parsed < 1:
+            parsed = fallback
+
+        return parsed
+
+    def use_ranged_requests_during_docker_pulls(self):
+        return parse_bool_safe(value=self.__env_dict.get("USE_RANGE_REQUESTS_IN_DOCKER_IMAGE_PULLS"), fallback=True)
+
+    def get_wo_instance_ssl_verify(self) -> bool | str:
+        ssl_verify, path = parse_bool_safe_and_get_raw_val(value=self.__env_dict.get("WO_VERIFY_SSL"), fallback=True)
+        if ssl_verify is True and path is not None and isinstance(path, str):
+            if not os.path.exists(path) or not os.path.isfile(path):
+                logger.error(f"WO SSL verification certificate path not found or could not be accessed: {str(path)}")
+                sys.exit(1)
+
+            return str(path)
+
+        return ssl_verify

ibm_watsonx_orchestrate/utils/exceptions.py

@@ -20,4 +20,4 @@ class BadRequest(Exception):
         return "PYTEST_CURRENT_TEST" in os.environ
 
     def __str__(self):
-        return self.message
+        return self.message

ibm_watsonx_orchestrate/utils/tokens.py

@@ -0,0 +1,51 @@
+from datetime import timezone, datetime
+from urllib.parse import urlparse
+
+import jwt
+
+from ibm_watsonx_orchestrate.cli.commands.environment.types import EnvironmentAuthType
+from ibm_watsonx_orchestrate.client.client import Client
+from ibm_watsonx_orchestrate.client.client_errors import ClientError
+from ibm_watsonx_orchestrate.client.credentials import Credentials
+from ibm_watsonx_orchestrate.utils.environment import EnvSettingsService
+
+
+class CpdWxOTokenService:
+
+    def __init__ (self, env_settings: EnvSettingsService):
+        self.__token = None
+        self.__token_exp_at = None
+        self.__env_settings = env_settings
+
+    def get_token (self) -> str:
+        if self.__token is None or self.__has_token_expired():
+            self.__refresh_token()
+
+        return self.__token
+
+    def __has_token_expired (self) -> bool:
+        return self.__token is None or self.__token_exp_at is None or datetime.now(
+            timezone.utc).timestamp() >= self.__token_exp_at
+
+    def __refresh_token (self) -> None:
+        try:
+            self.__token = None
+            self.__token_exp_at = None
+
+            creds = Credentials(
+                url=self.__env_settings.get_wo_instance_url(),
+                api_key=self.__env_settings.get_wo_api_key(),
+                username=self.__env_settings.get_wo_username(),
+                password=self.__env_settings.get_wo_password(),
+                iam_url=None,
+                auth_type=EnvironmentAuthType.CPD
+            )
+
+            client = Client(creds)
+            self.__token = client.token
+
+            decoded_token = jwt.decode(self.__token, options={"verify_signature": False})
+            self.__token_exp_at = decoded_token.get("exp")
+
+        except ClientError as ex:
+            raise ClientError(ex)

ibm_watsonx_orchestrate/utils/utils.py

@@ -1,7 +1,7 @@
 import re
 import zipfile
 import yaml
-from typing import BinaryIO
+from typing import BinaryIO, Any, Tuple
 
 # disables the automatic conversion of date-time objects to datetime objects and leaves them as strings
 yaml.constructor.SafeConstructor.yaml_constructors[u'tag:yaml.org,2002:timestamp'] = \
@@ -11,8 +11,67 @@ def yaml_safe_load(file : BinaryIO) -> dict:
     return yaml.safe_load(file)
 
 def sanitize_app_id(app_id: str) -> str:
-    sanatize_pattern = re.compile(r"[^a-zA-Z0-9]+")
-    return re.sub(sanatize_pattern,'_', app_id)
+    sanitize_pattern = re.compile(r"[^a-zA-Z0-9]+")
+    return re.sub(sanitize_pattern,'_', app_id)
+
+def sanitize_catalog_label(label: str) -> str:
+    sanitize_pattern = re.compile(r"[^a-zA-Z0-9]+")
+    return re.sub(sanitize_pattern,'_', label)
 
 def check_file_in_zip(file_path: str, zip_file: zipfile.ZipFile) -> bool:
-    return any(x.startswith("%s/" % file_path.rstrip("/")) for x in zip_file.namelist())
+    return any(x.startswith("%s/" % file_path.rstrip("/")) for x in zip_file.namelist())
+
+def parse_bool_safe (value, fallback = False) -> bool:
+    if value is not None:
+        if isinstance(value, bool):
+            return value
+
+        elif isinstance(value, str):
+            value = value.lower().strip()
+            if value in ("yes", "true", "t", "1"):
+                return True
+
+            elif value in ("no", "false", "f", "0"):
+                return False
+
+        elif value in (0, 1):
+            return parse_bool_safe(str(value), fallback)
+
+    return fallback
+
+def parse_bool_safe_and_get_raw_val (value, fallback: bool = False) -> Tuple[bool, Any | None]:
+    if value is not None:
+        if isinstance(value, bool):
+            return value, None
+
+        elif isinstance(value, str):
+            value_str = value.lower().strip()
+            if value_str in ("yes", "true", "t", "1"):
+                return True, None
+
+            elif value_str in ("no", "false", "f", "0"):
+                return False, None
+
+        elif value in (0, 1):
+            return parse_bool_safe_and_get_raw_val(str(value), fallback)
+
+    return fallback, value
+
+def parse_int_safe (value, base: int = 10, fallback: int | None = None) -> int:
+    if value is not None:
+        if isinstance(value, int):
+            return value
+
+        elif isinstance(value, float):
+            return int(value)
+
+        elif isinstance(value, str):
+            value = value.strip()
+
+            try:
+                return int(value, base)
+
+            except ValueError as ex:
+                pass
+
+    return fallback

{ibm_watsonx_orchestrate-1.12.0b0.dist-info → ibm_watsonx_orchestrate-1.13.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ibm-watsonx-orchestrate
-Version: 1.12.0b0
+Version: 1.13.0b0
 Summary: IBM watsonx.orchestrate SDK
 Author-email: IBM <support@ibm.com>
 License: MIT License
@@ -11,7 +11,7 @@ Requires-Dist: click<8.2.0,>=8.0.0
 Requires-Dist: docstring-parser<1.0,>=0.16
 Requires-Dist: httpx<1.0.0,>=0.28.1
 Requires-Dist: ibm-cloud-sdk-core>=3.24.2
-Requires-Dist: ibm-watsonx-orchestrate-evaluation-framework==1.1.2
+Requires-Dist: ibm-watsonx-orchestrate-evaluation-framework==1.1.3
 Requires-Dist: jsonref==1.1.0
 Requires-Dist: langchain-core<=0.3.63
 Requires-Dist: langsmith<=0.3.45