hcs-core 0.1.250__py3-none-any.whl → 0.1.316__py3-none-any.whl

hcs_core/sglib/client_util.py

@@ -13,78 +13,228 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-from typing import Callable
+import json
+import logging
+import sys
+import threading
 import time
-from hcs_core.ctxp import profile, panic, CtxpException
-from hcs_core.sglib import hcs_client
-from hcs_core.util.query_util import with_query, PageRequest
-from hcs_core.util import duration
-from hcs_core.util import exit
+from typing import Callable, Iterator
 
+from hcs_core.ctxp import CtxpException, panic, profile
+from hcs_core.sglib.ez_client import EzClient
+from hcs_core.util import duration, exit
+from hcs_core.util.query_util import PageRequest, with_query
 
-def hdc_service_client(service_name: str):
-    url = profile.current().hcs.url
-    if not url.endswith("/"):
-        url += "/"
-    url += service_name
-    return hcs_client(url)
+log = logging.getLogger(__name__)
 
+_caches = {}
 
-def _get_region_url(region_name: str):
+_client_instance_lock = threading.RLock()
+
+
+def _get_service_override(service_name: str):
+    profile_data = profile.current()
+    override = profile_data.get("override", {})
+    service_override = {}
+    for k, v in override.items():
+        if service_name == k.lower():
+            service_override = v
+
+    if not service_override:
+        if service_name == "org-service":
+            service_override = override.get("org", {})
+        elif service_name.find("-") >= 0:
+            camel_name = "".join(word.capitalize() for word in service_name.split("-"))
+            camel_name = camel_name[0].lower() + camel_name[1:]
+            service_override = override.get(camel_name, {})
+    return service_override
+
+
+def _lazy_init(service_name: str, hdc: str = None, region: str = None):  # make it deferred so no need to initialize profile
+    if region and hdc:
+        raise Exception("region and hdc cannot be specified at the same time.")
+
+    profile_data = profile.current()
+
+    service_override = _get_service_override(service_name)
+    service_override_url = service_override.get("url")
+    if service_override_url:
+        log.debug(f"Using per-service override for {service_name}: {service_override_url}")
+        url = service_override_url
+    else:
+        if hdc:
+            # prod only
+            hdc = hdc.upper()
+            if hdc == "US":
+                url = profile_data.hcs.url
+            elif hdc == "EU":
+                url = profile_data.hcs.url.replace("cloud-sg-us", "cloud-sg-eu")
+            elif hdc == "JP":
+                url = profile_data.hcs.url.replace("cloud-sg-us", "cloud-sg-jp")
+            else:
+                raise CtxpException(f"Invalid HDC name: {hdc}. Supported HDC names: US, EU, JP.")
+        elif region:
+            url = _get_region_url(region)
+            if not url:
+                panic("Missing profile property: hcs.regions")
+        else:
+            url = profile_data.hcs.url
+    url = url.rstrip("/") + "/" + service_name
+
+    # TODO
+    client_id = service_override.get("client-id", None)
+    if not client_id:
+        client_id = service_override.get("clientId", None)
+    client_secret = service_override.get("client-secret", None)
+    if not client_secret:
+        client_secret = service_override.get("clientSecret", None)
+    api_token = service_override.get("api-token", None)
+    if not api_token:
+        api_token = service_override.get("apiToken", None)
+    provider = service_override.get("provider", "vmwarecsp")
+    if provider == "vmwarecsp":
+        token_url = profile_data.csp.url
+    elif provider == "auth-service":
+        token_url = profile_data.auth["tokenUrl"]
+    else:
+        raise CtxpException(f"Unknown provider: {provider}. Supported providers: vmwarecsp, auth-service.")
+
+    if client_id:
+        if not client_secret:
+            panic(f"Client ID is specified but missing clientSecret for service {service_name} in profile override.")
+    else:
+        if client_secret:
+            panic(f"Client secret is specified but missing clientId for service {service_name} in profile override.")
+
+    if client_id:
+        auth_config = {
+            "url": token_url,
+            "client_id": client_id,
+            "client_secret": client_secret,
+        }
+    elif api_token:
+        auth_config = {
+            "url": token_url,
+            "api_token": api_token,
+        }
+    else:
+        auth_config = None
+
+    from hcs_core.sglib import auth
+
+    oauth_client = auth.oauth_client(auth_config=auth_config)
+    return url, oauth_client
+
+
+def hdc_service_client(service_name: str, hdc: str = None) -> EzClient:
+    """A client for HDC service. Cached per service name, thread-safe, lazy initialization."""
+    if hdc:
+        hdc = hdc.upper()
+        if hdc not in ["US", "EU", "JP"]:
+            panic(f"Invalid HDC name: {hdc}. Supported HDC names: US, EU, JP.")
+    else:
+        hdc = "US"
+
+    k = f"{service_name}#{hdc}"
+    with _client_instance_lock:
+        instance = _caches.get(k)
+        if not instance:
+            instance = EzClient(lazy_init=lambda: _lazy_init(service_name=service_name, hdc=hdc))
+            _caches[k] = instance
+        return instance
+
+
+def _get_region_url(region: str):
     regions = profile.current().hcs.regions
-    if not region_name:
+    if not region:
         return regions[0].url
     for r in regions:
-        if r.name.lower() == region_name.lower():
+        if r.name.lower() == region.lower():
             return r.url
     names = []
     for r in regions:
         names.append(r.name)
-    panic(f"Region not found: {region_name}. Available regions: {names}")
+    panic(f"Region not found: {region}. Available regions: {names}")
 
 
-def regional_service_client(region_name: str, service_name: str):
+def regional_service_client(service_name: str, region: str = None):
     # 'https://dev1b-westus2-cp103a.azcp.horizon.vmware.com/vmhub'
-    url = _get_region_url(region_name)
-    if not url:
-        panic("Missing profile property: hcs.regions")
-    if not url.endswith("/"):
-        url += "/"
-    url += service_name
-    return hcs_client(url)
+    region_names = [r.name.lower() for r in profile.current().hcs.regions]
+    if region:
+        region = region.lower()
+        if region not in region_names:
+            panic(f"Invalid region name: {region}. Available regions: {', '.join(region_names)}")
+    else:
+        region = region_names[0]
 
+    k = f"{service_name}#{region}"
+    with _client_instance_lock:
+        instance = _caches.get(k)
+        if not instance:
+            instance = EzClient(lazy_init=lambda: _lazy_init(service_name=service_name, region=region))
+            _caches[k] = instance
+        return instance
 
-def _with_org_id(url: str, org_id: str):
-    if org_id:
-        if url.find("?") < 0:
-            url += "?"
-        url += "org_id=" + org_id
-    return url
+
+def is_regional_service(service_name: str):
+    regional_services = ["vmhub", "connection-service"]
+    return service_name in regional_services
+
+
+def service_client(service_name: str, region: str = None, hdc: str = None):
+    if is_regional_service(service_name):
+        return regional_service_client(service_name, region)
+    else:
+        return hdc_service_client(service_name, hdc)
 
 
 class default_crud:
     def __init__(self, client, base_context: str, resource_type_name: str):
-        self._client = client
+        self._client_impl = client
         self._base_context = base_context
         self._resource_type_name = resource_type_name
 
+    def _client(self):
+        if callable(self._client_impl):
+            self._client_impl = self._client_impl()
+        elif isinstance(self._client_impl, str):
+            self._client_impl = hdc_service_client(self._client_impl)
+        else:
+            pass
+        if isinstance(self._client_impl, EzClient):
+            return self._client_impl
+        raise CtxpException(f"Invalid client implementation: {self._client_impl}")
+
     def get(self, id: str, org_id: str, **kwargs):
         if org_id:
             kwargs["org_id"] = org_id
+            kwargs["orgId"] = org_id
         url = with_query(f"{self._base_context}/{id}", **kwargs)
         # print(url)
-        return self._client.get(url)
+        return self._client().get(url)
 
-    def list(self, org_id: str, **kwargs):
+    def list(self, org_id: str, fn_filter: Callable = None, **kwargs) -> list:
         if org_id:
             kwargs["org_id"] = org_id
+            kwargs["orgId"] = org_id
 
         def _get_page(query_string):
             url = self._base_context + "?" + query_string
             # print(url)
-            return self._client.get(url)
+            return self._client().get(url)
+
+        return PageRequest(_get_page, fn_filter, **kwargs).get()
+
+    def items(self, org_id: str, fn_filter: Callable = None, **kwargs) -> Iterator:
+        if org_id:
+            kwargs["org_id"] = org_id
+            kwargs["orgId"] = org_id
+
+        def _get_page(query_string):
+            url = self._base_context + "?" + query_string
+            return self._client().get(url)
 
-        return PageRequest(_get_page, **kwargs).get()
+        return PageRequest(_get_page, fn_filter, **kwargs).items()
 
     def create(self, payload: dict, headers: dict = None, **kwargs):
         url = with_query(f"{self._base_context}", **kwargs)
@@ -92,32 +242,37 @@ class default_crud:
         # import json
         # print(json.dumps(payload, indent=4))
         if isinstance(payload, str):
-            return self._client.post(url, text=payload, headers=headers)
+            return self._client().post(url, text=payload, headers=headers)
         if isinstance(payload, dict):
-            return self._client.post(url, json=payload, headers=headers)
-        return self._client.post(url, json=payload, headers=headers)
+            return self._client().post(url, json=payload, headers=headers)
+        return self._client().post(url, json=payload, headers=headers)
 
     def upload(self, files, **kwargs):
         url = with_query(f"{self._base_context}", **kwargs)
-        return self._client.post(url, files=files)
+        return self._client().post(url, files=files)
 
     def delete(self, id: str, org_id: str, **kwargs):
         if org_id:
             kwargs["org_id"] = org_id
+            kwargs["orgId"] = org_id
         url = with_query(f"{self._base_context}/{id}", **kwargs)
         # print(url)
-        return self._client.delete(url)
+        return self._client().delete(url)
 
     def wait_for_deleted(self, id: str, org_id: str, timeout: str, fn_is_error: Callable = None):
         name = self._resource_type_name + "/" + id
-        fn_get = lambda: self.get(id, org_id)
+
+        def fn_get():
+            return self.get(id, org_id)
+
         return wait_for_res_deleted(name, fn_get, timeout=timeout, fn_is_error=fn_is_error)
 
     def update(self, id: str, org_id: str, data: dict, **kwargs):
         if org_id:
             kwargs["org_id"] = org_id
+            kwargs["orgId"] = org_id
         url = with_query(f"{self._base_context}/{id}")
-        return self._client.patch(url, data)
+        return self._client().patch(url, data)
 
 
 def _parse_timeout(timeout: str):
@@ -133,10 +288,13 @@ def wait_for_res_deleted(
     resource_name: str,
     fn_get: Callable,
     timeout: str,
-    polling_interval_seconds: int = 10,
+    polling_interval: int = 10,
     fn_is_error: Callable = None,
 ):
     timeout_seconds = _parse_timeout(timeout)
+    polling_interval_seconds = _parse_timeout(polling_interval)
+    if polling_interval_seconds < 3:
+        polling_interval_seconds = 3
     start = time.time()
     while True:
         t = fn_get()
@@ -155,9 +313,10 @@ def wait_for_res_deleted(
         sleep_seconds = remaining_seconds
         if sleep_seconds > polling_interval_seconds:
             sleep_seconds = polling_interval_seconds
-        exit.sleep(sleep_seconds)
+        time.sleep(sleep_seconds)
 
 
+# flake8: noqa=E731
 def wait_for_res_status(
     resource_name: str,
     fn_get: Callable,
@@ -181,21 +340,25 @@ def wait_for_res_status(
         field_name = get_status
         get_status = lambda t: t[field_name]
     if status_map:
-        if isinstance(status_map["ready"], str):
-            status_map["ready"] = [status_map["ready"]]
-        if isinstance(status_map["transition"], str):
-            status_map["transition"] = [status_map["transition"]]
-        if isinstance(status_map["error"], str):
-            status_map["error"] = [status_map["error"]]
         if is_ready:
             raise CtxpException("Can not specify is_ready when status_map is provided.")
         if is_error:
             raise CtxpException("Can not specify is_error when status_map is provided.")
         if is_transition:
             raise CtxpException("Can not specify is_transition when status_map is provided.")
-        is_ready = lambda s: s in status_map["ready"]
-        is_error = lambda s: s in status_map["error"]
-        is_transition = lambda s: s in status_map["transition"]
+
+        ready_status = status_map["ready"]
+        error_status = status_map["error"]
+        transition_status = status_map["transition"]
+        if isinstance(ready_status, str):
+            ready_status = [ready_status]
+        if isinstance(error_status, str):
+            error_status = [error_status]
+        if isinstance(transition_status, str):
+            transition_status = [transition_status]
+        is_ready = lambda s: s in ready_status
+        is_error = lambda s: error_status is None or s in error_status
+        is_transition = lambda s: transition_status is None or s in transition_status
     else:
         if not is_ready:
             raise CtxpException("Either status_map or is_ready must be specified.")
@@ -203,6 +366,9 @@ def wait_for_res_status(
             raise CtxpException("Either status_map or is_error must be specified.")
         if not is_transition:
             raise CtxpException("Either status_map or is_transition must be specified.")
+        ready_status = None
+        error_status = None
+        transition_status = None
 
     while True:
         t = fn_get()
@@ -211,26 +377,28 @@ def wait_for_res_status(
                 return
             raise CtxpException(prefix + "Not found.")
         status = get_status(t)
-        # print(f"DDD - status {status}")
         if is_error(status):
-            msg = prefix + f"Status error. Actual={status}"
-            if status_map:
-                msg += f", expected={status_map['ready']}"
+            msg = prefix + f"Unexpected terminal state. Actual={status}"
+            if ready_status:
+                msg += f", expected={ready_status}"
+            print("-- DUMP START --", file=sys.stderr)
+            print(json.dumps(t, indent=4), file=sys.stderr)
+            print("-- DUMP END --", file=sys.stderr)
             raise CtxpException(msg)
         if is_ready(status):
             return t
         if not is_transition(status):
-            raise CtxpException(
-                prefix + f"Unexpected status: {status}. If this is a transition, add it to status_map['transition']."
-            )
+            raise CtxpException(prefix + f"Unexpected status: {status}. If this is a transition, add it to status_map['transition'].")
 
         now = time.time()
         remaining_seconds = timeout_seconds - (now - start)
         if remaining_seconds < 1:
-            raise TimeoutError(prefix + "Timeout.")
+            msg = prefix + f"Timeout. Current: {status}."
+            if ready_status:
+                msg += f" Expected: {ready_status}."
+            raise TimeoutError(msg)
         sleep_seconds = remaining_seconds
         if sleep_seconds > polling_interval_seconds:
             sleep_seconds = polling_interval_seconds
 
-        # print(f"DDD sleeping {sleep_seconds}")
         exit.sleep(sleep_seconds)

hcs_core/sglib/csp.py

@@ -13,10 +13,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import httpx
 import json
 import logging
 
+import httpx
+
 log = logging.getLogger(__name__)
 
 log_http_details = False
@@ -64,8 +65,25 @@ def _log_response(response: httpx.Response):
     log.debug("\n")
 
 
+def _decode_http_basic_auth_token(basic_token: str):
+    import base64
+
+    try:
+        decoded = base64.b64decode(basic_token).decode("utf-8")
+        client_id, client_secret = decoded.split(":")
+        return client_id, client_secret
+    except Exception as e:
+        raise Exception(f"Invalid basic http auth token: {e}")
+
+
 class CspClient:
     def __init__(self, url: str, oauth_token: dict = None, org_id: str = None) -> None:
+        if url.endswith("/auth/v1/oauth/token"):
+            # To workaround that post with "" results "/" and fail the process.
+            url = url[: -len("/auth/v1/oauth/token")]
+            self._auth_mode = "hcs-auth-svc"
+        else:
+            self._auth_mode = "csp"
         self._base_url = url
         self._oauth_token = oauth_token
         self._org_id = org_id
@@ -80,6 +98,7 @@ class CspClient:
         )
 
     def login_with_api_token(self, api_token: str) -> dict:
+        log.debug(f"Logging in with api_token: {api_token}, url: {self._base_url}")
         # https://console-stg.cloud.vmware.com/csp/gateway/authn/api/swagger-ui.html#/Authentication/getAccessTokenByApiRefreshTokenUsingPOST
 
         # curl -X 'POST' \
@@ -87,19 +106,18 @@ class CspClient:
         # -H 'accept: application/json' \
         # -H 'Content-Type: application/x-www-form-urlencoded' \
         # -d 'refresh_token=<the-refresh-token>'
-
+        # print(f"Logging in with api_token: {api_token}, url: {self._base_url}")
         headers = {
             "Content-Type": "application/x-www-form-urlencoded",
             "Accept": "application/json",
         }
         # <no org id for this API>
-        resp = self._client.post(
-            "/csp/gateway/am/api/auth/api-tokens/authorize", headers=headers, data=f"api_token={api_token}"
-        )
+        resp = self._client.post("/csp/gateway/am/api/auth/api-tokens/authorize", headers=headers, data=f"api_token={api_token}")
         self._oauth_token = resp.json()
         return self._oauth_token
 
     def login_with_client_id_and_secret(self, client_id: str, client_secret: str, org_id: str) -> dict:
+        log.debug(f"Logging in with client_id: {client_id}, org_id: {org_id}, url: {self._base_url}")
         headers = {
             "Content-Type": "application/x-www-form-urlencoded",
             "Accept": "application/json",
@@ -109,8 +127,9 @@ class CspClient:
         }
         if org_id:
             params["orgId"] = org_id
+        url = "/auth/v1/oauth/token" if self._auth_mode == "hcs-auth-svc" else "/csp/gateway/am/api/auth/authorize"
         resp = self._client.post(
-            "/csp/gateway/am/api/auth/authorize",
+            url,
             auth=(client_id, client_secret),
             headers=headers,
             params=params,
@@ -118,6 +137,54 @@ class CspClient:
         self._oauth_token = resp.json()
         return self._oauth_token
 
+    @staticmethod
+    def create(
+        url: str,
+        org_id: str = None,
+        client_id: str = None,
+        client_secret: str = None,
+        api_token: str = None,
+        basic: str = None,
+        **kwargs,
+    ) -> "CspClient":
+        client = CspClient(url=url, org_id=org_id)
+
+        if not client_id:
+            client_id = kwargs.get("clientId")
+        if not client_secret:
+            client_secret = kwargs.get("clientSecret")
+        if not api_token:
+            api_token = kwargs.get("apiToken")
+
+        if client_id:
+            if not client_secret:
+                raise ValueError("client_secret is required when client_id is provided")
+            if api_token:
+                raise ValueError("api_token and client_id/client_secret cannot be used together")
+            if basic:
+                raise ValueError("basic auth and client_id/client_secret cannot be used together")
+            client.login_with_client_id_and_secret(client_id=client_id, client_secret=client_secret, org_id=org_id)
+        elif api_token:
+            if client_id or client_secret:
+                raise ValueError("api_token and client_id/client_secret cannot be used together")
+            if basic:
+                raise ValueError("api_token and basic auth cannot be used together")
+            client.login_with_api_token(api_token)
+        elif basic:
+            if client_id or client_secret:
+                raise ValueError("basic auth and client_id/client_secret cannot be used together")
+            if api_token:
+                raise ValueError("basic auth and api_token cannot be used together")
+            client_id, client_secret = _decode_http_basic_auth_token(basic)
+            client.login_with_client_id_and_secret(client_id=client_id, client_secret=client_secret, org_id=org_id)
+        else:
+            raise Exception("Unrecognized CSP authentication method.")
+
+        return client
+
+    def oauth_token(self) -> dict:
+        return self._oauth_token
+
     # def get_oauth_token(self, force=False):
     #     if self._oauth_token and not force:
     #         return self._oauth_token