oracle-ads 2.13.9rc0__py3-none-any.whl → 2.13.10rc0__py3-none-any.whl

ads/opctl/backend/marketplace/marketplace_utils.py

@@ -0,0 +1,212 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import os
+import oci
+from typing import List, Dict
+from ads.opctl.backend.marketplace.models.bearer_token import BearerToken
+from ads.common.oci_client import OCIClientFactory
+from ads.common import auth as authutil
+from ads.opctl.backend.marketplace.models.marketplace_type import (
+    HelmMarketplaceListingDetails,
+)
+
+
+class Color:
+    PURPLE = "\033[95m"
+    CYAN = "\033[96m"
+    DARKCYAN = "\033[36m"
+    BLUE = "\033[94m"
+    GREEN = "\033[92m"
+    YELLOW = "\033[93m"
+    RED = "\033[91m"
+    BOLD = "\033[1m"
+    UNDERLINE = "\033[4m"
+    END = "\033[0m"
+
+
+WARNING = f"{Color.RED}{Color.BOLD}WARNING: {Color.END}"
+
+
+class StatusIcons:
+    CHECK = "\u2705 "
+    CROSS = "\u274C "
+    LOADING = "\u274d "
+    TADA = "\u2728 "
+
+
+def print_ticker(text: str, iteration: int):
+    ticker = ["/", "-", "\\"]
+    text = f"{text} [{ticker[iteration % len(ticker)]}]"
+    end = "\n"
+    try:
+        size = os.get_terminal_size()
+        backspaces_needed = int((len(text) + 1) / size.columns) + 1
+    except OSError:
+        end = "\r"
+    else:
+        if iteration != 0:
+            print(f"\033[{backspaces_needed}F", end="", flush=True)
+
+    print(text, end=end, flush=True)
+
+
+def print_heading(
+    heading: str,
+    prefix_newline_count: int = 0,
+    suffix_newline_count: int = 0,
+    colors: List[str] = (Color.BOLD,),
+) -> None:
+    colors = [f"{color}" for color in colors]
+    colors = " ".join(colors)
+    print(
+        f"{colors}"
+        + "\n" * prefix_newline_count
+        + "*" * 30
+        + heading
+        + "*" * 30
+        + "\n" * suffix_newline_count
+        + f"{Color.END}"
+    )
+
+
+def set_kubernetes_session_token_env(profile: str = "DEFAULT") -> None:
+    os.environ["OCI_CLI_AUTH"] = "security_token"
+    os.environ["OCI_CLI_PROFILE"] = profile
+
+
+def get_marketplace_client() -> oci.marketplace.MarketplaceClient:
+    return OCIClientFactory(**authutil.default_signer()).marketplace
+
+
+def get_marketplace_composite_client() -> (
+    oci.marketplace.MarketplaceClientCompositeOperations
+):
+    return oci.marketplace.MarketplaceClientCompositeOperations(
+        client=get_marketplace_client()
+    )
+
+
+def get_docker_bearer_token(ocir_repo: str) -> BearerToken:
+    def get_ocir_url(repo: str):
+        repo = repo.lstrip("https://")
+        repo = repo.rstrip("/")
+        repo = f"https://{repo}/20180419"
+        return repo
+
+    token_client: oci.BaseClient = OCIClientFactory(
+        **authutil.default_signer(
+            client_kwargs={
+                "service": "docker",
+                "service_endpoint": get_ocir_url(ocir_repo),
+                "type_mapping": {"SecurityToken": BearerToken},
+            },
+        )
+    ).create_client(oci.BaseClient)
+    token: BearerToken = token_client.call_api(
+        resource_path="/docker/token", method="GET", response_type="SecurityToken"
+    ).data
+    return token
+
+
+def _export_helm_chart_(listing_details: HelmMarketplaceListingDetails):
+    client = get_marketplace_client()
+    export_listing_work_request: oci.marketplace.models.WorkRequest = (
+        client.export_listing(
+            listing_id=listing_details.listing_id,
+            package_version=listing_details.version,
+            export_package_details=oci.marketplace.models.ExportPackageDetails(
+                compartment_id=listing_details.compartment_id,
+                container_repository_path=listing_details.ocir_image_path,
+            ),
+        ).data
+    )
+
+    export_listing_work_request = oci.wait_until(
+        client,
+        client.get_work_request(export_listing_work_request.id),
+        evaluate_response=lambda r: getattr(r.data, "status")
+        and getattr(r.data, "status").lower() in ["succeeded", "failed"],
+        wait_callback=lambda times_checked, _: print_ticker(
+            "Waiting for marketplace export to finish", iteration=times_checked - 1
+        ),
+        max_interval_seconds=1,
+    ).data
+    if export_listing_work_request.status == "FAILED":
+        print(f"Couldn't export images from marketplace to OCIR {StatusIcons.CROSS}")
+        # TODO: Raise proper exception
+        raise Exception
+    else:
+        print(
+            f"Images were successfully exported to OCIR from marketplace {StatusIcons.CHECK}"
+        )
+    # Get the data from response
+
+
+def list_container_images(
+    compartment_id: str, ocir_image_path: str
+) -> oci.artifacts.models.ContainerImageCollection:
+    artifact_client = OCIClientFactory(**authutil.default_signer()).artifacts
+    list_container_images_response = artifact_client.list_container_images(
+        compartment_id=compartment_id,
+        sort_by="TIMECREATED",
+        repository_name=ocir_image_path,
+    )
+    return list_container_images_response.data
+
+
+def export_if_tags_not_exist(
+    listing_details: HelmMarketplaceListingDetails,
+) -> Dict[str, str]:
+    class ImageTagPatternNotFound(Exception):
+        def __init__(self, pattern: List[str]):
+            self.pattern = pattern
+
+        def __repr__(self):
+            print_heading(
+                f"Couldn't find images with tags: {listing_details.container_tag_pattern} requested by the operator.",
+                colors=[Color.RED],
+            )
+
+    tags_map = _get_tags_map_(listing_details)
+
+    if not tags_map:
+        _export_helm_chart_(listing_details)
+        tags_map = _get_tags_map_(listing_details)
+        if not tags_map:
+            raise ImageTagPatternNotFound(pattern=listing_details.container_tag_pattern)
+    else:
+        print(
+            f"Images already exist in the path. Continuing without export.{StatusIcons.CHECK}"
+        )
+    return tags_map
+
+
+def get_kubernetes_service(listings_details: HelmMarketplaceListingDetails):
+    import kubernetes
+
+    kubernetes.config.load_kube_config()
+    k8 = kubernetes.client.CoreV1Api()
+    return k8.list_namespaced_service(
+        namespace=listings_details.namespace,
+        label_selector=f"app.kubernetes.io/instance={listings_details.helm_app_name}",
+    )
+
+
+def _get_tags_map_(
+    listing_details: HelmMarketplaceListingDetails,
+) -> Dict[str, str]:
+    images = list_container_images(
+        compartment_id=listing_details.compartment_id,
+        ocir_image_path=listing_details.ocir_image_path,
+    )
+    tags_map = {}
+    for image in images.items:
+        for container_tag_pattern in listing_details.container_tag_pattern:
+            if container_tag_pattern in image.display_name:
+                tags_map[container_tag_pattern] = image.display_name.split(":")[1]
+                break
+    return tags_map

ads/opctl/backend/marketplace/models/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/

ads/opctl/backend/marketplace/models/bearer_token.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/from oci.decorators import init_model_state_from_kwargs
+
+from oci.decorators import init_model_state_from_kwargs
+from oci.util import formatted_flat_dict
+
+BEARER_TOKEN_USERNAME = "BEARER_TOKEN"
+
+
+@init_model_state_from_kwargs
+class BearerToken(object):
+    """
+    List container image results.
+    """
+
+    def __init__(self, **kwargs):
+        """
+        Initializes a new ContainerImageCollection object with values from keyword arguments.
+        The following keyword arguments are supported (corresponding to the getters/setters of this class):
+
+        :param items:
+            The value to assign to the items property of this ContainerImageCollection.
+        :type items: list[oci.artifacts.models.ContainerImageSummary]
+
+        :param remaining_items_count:
+            The value to assign to the remaining_items_count property of this ContainerImageCollection.
+        :type remaining_items_count: int
+
+        """
+        self.swagger_types = {
+            "token": "str",
+            "access_token": "str",
+            "expires_in": "int",
+            "scope": "str",
+        }
+
+        self.attribute_map = {
+            "token": "token",
+            "access_token": "access_token",
+            "expires_in": "expires_in",
+            "scope": "scope",
+        }
+
+        self._token = None
+        self._access_token = None
+        self._expires_in = None
+        self._scope = None
+
+    @property
+    def token(self):
+        return self._token
+
+    @token.setter
+    def token(self, token):
+        self._token = token
+
+    @property
+    def access_token(self):
+        return self._access_token
+
+    @access_token.setter
+    def access_token(self, access_token: str):
+        self._access_token = access_token
+
+    @property
+    def scope(self):
+        return self._scope
+
+    @scope.setter
+    def scope(self, scope: str):
+        self._scope = scope
+
+    @property
+    def expires_in(self):
+        return self._expires_in
+
+    @expires_in.setter
+    def expires_in(self, expires_in: int):
+        self._expires_in = expires_in
+
+    def __repr__(self):
+        return formatted_flat_dict(self)
+
+    def __eq__(self, other):
+        if other is None:
+            return False
+
+        return self.__dict__ == other.__dict__
+
+    def __ne__(self, other):
+        return not self == other

ads/opctl/backend/marketplace/models/marketplace_type.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from abc import ABC
+from enum import Enum
+from typing import List
+from ads.common.serializer import DataClassSerializable
+from ads.opctl.backend.marketplace.models.ocir_details import OCIRDetails
+
+
+class MarketplaceListingDetails(DataClassSerializable, ABC):
+    def __init__(
+        self,
+        listing_id: str,
+        compartment_id: str,
+        version: str,
+    ):
+        self.listing_id = listing_id
+        self.compartment_id = compartment_id
+        self.version = version
+
+
+class SecretStrategy(Enum):
+    PROMPT = "prompt"
+    AUTOMATIC = "automatic"
+
+
+class HelmMarketplaceListingDetails(MarketplaceListingDetails):
+    def __init__(
+        self,
+        listing_id: str,
+        compartment_id: str,
+        marketplace_version: str,
+        ocir_details: OCIRDetails,
+        helm_chart_tag: str,
+        image_tag_pattern: List[str],
+        helm_values: dict,
+        helm_app_name: str,
+        namespace: str,
+        docker_registry_secret: str,
+        secret_strategy: SecretStrategy,
+    ):
+        super().__init__(listing_id, compartment_id, marketplace_version)
+        self.ocir_details = ocir_details
+        self.compartment_id = compartment_id
+        self.helm_values = helm_values
+        self.helm_chart_tag = helm_chart_tag
+        self.container_tag_pattern = image_tag_pattern
+        self.helm_app_name = helm_app_name
+        self.namespace = namespace
+        self.docker_registry_secret = docker_registry_secret
+        self.secret_strategy = secret_strategy
+
+    @property
+    def ocir_url(self):
+        return self.ocir_details.ocir_url
+
+    @property
+    def helm_fully_qualified_url(self):
+        return f"oci://{self.ocir_url}"
+
+    @property
+    def ocir_registry(self):
+        return self.ocir_details.ocir_region_url
+
+    @property
+    def ocir_image_path(self):
+        return self.ocir_details.path_in_tenancy

ads/opctl/backend/marketplace/models/ocir_details.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from typing import List
+
+
+class OCIRDetails:
+    class InvalidOCIRUrlException(Exception):
+        def __init__(self, ocir_url: str):
+            self.ocir_url = ocir_url
+            pass
+
+        def __repr__(self):
+            print(
+                f"OCIR URL: {self.ocir_url}"
+                + " is invalid. It should be of the pattern {region}.ocir.io/{tenancy_namespace}/{repository_path}"
+            )
+
+    def __init__(self, ocir_url: str):
+        self._ocir_url: str = ocir_url.rstrip("/")
+        ocir_arr: List[str] = ocir_url.split("/")
+        if len(ocir_arr) < 3:
+            raise self.InvalidOCIRUrlException(ocir_url=ocir_url)
+        else:
+            self._ocir_region_url = ocir_arr[0]
+            self._namespace = ocir_arr[1]
+            self._path_in_tenancy = "/".join(ocir_arr[2:])
+            self._image = ocir_arr[-1]
+            self._repository_url = "/".join(ocir_arr[:-1])
+
+    @property
+    def ocir_region_url(self) -> str:
+        return self._ocir_region_url
+
+    @property
+    def path_in_tenancy(self) -> str:
+        return self._path_in_tenancy
+
+    @property
+    def namespace(self) -> str:
+        return self._namespace
+
+    @property
+    def ocir_url(self) -> str:
+        return self._ocir_url
+
+    @property
+    def image(self) -> str:
+        return self._image
+
+    @property
+    def repository_url(self) -> str:
+        return self._repository_url

ads/opctl/backend/marketplace/prerequisite_checker.py

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import base64
+import json
+import re
+import subprocess
+import time
+import webbrowser
+from typing import List
+
+import click
+from ads.opctl.backend.marketplace.models.bearer_token import BEARER_TOKEN_USERNAME
+
+from ads.opctl import logger
+
+from ads.opctl.backend.marketplace.models.marketplace_type import (
+    MarketplaceListingDetails,
+    HelmMarketplaceListingDetails,
+    SecretStrategy,
+)
+import oci.marketplace.models as models
+
+from ads.opctl.backend.marketplace.marketplace_utils import (
+    get_marketplace_client,
+    Color,
+    StatusIcons,
+    get_docker_bearer_token,
+    print_heading,
+    WARNING,
+)
+
+DOCKER_SECRET_TYPE = "kubernetes.io/dockerconfigjson"
+
+
+class BinaryValidation:
+    def __init__(self, binary: str, installation_path: str):
+        self.binary = binary
+        self.installation_path = installation_path
+
+
+def get_highlighted_text(text: str, colors: List[Color] = (Color.BLUE,)) -> str:
+    colors = [f"{color}" for color in colors]
+    colors = " ".join(colors)
+    return f"{colors}'{text}'{Color.END}"
+
+
+def check_prerequisites(listing_details: MarketplaceListingDetails):
+    print_heading(f"Checking Prerequisites", colors=[Color.BLUE, Color.BOLD])
+    _check_license_for_listing_(listing_details)
+    if isinstance(listing_details, HelmMarketplaceListingDetails):
+        _check_binaries_(
+            [
+                BinaryValidation("helm", "https://helm.sh/docs/intro/install/"),
+                BinaryValidation("kubectl", "https://kubernetes.io/docs/tasks/tools/"),
+            ]
+        )
+        _prompt_kubernetes_confirmation_()
+        _check_kubernetes_secret_(listing_details)
+    print_heading(
+        f"Completed prerequisites check {StatusIcons.TADA}",
+        colors=[Color.BLUE, Color.BOLD],
+    )
+
+
+def _prompt_kubernetes_confirmation_():
+    print(
+        f"{WARNING}{Color.RED}Please ensure that kubectl is connected to the exact cluster where the listing needs to be deployed. Failure to do so can lead to unintended consequences.{Color.END}"
+    )
+    click.confirm(
+        text="Is it safe to proceed?",
+        default=False,
+        abort=True,
+    )
+
+
+def _check_kubernetes_secret_(listing_details: HelmMarketplaceListingDetails):
+    from kubernetes.client import V1SecretList, V1Secret, V1ObjectMeta
+
+    import kubernetes
+
+    print(
+        f"Starting docker registry secret verification for secret: {get_highlighted_text(listing_details.docker_registry_secret)} in namespace: {get_highlighted_text(listing_details.namespace)}"
+    )
+    create_secret = True
+    kubernetes.config.load_kube_config()
+    v1 = kubernetes.client.CoreV1Api()
+    secrets: V1SecretList = v1.list_namespaced_secret(
+        namespace=listing_details.namespace
+    )
+    v1.list_namespaced_service(
+        namespace="feature-store",
+    )
+    secret_strategy: SecretStrategy = listing_details.secret_strategy
+    for secret in secrets.items:
+        secret: V1Secret = secret
+        metadata: V1ObjectMeta = secret.metadata
+        if (
+            secret.type == DOCKER_SECRET_TYPE
+            and metadata.name == listing_details.docker_registry_secret
+        ):
+            annotations: dict = metadata.annotations
+            # Check expiry date with 3 minutes buffer to accommodate for pull time
+            if int(annotations.get("expiry")) > (time.time() + 60 * 3):
+                print(
+                    f"Secret {get_highlighted_text(listing_details.docker_registry_secret)} exists in namespace {get_highlighted_text(listing_details.namespace)} {StatusIcons.CHECK}"
+                )
+                return
+            else:
+                print(
+                    f"Operator generated docker secret {get_highlighted_text(listing_details.docker_registry_secret)} has expired.\nRecreating secret using bearer token"
+                )
+                v1.delete_namespaced_secret(
+                    name=metadata.name, namespace=listing_details.namespace
+                )
+                secret_strategy = SecretStrategy.AUTOMATIC
+
+    if secret_strategy == SecretStrategy.PROMPT:
+        create_secret = click.confirm(
+            text=f"Docker registry secret: {listing_details.docker_registry_secret} wasn't found in namespace: {listing_details.namespace}. Do you want to allow operator to automatically try to fix the issue by setting up bearer token authentication?",
+            default=True,
+            abort=True,
+        )
+    if create_secret:
+        username = BEARER_TOKEN_USERNAME
+        token = get_docker_bearer_token(ocir_repo=listing_details.ocir_registry)
+        auth = base64.b64encode(f"{username}:{token.token}".encode("utf-8")).decode(
+            "utf-8"
+        )
+        docker_config_dict = {
+            "auths": {
+                listing_details.ocir_registry: {
+                    "username": username,
+                    "password": token.token,
+                    "auth": auth,
+                }
+            }
+        }
+        docker_config = base64.b64encode(
+            json.dumps(docker_config_dict).encode("utf-8")
+        ).decode("utf-8")
+        secret = kubernetes.client.models.V1Secret(
+            type=DOCKER_SECRET_TYPE,
+            metadata=kubernetes.client.models.V1ObjectMeta(
+                name=listing_details.docker_registry_secret,
+                namespace=listing_details.namespace,
+                annotations={"expiry": str(token.expires_in + int(time.time()))},
+            ),
+            data={".dockerconfigjson": docker_config},
+        )
+        response = v1.create_namespaced_secret(
+            namespace=listing_details.namespace, body=secret
+        )
+        print(
+            f"Successfully created secret {get_highlighted_text(listing_details.docker_registry_secret)} in namespace {get_highlighted_text(listing_details.namespace)} {StatusIcons.CHECK}"
+        )
+
+
+def _check_binaries_(binaries: List[BinaryValidation]):
+    for binary in binaries:
+        result = subprocess.run(["which", f"{binary.binary}"], capture_output=True)
+        if result.returncode == 0:
+            print(f"{binary.binary.capitalize()} is present {StatusIcons.CHECK}")
+        else:
+            print(
+                f"{binary.binary} is not present in PATH {StatusIcons.CROSS}.\nFor installation instructions please check: {binary.installation_path}"
+            )
+            raise Exception
+
+
+def _check_license_for_listing_(listing_details: MarketplaceListingDetails):
+    compartment_id = listing_details.compartment_id
+    package_version = listing_details.version
+    listing_id = listing_details.listing_id
+    marketplace = get_marketplace_client()
+    listing: models.Listing = marketplace.get_listing(
+        listing_id=listing_id, compartment_id=compartment_id
+    ).data
+    logger.debug(f"Checking license agreements for listing: {listing.name}")
+    accepted_agreements: List[
+        models.AcceptedAgreementSummary
+    ] = marketplace.list_accepted_agreements(
+        listing_id=listing_id,
+        compartment_id=compartment_id,
+    ).data
+
+    agreement_summaries: List[models.AgreementSummary] = marketplace.list_agreements(
+        listing_id=listing_id,
+        package_version=package_version,
+        compartment_id=listing_details.compartment_id,
+    ).data
+
+    accepted_agreement_ids: List[str] = []
+    for accepted_agreement in accepted_agreements:
+        accepted_agreement_ids.append(accepted_agreement.agreement_id)
+
+    for agreement_summary in agreement_summaries:
+        if agreement_summary.id not in accepted_agreement_ids:
+            agreement: models.Agreement = marketplace.get_agreement(
+                listing_id=listing_id,
+                package_version=package_version,
+                agreement_id=agreement_summary.id,
+                compartment_id=listing_details.compartment_id,
+            ).data
+            print(
+                f"Agreement from {get_highlighted_text(agreement.author)} with id: {agreement_summary.id} is not accepted. \
+                Opening terms and conditions in default browser {StatusIcons.LOADING}"
+            )
+            webbrowser.open(agreement.content_url)
+            time.sleep(1)
+            answer = click.confirm(
+                f'{re.sub("<.*?>", "", agreement.prompt)}',
+                default=False,
+            )
+            if not answer:
+                print_heading(
+                    f"Agreement from author: {get_highlighted_text(agreement_summary.author)} with id: {agreement_summary.id} "
+                    f"is rejected {StatusIcons.CROSS} "
+                )
+                raise Exception
+            else:
+                create_accepted_agreement_details: models.CreateAcceptedAgreementDetails = (
+                    models.CreateAcceptedAgreementDetails()
+                )
+                create_accepted_agreement_details.agreement_id = agreement_summary.id
+                create_accepted_agreement_details.listing_id = listing_id
+                create_accepted_agreement_details.compartment_id = compartment_id
+                create_accepted_agreement_details.signature = agreement.signature
+                create_accepted_agreement_details.package_version = package_version
+                marketplace.create_accepted_agreement(create_accepted_agreement_details)
+
+        print(
+            f"Agreement from author: {get_highlighted_text(agreement_summary.author)} with id: {agreement_summary.id} is accepted "
+            f"{StatusIcons.CHECK} "
+        )