qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/ocm_oidc_idp.py

@@ -1,198 +0,0 @@
-import logging
-import sys
-from collections.abc import (
-    Callable,
-    Iterable,
-    Sequence,
-)
-from typing import Any
-
-from reconcile import queries
-from reconcile.gql_definitions.ocm_oidc_idp.clusters import (
-    ClusterAuthOIDCV1,
-    ClusterV1,
-)
-from reconcile.gql_definitions.ocm_oidc_idp.clusters import query as cluster_query
-from reconcile.ocm.types import OCMOidcIdp
-from reconcile.status import ExitCodes
-from reconcile.utils import gql
-from reconcile.utils.disabled_integrations import integration_is_enabled
-from reconcile.utils.ocm import OCMMap
-from reconcile.utils.secret_reader import SecretReader
-
-QONTRACT_INTEGRATION = "ocm-oidc-idp"
-
-DEFAULT_EMAIL_CLAIMS: list[str] = ["email"]
-DEFAULT_NAME_CLAIMS: list[str] = ["name"]
-DEFAULT_USERNAME_CLAIMS: list[str] = ["preferred_username"]
-DEFAULT_GROUPS_CLAIMS: list[str] = []
-
-
-def fetch_current_state(
-    ocm_map: OCMMap, clusters: Iterable[ClusterV1]
-) -> list[OCMOidcIdp]:
-    """Fetch all current configured OIDC identity providers."""
-    current_state = []
-
-    for cluster in clusters:
-        ocm = ocm_map.get(cluster.name)
-        current_state += ocm.get_oidc_idps(cluster.name)
-
-    return current_state
-
-
-def fetch_desired_state(
-    secret_reader: SecretReader,
-    clusters: Iterable[ClusterV1],
-    vault_input_path: str,
-) -> list[OCMOidcIdp]:
-    """Compile a list of desired OIDC identity providers from app-interface."""
-    desired_state = []
-    error = False
-    for cluster in clusters:
-        for auth in cluster.auth:
-            if not isinstance(auth, ClusterAuthOIDCV1):
-                continue
-
-            if not auth.issuer:
-                logging.error(
-                    f"{cluster.name} auth={auth.name} doesn't have an issuer url set."
-                )
-                sys.exit(1)
-
-            secret = {
-                "path": f"{vault_input_path.rstrip('/')}/{QONTRACT_INTEGRATION}/{auth.name}/{cluster.name}"
-            }
-            try:
-                oauth_data = secret_reader.read_all(secret)
-                client_id = oauth_data["client_id"]
-                client_secret = oauth_data["client_secret"]
-            except Exception:
-                logging.error(f"unable to read secret in path {secret['path']}")
-                error = True
-                continue
-            ec = (
-                auth.claims.email
-                if auth.claims and auth.claims.email
-                else DEFAULT_EMAIL_CLAIMS
-            )
-            nc = (
-                auth.claims.name
-                if auth.claims and auth.claims.name
-                else DEFAULT_NAME_CLAIMS
-            )
-            uc = (
-                auth.claims.username
-                if auth.claims and auth.claims.username
-                else DEFAULT_USERNAME_CLAIMS
-            )
-            gc = (
-                auth.claims.groups
-                if auth.claims and auth.claims.groups
-                else DEFAULT_GROUPS_CLAIMS
-            )
-            desired_state.append(
-                OCMOidcIdp(
-                    cluster=cluster.name,
-                    name=auth.name,
-                    client_id=client_id,
-                    client_secret=client_secret,
-                    issuer=auth.issuer,
-                    email_claims=ec,
-                    name_claims=nc,
-                    username_claims=uc,
-                    groups_claims=gc,
-                )
-            )
-
-    if error:
-        sys.exit(1)
-
-    return desired_state
-
-
-def act(
-    dry_run: bool,
-    ocm_map: OCMMap,
-    current_state: Sequence[OCMOidcIdp],
-    desired_state: Sequence[OCMOidcIdp],
-) -> None:
-    """Compare current and desired OIDC identity providers and add, remove, or update them."""
-    to_add = set(desired_state) - set(current_state)
-    to_remove = set(current_state) - set(desired_state)
-    to_compare = set(current_state) & set(desired_state)
-    for idp in to_add:
-        logging.info(["create_oidc_idp", idp.cluster, idp.name])
-        if not dry_run:
-            ocm = ocm_map.get(idp.cluster)
-            ocm.create_oidc_idp(idp)
-
-    for idp in to_remove:
-        logging.info(["remove_oidc_idp", idp.cluster, idp.name])
-        if not idp.id:
-            logging.error(
-                "No identity provider id was given. This should never ever happen!"
-            )
-            sys.exit(1)
-        if not dry_run:
-            ocm = ocm_map.get(idp.cluster)
-            ocm.delete_idp(idp.cluster, idp.id)
-
-    for idp in to_compare:
-        current_idp = current_state[current_state.index(idp)]
-        desired_idp = desired_state[desired_state.index(idp)]
-        if not current_idp.differ(desired_idp):
-            # no changes detected
-            continue
-
-        logging.info(["update_oidc_idp", desired_idp.cluster, desired_idp.name])
-        if not current_idp.id:
-            logging.error(
-                "No identity provider id was given. This should never ever happen!"
-            )
-            sys.exit(1)
-        if not dry_run:
-            ocm = ocm_map.get(desired_idp.cluster)
-            ocm.update_oidc_idp(current_idp.id, desired_idp)
-
-
-def get_clusters(query_func: Callable) -> list[ClusterV1]:
-    """Get all clusters with an OCM relation from app-interface."""
-    data = cluster_query(query_func, variables={})
-    return [
-        c
-        for c in data.clusters or []
-        if integration_is_enabled(QONTRACT_INTEGRATION, c) and c.ocm is not None
-    ]
-
-
-def run(dry_run: bool, vault_input_path: str) -> None:
-    if not vault_input_path:
-        logging.error("must supply vault input path")
-        sys.exit(1)
-    gqlapi = gql.get_api()
-    settings = queries.get_app_interface_settings()
-    secret_reader = SecretReader(settings=settings)
-
-    # data query
-    clusters = get_clusters(gqlapi.query)
-    if not clusters:
-        logging.debug("No oidc-idp definitions found in app-interface")
-        sys.exit(ExitCodes.SUCCESS)
-
-    # APIs
-    ocm_map = OCMMap(
-        clusters=[cluster.dict(by_alias=True) for cluster in clusters],
-        integration=QONTRACT_INTEGRATION,
-        settings=settings,
-    )
-
-    # run
-    current_state = fetch_current_state(ocm_map, clusters)
-    desired_state = fetch_desired_state(secret_reader, clusters, vault_input_path)
-    act(dry_run, ocm_map, current_state, desired_state)
-
-
-def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
-    gqlapi = gql.get_api()
-    return {"clusters": [c.dict() for c in get_clusters(gqlapi.query)]}

reconcile/ocp_release_mirror.py

@@ -1,373 +0,0 @@
-import base64
-import logging
-import sys
-from collections import namedtuple
-from collections.abc import Mapping
-from typing import (
-    Any,
-    Optional,
-)
-from urllib.parse import urlparse
-
-from sretoolbox.container import Image
-
-from reconcile import queries
-from reconcile.gql_definitions.ocp_release_mirror.ocp_release_mirror import (
-    NamespaceV1,
-    OcpReleaseMirrorV1,
-)
-from reconcile.quay_base import (
-    OrgKey,
-    get_quay_api_store,
-)
-from reconcile.status import ExitCodes
-from reconcile.typed_queries.app_interface_vault_settings import (
-    get_app_interface_vault_settings,
-)
-from reconcile.typed_queries.ocp_release_mirror import get_ocp_release_mirrors
-from reconcile.utils.aws_api import AWSApi
-from reconcile.utils.external_resources import get_external_resource_specs
-from reconcile.utils.oc import OCLocal
-from reconcile.utils.oc_map import init_oc_map_from_clusters
-from reconcile.utils.ocm import OCMMap
-from reconcile.utils.secret_reader import create_secret_reader
-
-QONTRACT_INTEGRATION = "ocp-release-mirror"
-
-LOG = logging.getLogger(__name__)
-
-OcpReleaseInfo = namedtuple("OcpReleaseInfo", ["ocp_release", "tag"])
-
-
-class OcpReleaseMirrorError(Exception):
-    """
-    Used by the OcpReleaseMirror.
-    """
-
-
-class OcpReleaseMirror:
-    def __init__(self, dry_run: bool, instance: OcpReleaseMirrorV1):
-        self.dry_run = dry_run
-        settings = get_app_interface_vault_settings()
-        secret_reader = create_secret_reader(use_vault=settings.vault)
-
-        cluster_info = instance.hive_cluster
-        hive_cluster = instance.hive_cluster.name
-
-        # Getting the OCM Client for the hive cluster
-        ocm_map = OCMMap(
-            clusters=[cluster_info.dict(by_alias=True)],
-            integration=QONTRACT_INTEGRATION,
-            settings=settings.dict(by_alias=True),
-        )
-
-        self.ocm_cli = ocm_map.get(hive_cluster)
-        if self.ocm_cli is None:
-            raise OcpReleaseMirrorError(
-                f"Can't create ocm client for " f"cluster {hive_cluster}"
-            )
-
-        # Getting the OC Client for the hive cluster
-        oc_map = init_oc_map_from_clusters(
-            clusters=[cluster_info],
-            integration=QONTRACT_INTEGRATION,
-            secret_reader=secret_reader,
-        )
-        self.oc_cli = oc_map.get(hive_cluster)
-        if not self.oc_cli:
-            raise OcpReleaseMirrorError(
-                f"Can't create oc client for " f"cluster {hive_cluster}"
-            )
-
-        namespace = instance.ecr_resources_namespace
-        ocp_release_identifier = instance.ocp_release_ecr_identifier
-        ocp_art_dev_identifier = instance.ocp_art_dev_ecr_identifier
-
-        ocp_release_info = self._get_tf_resource_info(
-            namespace=namespace, identifier=ocp_release_identifier
-        )
-        if ocp_release_info is None:
-            raise OcpReleaseMirrorError(
-                f"Could not find rds "
-                f"identifier "
-                f"{ocp_release_identifier} in "
-                f"namespace {namespace.name}"
-            )
-
-        ocp_art_dev_info = self._get_tf_resource_info(
-            namespace=namespace, identifier=ocp_art_dev_identifier
-        )
-        if ocp_art_dev_info is None:
-            raise OcpReleaseMirrorError(
-                f"Could not find rds identifier"
-                f" {ocp_art_dev_identifier} in"
-                f"namespace {namespace.name}"
-            )
-
-        # Getting the AWS Client for the accounts
-        aws_accounts = [
-            self._get_aws_account_info(account=ocp_release_info["account"]),
-            self._get_aws_account_info(account=ocp_art_dev_info["account"]),
-        ]
-        self.aws_cli = AWSApi(
-            thread_pool_size=1,
-            accounts=aws_accounts,
-            secret_reader=secret_reader,
-            init_ecr_auth_tokens=True,
-        )
-        self.aws_cli.map_ecr_resources()
-
-        self.ocp_release_ecr_uri = self._get_image_uri(
-            account=ocp_release_info["account"], repository=ocp_release_identifier
-        )
-        if self.ocp_release_ecr_uri is None:
-            raise OcpReleaseMirrorError(
-                f"Could not find the " f"ECR repository " f"{ocp_release_identifier}"
-            )
-
-        self.ocp_art_dev_ecr_uri = self._get_image_uri(
-            account=ocp_art_dev_info["account"], repository=ocp_art_dev_identifier
-        )
-        if self.ocp_art_dev_ecr_uri is None:
-            raise OcpReleaseMirrorError(
-                f"Could not find the " f"ECR repository " f"{ocp_art_dev_identifier}"
-            )
-
-        # Process all the quayOrgTargets
-        quay_api_store = get_quay_api_store()
-        self.quay_target_orgs: list[dict[str, Any]] = []
-        for quay_target_org in instance.quay_target_orgs:
-            org_name = quay_target_org.name
-            instance_name = quay_target_org.instance.name
-            org_key = OrgKey(instance_name, org_name)
-            org_info = quay_api_store[org_key]
-
-            if not org_info["push_token"]:
-                raise OcpReleaseMirrorError(f"{org_key} has no push_token defined.")
-
-            url = org_info["url"]
-            user = org_info["push_token"]["user"]
-            token = org_info["push_token"]["token"]
-
-            self.quay_target_orgs.append(
-                {
-                    "url": url,
-                    "dest_ocp_release": f"{url}/{org_name}/ocp-release",
-                    "dest_ocp_art_dev": f"{url}/{org_name}/ocp-v4.0-art-dev",
-                    "auths": self._build_quay_auths(url, user, token),
-                }
-            )
-
-        # Getting all the credentials
-        quay_creds = self._get_quay_creds()
-        ocp_release_creds = self._get_ecr_creds(
-            account=ocp_release_info["account"], region=ocp_release_info["region"]
-        )
-        ocp_art_dev_creds = self._get_ecr_creds(
-            account=ocp_art_dev_info["account"], region=ocp_art_dev_info["region"]
-        )
-
-        # Creating a single dictionary with all credentials to be used by the
-        # "oc adm release mirror" command
-        self.registry_creds = {
-            "auths": {
-                **quay_creds["auths"],
-                **ocp_release_creds["auths"],
-                **ocp_art_dev_creds["auths"],
-            }
-        }
-
-        # Append quay_target_orgs auths to registry_creds
-        for quay_target_org_dict in self.quay_target_orgs:
-            url = quay_target_org_dict["url"]
-
-            if url in self.registry_creds["auths"].keys():
-                raise OcpReleaseMirrorError(
-                    f"Cannot mirror to the same Quay instance multiple times: {url}"
-                )
-
-            self.registry_creds["auths"].update(quay_target_org_dict["auths"])
-
-        # Initiate channel groups
-        self.channel_groups = instance.mirror_channels
-
-    def cleanup(self):
-        self.aws_cli.cleanup()
-
-    def run(self):
-        ocp_releases = self._get_ocp_releases()
-        if not ocp_releases:
-            raise RuntimeError("No OCP Releases found")
-
-        for ocp_release_info in ocp_releases:
-            ocp_release = ocp_release_info.ocp_release
-            tag = ocp_release_info.tag
-
-            # mirror to ecr
-            dest_ocp_release = f"{self.ocp_release_ecr_uri}:{tag}"
-            self._run_mirror(
-                ocp_release=ocp_release,
-                dest_ocp_release=dest_ocp_release,
-                dest_ocp_art_dev=self.ocp_art_dev_ecr_uri,
-            )
-
-            # mirror to all quay target orgs
-            for quay_target_org in self.quay_target_orgs:
-                dest_ocp_release = f'{quay_target_org["dest_ocp_release"]}:' f"{tag}"
-                dest_ocp_art_dev = quay_target_org["dest_ocp_art_dev"]
-                self._run_mirror(
-                    ocp_release=ocp_release,
-                    dest_ocp_release=dest_ocp_release,
-                    dest_ocp_art_dev=dest_ocp_art_dev,
-                )
-
-    def _run_mirror(self, ocp_release, dest_ocp_release, dest_ocp_art_dev):
-        # Checking if the image is already there
-        if self._is_image_there(dest_ocp_release):
-            LOG.debug(f"Image {ocp_release} already in " f"the mirror. Skipping.")
-            return
-
-        LOG.info(
-            f"Mirroring {ocp_release} to {dest_ocp_art_dev} "
-            f"to_release {dest_ocp_release}"
-        )
-
-        if self.dry_run:
-            return
-
-        # Creating a new, bare, OC client since we don't
-        # want to run this against any cluster or via
-        # a jump host
-        oc_cli = OCLocal("cluster", None, None, local=True)
-        oc_cli.release_mirror(
-            from_release=ocp_release,
-            to=dest_ocp_art_dev,
-            to_release=dest_ocp_release,
-            dockerconfig=self.registry_creds,
-        )
-
-    def _is_image_there(self, image):
-        image_obj = Image(image)
-
-        for registry, creds in self.registry_creds["auths"].items():
-            # Getting the credentials for the image_obj
-            if "//" not in registry:
-                registry = "//" + registry
-            registry_obj = urlparse(registry)
-
-            if registry_obj.netloc != image_obj.registry:
-                continue
-            image_obj.auth = (creds["username"], creds["password"])
-
-            # Checking if the image is already
-            # in the registry
-            if image_obj:
-                return True
-
-        return False
-
-    @staticmethod
-    def _get_aws_account_info(
-        account: Optional[Mapping[str, Any]]
-    ) -> Optional[dict[str, Any]]:
-        for account_info in queries.get_aws_accounts():
-            if "name" not in account_info:
-                continue
-            if account_info["name"] != account:
-                continue
-            return account_info
-        return None
-
-    def _get_ocp_releases(self):
-        ocp_releases = []
-        clusterimagesets = self.oc_cli.get_all("ClusterImageSet")
-        for clusterimageset in clusterimagesets["items"]:
-            release_image = clusterimageset["spec"]["releaseImage"]
-            # There are images in some ClusterImagesSets not coming
-            # from quay.io, e.g.:
-            # registry.svc.ci.openshift.org/ocp/release:4.2.0-0.nightly-2020-11-04-053758
-            # Let's filter out everything not from quay.io
-            if not release_image.startswith("quay.io"):
-                continue
-            labels = clusterimageset["metadata"]["labels"]
-            name = clusterimageset["metadata"]["name"]
-            # ClusterImagesSets may be enabled or disabled.
-            # Let's only mirror enabled ones
-            enabled = labels["api.openshift.com/enabled"]
-            if enabled == "false":
-                continue
-            # ClusterImageSets may be in different channels.
-            channel_group = labels["api.openshift.com/channel-group"]
-            if channel_group not in self.channel_groups:
-                continue
-            ocp_releases.append(OcpReleaseInfo(release_image, name))
-        return ocp_releases
-
-    def _get_quay_creds(self):
-        return self.ocm_cli.get_pull_secrets()
-
-    def _get_ecr_creds(self, account, region):
-        if region is None:
-            region = self.aws_cli.accounts[account]["resourcesDefaultRegion"]
-        auth_token = f"{account}/{region}"
-        data = self.aws_cli.auth_tokens[auth_token]
-        auth_data = data["authorizationData"][0]
-        server = auth_data["proxyEndpoint"]
-        token = auth_data["authorizationToken"]
-        password = base64.b64decode(token).decode("utf-8").split(":")[1]
-
-        return {
-            "auths": {
-                server: {
-                    "username": "AWS",
-                    "password": password,
-                    "email": "sd-app-sre@redhat.com",
-                    "auth": token,
-                }
-            }
-        }
-
-    @staticmethod
-    def _get_tf_resource_info(
-        namespace: NamespaceV1, identifier: str
-    ) -> Optional[dict[str, Any]]:
-        specs = get_external_resource_specs(namespace.dict(by_alias=True))
-        for spec in specs:
-            if spec.provider != "ecr":
-                continue
-
-            if spec.identifier != identifier:
-                continue
-
-            return {
-                "account": spec.provisioner_name,
-                "region": spec.resource.get("region"),
-            }
-        return None
-
-    def _get_image_uri(self, account: str, repository: str) -> Any:
-        for repo in self.aws_cli.resources[account]["ecr"]:
-            if repo["repositoryName"] == repository:
-                return repo["repositoryUri"]
-
-    @staticmethod
-    def _build_quay_auths(url, user, token):
-        auth_bytes = bytes(f"{user}:{token}", "utf-8")
-        auth = base64.b64encode(auth_bytes).decode("utf-8")
-        return {url: {"username": user, "password": token, "email": "", "auth": auth}}
-
-
-def run(dry_run: bool) -> None:
-    instances = get_ocp_release_mirrors()
-    for instance in instances:
-        try:
-            quay_mirror = OcpReleaseMirror(dry_run=dry_run, instance=instance)
-            quay_mirror.run()
-        except OcpReleaseMirrorError as details:
-            LOG.error(str(details))
-            sys.exit(ExitCodes.ERROR)
-        finally:
-            try:
-                quay_mirror.cleanup()
-            except NameError:
-                LOG.debug("No quay_mirror to cleanup")