qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/query_validator.py

@@ -27,9 +27,7 @@ QUERY_VALIDATIONS_QUERY = """
     }
   }
 }
-""" % (
-    indent(OPENSHIFT_RESOURCE, 6 * " "),
-)
+""" % (indent(OPENSHIFT_RESOURCE, 6 * " "),)
 
 
 def run(dry_run):
@@ -44,13 +42,13 @@ def run(dry_run):
                 gqlapi.query(gql.get_resource(q["path"])["content"])
             except (gql.GqlGetResourceError, gql.GqlApiError) as e:
                 error = True
-                logging.error(f"query validation error in {qv_name}: {str(e)}")
+                logging.error(f"query validation error in {qv_name}: {e!s}")
         for r in qv.get("resources") or []:
             try:
                 fetch_openshift_resource(r, qv, settings=settings, skip_validation=True)
             except Exception as e:
                 error = True
-                logging.error(f"query validation error in {qv_name}: {str(e)}")
+                logging.error(f"query validation error in {qv_name}: {e!s}")
 
     if error:
         sys.exit(ExitCodes.ERROR)

reconcile/resource_scraper.py

@@ -1,7 +1,6 @@
 import logging
 import sys
 from typing import (
-    Optional,
     cast,
 )
 
@@ -23,9 +22,9 @@ QONTRACT_INTEGRATION = "resource-scraper"
 
 def run(
     dry_run: bool,
-    namespace_name: Optional[str],
-    resource_kind: Optional[str],
-    vault_output_path: Optional[str],
+    namespace_name: str | None,
+    resource_kind: str | None,
+    vault_output_path: str | None,
 ) -> None:
     """Get resources from clusters and store in Vault."""
     if not namespace_name:

reconcile/{template_tester.py → resource_template_tester.py}

@@ -11,13 +11,13 @@ from reconcile.status import ExitCodes
 from reconcile.utils import gql
 from reconcile.utils.semver_helper import make_semver
 
-QONTRACT_INTEGRATION = "template-tester"
+QONTRACT_INTEGRATION = "resource-template-tester"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
 
 
 TEMPLATE_TESTS_QUERY = """
 {
-  tests: template_tests_v1 {
+  tests: resource_template_tests_v1 {
     name
     resourcePath
     expectedResult
@@ -30,7 +30,7 @@ def load_resource(path: str) -> dict:
     return yaml.safe_load(gql.get_resource(path)["content"])
 
 
-def run(dry_run):
+def run(dry_run: bool) -> None:
     gqlapi = gql.get_api()
     template_tests = gqlapi.query(TEMPLATE_TESTS_QUERY)["tests"]
     settings = queries.get_app_interface_settings()

reconcile/rhidp/common.py

@@ -0,0 +1,214 @@
+from collections import Counter
+from collections.abc import (
+    Iterable,
+    MutableMapping,
+)
+from enum import StrEnum
+from typing import Any
+from urllib.parse import urlparse
+
+from pydantic import (
+    BaseModel,
+    root_validator,
+)
+
+from reconcile.gql_definitions.common.ocm_environments import (
+    query as ocm_environment_query,
+)
+from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.gql_definitions.rhidp.organizations import (
+    OpenShiftClusterManagerV1,
+)
+from reconcile.gql_definitions.rhidp.organizations import (
+    query as ocm_orgs_query,
+)
+from reconcile.rhidp.metrics import RhIdpClusterCounter
+from reconcile.utils import gql
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.metrics import MetricsContainer
+from reconcile.utils.ocm.base import OCMCluster
+from reconcile.utils.ocm.clusters import (
+    ClusterDetails,
+    discover_clusters_by_labels,
+)
+from reconcile.utils.ocm.labels import subscription_label_filter
+from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+# Generates label keys for rhidp, compliant with the naming schema defined in
+# https://service.pages.redhat.com/dev-guidelines/docs/sre-capabilities/framework/ocm-labels/
+RHIDP_NAMESPACE_LABEL_KEY = sre_capability_label_key("rhidp")
+STATUS_LABEL_KEY = sre_capability_label_key("rhidp", "status")
+ISSUER_LABEL_KEY = sre_capability_label_key("rhidp", "issuer")
+AUTH_NAME_LABEL_KEY = sre_capability_label_key("rhidp", "name")
+
+
+class StatusValue(StrEnum):
+    # rhidp and oidc are enabled
+    ENABLED = "enabled"
+    # rhidp and oidc are disabled
+    DISABLED = "disabled"
+    # rhidp is enabled and oidc will delete all other configured idps
+    ENFORCED = "enforced"
+    # rhidp is enabled and oidc is skipped
+    RHIDP_ONLY = "sso-client-only"
+
+
+class ClusterAuth(BaseModel):
+    name: str
+    issuer: str
+    status: str
+
+    @root_validator
+    def name_no_spaces(  # pylint: disable=no-self-argument
+        cls, values: MutableMapping[str, Any]
+    ) -> MutableMapping[str, Any]:
+        values["name"] = values["name"].replace(" ", "-")
+        return values
+
+    @property
+    def rhidp_enabled(self) -> bool:
+        return self.status != StatusValue.DISABLED.value
+
+    @property
+    def oidc_enabled(self) -> bool:
+        return self.status not in {
+            StatusValue.DISABLED.value,
+            StatusValue.RHIDP_ONLY.value,
+        }
+
+    @property
+    def enforced(self) -> bool:
+        return self.status == StatusValue.ENFORCED.value
+
+
+class Cluster(BaseModel):
+    ocm_cluster: OCMCluster
+    auth: ClusterAuth
+    organization_id: str
+
+    @property
+    def name(self) -> str:
+        return self.ocm_cluster.name
+
+    @property
+    def console_url(self) -> str | None:
+        return self.ocm_cluster.console.url if self.ocm_cluster.console else None
+
+
+def discover_clusters(
+    ocm_api: OCMBaseClient, org_ids: set[str] | None
+) -> list[ClusterDetails]:
+    """Discover all clusters that are part of the RHIDP service."""
+    clusters = discover_clusters_by_labels(
+        ocm_api=ocm_api,
+        label_filter=subscription_label_filter().like(
+            "key", f"{RHIDP_NAMESPACE_LABEL_KEY}%"
+        ),
+    )
+
+    # filter by org if org_id is specified
+    return [c for c in clusters if org_ids is None or c.organization_id in org_ids]
+
+
+def build_cluster_objects(
+    cluster_details: Iterable[ClusterDetails],
+    default_auth_name: str,
+    default_issuer_url: str,
+) -> list[Cluster]:
+    return [
+        Cluster(
+            ocm_cluster=cluster.ocm_cluster,
+            auth=ClusterAuth(
+                name=cluster.labels.get_label_value(AUTH_NAME_LABEL_KEY)
+                or default_auth_name,
+                issuer=cluster.labels.get_label_value(ISSUER_LABEL_KEY)
+                or default_issuer_url,
+                # "rhidp" label is deprecated, but we still need to support it
+                # "rhidp.status" is the new label
+                status=cluster.labels.get_label_value(RHIDP_NAMESPACE_LABEL_KEY)
+                or cluster.labels.get_label_value(STATUS_LABEL_KEY)
+                or StatusValue.DISABLED.value,
+            ),
+            organization_id=cluster.organization_id,
+        )
+        for cluster in cluster_details
+        # we can't calculate the redirect url w/o a console url
+        if cluster.ocm_cluster.console
+    ]
+
+
+def cluster_vault_secret_id(
+    org_id: str, cluster_name: str, auth_name: str, issuer_url: str
+) -> str:
+    """Returns the vault secret id for the given cluster."""
+    url = urlparse(issuer_url)
+    return f"{cluster_name}-{org_id}-{auth_name}-{url.hostname}"
+
+
+def cluster_vault_secret(
+    vault_input_path: str,
+    org_id: str | None = None,
+    cluster_name: str | None = None,
+    auth_name: str | None = None,
+    issuer_url: str | None = None,
+    vault_secret_id: str | None = None,
+) -> VaultSecret:
+    """Returns the vault secret path for the given cluster."""
+    if not vault_secret_id and (org_id and cluster_name and auth_name and issuer_url):
+        cid = cluster_vault_secret_id(org_id, cluster_name, auth_name, issuer_url)
+    elif vault_secret_id:
+        cid = vault_secret_id
+    else:
+        raise ValueError(
+            "vault_secret_id or org_id, cluster_name, auth_name, and issuer_url must be provided"
+        )
+    return VaultSecret(
+        path=f"{vault_input_path.rstrip('/')}/{cid}",
+        field="",
+        version=None,
+        format=None,
+    )
+
+
+def expose_base_metrics(
+    metrics_container: MetricsContainer,
+    integration_name: str,
+    ocm_environment: str,
+    clusters: Iterable[Cluster],
+) -> None:
+    clusters_per_org: Counter[str] = Counter()
+    for cluster in clusters:
+        clusters_per_org[cluster.organization_id] += 1
+
+    # clusters per org counter
+    for org_id, count in clusters_per_org.items():
+        metrics_container.set_gauge(
+            RhIdpClusterCounter(
+                integration=integration_name,
+                ocm_environment=ocm_environment,
+                org_id=org_id,
+            ),
+            value=count,
+        )
+
+
+def get_ocm_environments(env_name: str | None) -> list[OCMEnvironment]:
+    return ocm_environment_query(
+        gql.get_api().query,
+        variables={"name": env_name} if env_name else None,
+    ).environments
+
+
+def get_ocm_orgs_from_env(
+    env_name: str, int_name: str
+) -> list[OpenShiftClusterManagerV1]:
+    orgs = ocm_orgs_query(
+        gql.get_api().query,
+    ).organizations
+    return [
+        org
+        for org in orgs or []
+        if integration_is_enabled(int_name, org) and org.environment.name == env_name
+    ]

reconcile/rhidp/metrics.py

@@ -0,0 +1,20 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import GaugeMetric
+
+
+class RhIdpBaseMetric(BaseModel):
+    """Base class for RhIdp metrics"""
+
+    integration: str
+    ocm_environment: str
+
+
+class RhIdpClusterCounter(RhIdpBaseMetric, GaugeMetric):
+    """Number of managed clusters per organization."""
+
+    org_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_managed_clusters"

reconcile/rhidp/ocm_oidc_idp/base.py

@@ -0,0 +1,221 @@
+import logging
+from collections.abc import (
+    Iterable,
+    Sequence,
+)
+
+from pydantic import BaseModel
+
+from reconcile.rhidp.common import (
+    Cluster,
+    cluster_vault_secret,
+    expose_base_metrics,
+)
+from reconcile.rhidp.ocm_oidc_idp.metrics import (
+    RhIdpOCMOidcIdpReconcileCounter,
+    RhIdpOCMOidcIdpReconcileErrorCounter,
+)
+from reconcile.utils import metrics
+from reconcile.utils.differ import diff_iterables
+from reconcile.utils.keycloak import SSOClient
+from reconcile.utils.ocm.base import (
+    OCMOIdentityProvider,
+    OCMOIdentityProviderGithub,
+    OCMOIdentityProviderOidc,
+    OCMOIdentityProviderOidcOpenId,
+)
+from reconcile.utils.ocm.identity_providers import (
+    add_identity_provider,
+    delete_identity_provider,
+    get_identity_providers,
+    update_identity_provider,
+)
+from reconcile.utils.ocm_base_client import OCMBaseClient
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class IDPState(BaseModel):
+    cluster: Cluster
+    idp: OCMOIdentityProvider | OCMOIdentityProviderOidc | OCMOIdentityProviderGithub
+
+    def __eq__(self, __value: object) -> bool:
+        if not isinstance(__value, IDPState):
+            raise NotImplementedError("Cannot compare to non IDPState objects.")
+        return self.idp == __value.idp
+
+
+def run(
+    integration_name: str,
+    ocm_environment: str,
+    clusters: Iterable[Cluster],
+    secret_reader: SecretReaderBase,
+    ocm_api: OCMBaseClient,
+    vault_input_path: str,
+    dry_run: bool,
+    managed_idps: list[str] | None = None,
+) -> None:
+    with metrics.transactional_metrics(ocm_environment) as metrics_container:
+        # metrics
+        expose_base_metrics(
+            metrics_container, integration_name, ocm_environment, clusters
+        )
+
+        try:
+            # run
+            current_state = fetch_current_state(ocm_api, clusters)
+            desired_state = fetch_desired_state(
+                secret_reader, clusters, vault_input_path
+            )
+            act(
+                dry_run,
+                ocm_api,
+                current_state,
+                desired_state,
+                managed_idps=managed_idps or [],
+            )
+            metrics_container.inc_counter(
+                RhIdpOCMOidcIdpReconcileCounter(
+                    integration=integration_name, ocm_environment=ocm_environment
+                )
+            )
+        except Exception:
+            metrics_container.inc_counter(
+                RhIdpOCMOidcIdpReconcileErrorCounter(
+                    integration=integration_name, ocm_environment=ocm_environment
+                )
+            )
+            raise
+
+
+def fetch_current_state(
+    ocm_api: OCMBaseClient, clusters: Iterable[Cluster]
+) -> list[IDPState]:
+    """Fetch all current configured OIDC identity providers."""
+    current_state: list[IDPState] = []
+    for cluster in clusters:
+        for idp in get_identity_providers(
+            ocm_api=ocm_api, ocm_cluster=cluster.ocm_cluster
+        ):
+            current_state.append(
+                IDPState(
+                    cluster=cluster,
+                    idp=idp,
+                ),
+            )
+
+    return current_state
+
+
+def fetch_desired_state(
+    secret_reader: SecretReaderBase,
+    clusters: Iterable[Cluster],
+    vault_input_path: str,
+) -> list[IDPState]:
+    """Compile a list of desired OIDC identity providers from app-interface."""
+    desired_state: list[IDPState] = []
+    for cluster in clusters:
+        if not cluster.auth.oidc_enabled:
+            continue
+
+        secret = cluster_vault_secret(
+            org_id=cluster.organization_id,
+            cluster_name=cluster.name,
+            auth_name=cluster.auth.name,
+            issuer_url=cluster.auth.issuer,
+            vault_input_path=vault_input_path,
+        )
+        try:
+            oauth_data = secret_reader.read_all_secret(secret)
+        except Exception:
+            logging.warning(
+                f"Unable to read secret in path {secret.path}. "
+                f"Maybe not created yet? Skipping OIDC config for cluster {cluster.name}"
+            )
+            continue
+
+        sso_client = SSOClient(**oauth_data)
+        if sso_client.issuer != cluster.auth.issuer:
+            # this can only happen if someone manually change the secret or copied it
+            logging.error(
+                f"SSO client issuer {sso_client.issuer} does not match configured cluster issuer "
+                f"{cluster.auth.issuer}. Skipping OIDC config for cluster {cluster.name}"
+            )
+            continue
+        desired_state.append(
+            IDPState(
+                cluster=cluster,
+                idp=OCMOIdentityProviderOidc(
+                    name=cluster.auth.name,
+                    open_id=OCMOIdentityProviderOidcOpenId(
+                        client_id=sso_client.client_id,
+                        client_secret=sso_client.client_secret,
+                        issuer=cluster.auth.issuer,
+                    ),
+                ),
+            )
+        )
+
+    return desired_state
+
+
+def act(
+    dry_run: bool,
+    ocm_api: OCMBaseClient,
+    current_state: Sequence[IDPState],
+    desired_state: Sequence[IDPState],
+    managed_idps: list[str],
+) -> None:
+    """Compare current and desired OIDC identity providers and add, remove, or update them."""
+    diff_result = diff_iterables(
+        current_state,
+        desired_state,
+        key=lambda idp_state: ((
+            idp_state.cluster.organization_id,
+            idp_state.cluster.name,
+            idp_state.idp.type,
+            idp_state.idp.name,
+        )),
+        equal=lambda idp1, idp2: idp1 == idp2,
+    )
+
+    for idp_state in diff_result.delete.values():
+        if (
+            managed_idps
+            and idp_state.idp.name not in managed_idps
+            and not idp_state.cluster.auth.enforced
+        ):
+            logging.debug(f"Skipping removal of unmanged '{idp_state.idp.name}' IDP.")
+            continue
+        logging.info(["remove_oidc_idp", idp_state.cluster.name, idp_state.idp.name])
+        if not dry_run:
+            delete_identity_provider(ocm_api, idp_state.idp)
+
+    for idp_state in diff_result.add.values():
+        if not isinstance(idp_state.idp, OCMOIdentityProviderOidc):
+            logging.error(
+                f"Identity provider {idp_state.idp.name} is not an OIDC identity provider."
+            )
+            continue
+        logging.info(["create_oidc_idp", idp_state.cluster.name, idp_state.idp.name])
+        if not dry_run:
+            add_identity_provider(ocm_api, idp_state.cluster.ocm_cluster, idp_state.idp)
+
+    for diff_pair in diff_result.change.values():
+        current_idp_state = diff_pair.current
+        desired_idp_state = diff_pair.desired
+        current_idp = current_idp_state.idp
+        desired_idp = desired_idp_state.idp
+        desired_idp.href = current_idp.href
+
+        if not isinstance(desired_idp, OCMOIdentityProviderOidc):
+            logging.error(
+                f"Identity provider {desired_idp.name} is not an OIDC identity provider."
+            )
+            continue
+        logging.info([
+            "update_oidc_idp",
+            desired_idp_state.cluster.name,
+            desired_idp.name,
+        ])
+        if not dry_run:
+            update_identity_provider(ocm_api, desired_idp)

reconcile/rhidp/ocm_oidc_idp/integration.py

@@ -0,0 +1,56 @@
+from reconcile.rhidp.common import (
+    build_cluster_objects,
+    discover_clusters,
+    get_ocm_environments,
+    get_ocm_orgs_from_env,
+)
+from reconcile.rhidp.ocm_oidc_idp.base import run
+from reconcile.utils.ocm_base_client import init_ocm_base_client
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+
+QONTRACT_INTEGRATION = "ocm-oidc-idp"
+
+
+class OCMOidcIdpParams(PydanticRunParams):
+    vault_input_path: str
+    ocm_environment: str | None = None
+    default_auth_name: str
+    default_auth_issuer_url: str
+
+
+class OCMOidcIdp(QontractReconcileIntegration[OCMOidcIdpParams]):
+    """The OCM OIDC IDP integration manages the cluster OIDC OCM configuration and uses OCM labels to discover clusters."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def run(self, dry_run: bool) -> None:
+        for ocm_env in get_ocm_environments(self.params.ocm_environment):
+            ocm_api = init_ocm_base_client(ocm_env, self.secret_reader)
+            # data query
+            cluster_details = discover_clusters(
+                ocm_api=ocm_api,
+                org_ids={
+                    org.org_id for org in get_ocm_orgs_from_env(ocm_env.name, self.name)
+                },
+            )
+            clusters = build_cluster_objects(
+                cluster_details=cluster_details,
+                default_auth_name=self.params.default_auth_name,
+                default_issuer_url=self.params.default_auth_issuer_url,
+            )
+
+            run(
+                integration_name=self.name,
+                ocm_environment=ocm_env.name,
+                clusters=clusters,
+                secret_reader=self.secret_reader,
+                ocm_api=ocm_api,
+                vault_input_path=f"{self.params.vault_input_path}/{ocm_env.name}",
+                dry_run=dry_run,
+                managed_idps=[self.params.default_auth_name],
+            )

reconcile/rhidp/ocm_oidc_idp/metrics.py

@@ -0,0 +1,22 @@
+from reconcile.rhidp.metrics import RhIdpBaseMetric
+from reconcile.utils.metrics import CounterMetric
+
+
+class RhIdpOCMOidcIdpReconcileErrorCounter(RhIdpBaseMetric, CounterMetric):
+    """Counter for the failed reconcile runs."""
+
+    ocm_environment: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_ocm_oidc_idp_reconcile_errors"
+
+
+class RhIdpOCMOidcIdpReconcileCounter(RhIdpBaseMetric, CounterMetric):
+    """Counter for successful reconcile runs."""
+
+    ocm_environment: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "rhidp_ocm_oidc_idp_reconciled"