qontract-reconcile 0.10.1rc349__py3-none-any.whl → 0.10.1rc351__py3-none-any.whl

reconcile/ocm_labels/integration.py

@@ -0,0 +1,406 @@
+from __future__ import annotations
+
+import logging
+from collections.abc import (
+    Callable,
+    Iterable,
+)
+from typing import (
+    Any,
+    Optional,
+)
+
+from deepdiff import DeepHash
+from pydantic import validator
+
+from reconcile.aus.aus_label_source import (
+    init_aus_cluster_label_source,
+    init_aus_org_label_source,
+)
+from reconcile.gql_definitions.common.ocm_environments import (
+    query as ocm_environment_query,
+)
+from reconcile.gql_definitions.fragments.ocm_environment import OCMEnvironment
+from reconcile.gql_definitions.ocm_labels.clusters import ClusterV1
+from reconcile.gql_definitions.ocm_labels.clusters import query as cluster_query
+from reconcile.gql_definitions.ocm_labels.organizations import OpenShiftClusterManagerV1
+from reconcile.gql_definitions.ocm_labels.organizations import (
+    query as organization_query,
+)
+from reconcile.ocm_labels.label_sources import (
+    ClusterRef,
+    LabelSource,
+    LabelState,
+    OrgRef,
+)
+from reconcile.utils import gql
+from reconcile.utils.differ import diff_mappings
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.helpers import flatten
+from reconcile.utils.ocm.clusters import discover_clusters_for_organizations
+from reconcile.utils.ocm.labels import (
+    add_label,
+    build_organization_labels_href,
+    delete_label,
+    get_org_labels,
+    update_label,
+)
+from reconcile.utils.ocm.search_filters import Filter
+from reconcile.utils.ocm_base_client import (
+    OCMAPIClientConfigurationProtocol,
+    OCMBaseClient,
+    init_ocm_base_client,
+)
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+QONTRACT_INTEGRATION = "ocm-labels"
+
+
+class OcmLabelsIntegrationParams(PydanticRunParams):
+    managed_label_prefixes: list[str] = []
+
+    @validator("managed_label_prefixes")
+    def must_end_with_dot(  # pylint: disable=no-self-argument
+        cls, v: list[str]
+    ) -> list[str]:
+        return [prefix + "." if not prefix.endswith(".") else prefix for prefix in v]
+
+
+class ManagedLabelConflictError(Exception):
+    pass
+
+
+class OcmLabelsIntegration(QontractReconcileIntegration[OcmLabelsIntegrationParams]):
+    """Sync labels to subscription and organizations."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_clusters(self, query_func: Callable) -> list[ClusterV1]:
+        data = cluster_query(query_func)
+        return [
+            c
+            for c in data.clusters or []
+            if c.ocm is not None and integration_is_enabled(self.name, c)
+        ]
+
+    def get_organizations(
+        self, query_func: Callable
+    ) -> list[OpenShiftClusterManagerV1]:
+        return organization_query(query_func).organizations or []
+
+    def get_environments(self, query_func: Callable) -> list[OCMEnvironment]:
+        return ocm_environment_query(query_func).environments
+
+    def init_ocm_apis(
+        self,
+        environments: Iterable[OCMEnvironment],
+        init_ocm_base_client: Callable[
+            [OCMAPIClientConfigurationProtocol, SecretReaderBase], OCMBaseClient
+        ] = init_ocm_base_client,
+    ) -> dict[str, OCMBaseClient]:
+        return {
+            env.name: init_ocm_base_client(env, self.secret_reader)
+            for env in environments
+        }
+
+    def org_label_sources(self, query_func: Callable) -> list[LabelSource]:
+        return [
+            init_aus_org_label_source(query_func),
+        ]
+
+    def subscription_label_sources(
+        self, clusters: list[ClusterV1], query_func: Callable
+    ) -> list[LabelSource]:
+        return [
+            init_cluster_subscription_label_source(clusters),
+            init_aus_cluster_label_source(query_func),
+        ]
+
+    def get_early_exit_desired_state(self) -> Optional[dict[str, Any]]:
+        gqlapi = gql.get_api()
+        desired = {
+            "org_labels": self.fetch_desired_state(
+                self.org_label_sources(gqlapi.query)
+            ),
+            "subs_labels": self.fetch_desired_state(
+                self.subscription_label_sources(
+                    self.get_clusters(gqlapi.query), gqlapi.query
+                )
+            ),
+        }
+        # to figure out wheter to run a PR check of to exit early, a hash value
+        # of the desired state is sufficient
+        return {"hash": DeepHash(desired).get(desired)}
+
+    def run(self, dry_run: bool) -> None:
+        gqlapi = gql.get_api()
+        self.get_early_exit_desired_state()
+        clusters = self.get_clusters(gqlapi.query)
+        organizations = self.get_organizations(gqlapi.query)
+        environments = self.get_environments(gqlapi.query)
+
+        self.ocm_apis = self.init_ocm_apis(environments, init_ocm_base_client)
+
+        # organization labels
+        orgs_current_state, orgs_desired_state = self.fetch_organization_label_states(
+            organizations, gqlapi.query
+        )
+        self.reconcile(
+            dry_run=dry_run,
+            scope="organization",
+            current_state=orgs_current_state,
+            desired_state=orgs_desired_state,
+        )
+
+        # subscription labels
+        subs_current_state, subs_desired_state = self.fetch_subscription_label_states(
+            clusters, gqlapi.query
+        )
+        self.reconcile(
+            dry_run=dry_run,
+            scope="cluster",
+            current_state=subs_current_state,
+            desired_state=subs_desired_state,
+        )
+
+    def fetch_organization_label_states(
+        self, organizations: Iterable[OpenShiftClusterManagerV1], query_func: Callable
+    ) -> tuple[LabelState, LabelState]:
+        """
+        Returns the current and desired state of the organizations labels for
+        the given organizations.
+
+        Please note that the current state might not contain all requested organizations,
+        e.g. if a organization can't be found in OCM.
+        """
+        current_state = self.fetch_organization_label_current_state(
+            organizations, self.params.managed_label_prefixes
+        )
+        desired_state = self.fetch_desired_state(self.org_label_sources(query_func))
+        return current_state, desired_state
+
+    def fetch_organization_label_current_state(
+        self,
+        organizations: Iterable[OpenShiftClusterManagerV1],
+        managed_label_prefixes: list[str],
+    ) -> LabelState:
+        """
+        Fetches the current state of organizations labels for the given organizations.
+        If an organization can't be found in OCM, the resulting dict will not contain a
+        state for it, not even an empty one.
+
+        Only labels with a prefix in managed_label_prefixes are returned. Not every label
+        on an organizations is this integrations business.
+        """
+        states: LabelState = {
+            OrgRef(
+                org_id=org.org_id,
+                ocm_env=org.environment.name,
+                label_container_href=build_organization_labels_href(org.org_id),
+                name=org.name,
+            ): {}
+            for org in organizations
+        }
+
+        # prepare search filters
+        managed_label_filter = Filter()
+        for prefix in managed_label_prefixes:
+            managed_label_filter |= Filter().like("key", f"{prefix}%")
+
+        for env_name, ocm_api in self.ocm_apis.items():
+            env_orgs = {
+                o.org_id: o for o in organizations if o.environment.name == env_name
+            }
+            if not env_orgs:
+                continue
+            labels_by_org_id = get_org_labels(
+                ocm_api=ocm_api,
+                org_ids=set(env_orgs.keys()),
+                label_filter=managed_label_filter,
+            )
+            for org_id, labels in labels_by_org_id.items():
+                states[
+                    OrgRef(
+                        org_id=org_id,
+                        ocm_env=env_name,
+                        label_container_href=build_organization_labels_href(org_id),
+                        name=env_orgs[org_id].name,
+                    )
+                ] = labels.get_values_dict()
+
+        return states
+
+    def fetch_subscription_label_states(
+        self, clusters: list[ClusterV1], query_func: Callable
+    ) -> tuple[LabelState, LabelState]:
+        """
+        Returns the current and desired state of the subscription labels for
+        the given clusters.
+
+        Please note that the current state might not contain all requested clusters,
+        e.g. if a cluster can't be found in OCM or is not considered ready yet.
+        """
+        current_state = self.fetch_subscription_label_current_state(
+            clusters, self.params.managed_label_prefixes
+        )
+        desired_state = self.fetch_desired_state(
+            self.subscription_label_sources(clusters, query_func)
+        )
+        return current_state, desired_state
+
+    def fetch_subscription_label_current_state(
+        self, clusters: Iterable[ClusterV1], managed_label_prefixes: list[str]
+    ) -> LabelState:
+        """
+        Fetches the current state of subscription labels for the given clusters.
+        If a cluster can't be found in OCM, the resulting dict will not contain a
+        state for it, not even an empty one.
+
+        Only labels with a prefix in managed_label_prefixes are returned. Not every label
+        on a subscription is this integrations business.
+        """
+        cluster_ids = {c.spec.q_id for c in clusters if c.spec and c.spec.q_id}
+        states: LabelState = {}
+        for env_name, ocm_api in self.ocm_apis.items():
+            for cluster_details in discover_clusters_for_organizations(
+                ocm_api=ocm_api,
+                organization_ids=list(
+                    {
+                        c.ocm.org_id
+                        for c in clusters
+                        if c.ocm and c.ocm.environment.name == env_name
+                    }
+                ),
+            ):
+                if cluster_details.ocm_cluster.id not in cluster_ids:
+                    # there might be more clusters in an organization than we care about
+                    continue
+
+                filtered_labels = {
+                    label: value
+                    for label, value in cluster_details.subscription_labels.get_values_dict().items()
+                    if label.startswith(tuple(managed_label_prefixes))
+                }
+                states[
+                    ClusterRef(
+                        cluster_id=cluster_details.ocm_cluster.id,
+                        org_id=cluster_details.organization_id,
+                        ocm_env=env_name,
+                        name=cluster_details.ocm_cluster.name,
+                        label_container_href=f"{cluster_details.ocm_cluster.subscription.href}/labels",
+                    )
+                ] = filtered_labels
+        return states
+
+    def fetch_desired_state(self, sources: list[LabelSource]) -> LabelState:
+        states: LabelState = {}
+        for s in sources:
+            for owner_ref, labels in s.get_labels().items():
+                if owner_ref not in states:
+                    states[owner_ref] = {}
+                for label, value in labels.items():
+                    if label in states[owner_ref]:
+                        raise ManagedLabelConflictError(
+                            f"The label {label} on {owner_ref.identity_labels()} is already managed by another label source"
+                        )
+                    states[owner_ref][label] = value
+
+        return states
+
+    def reconcile(
+        self,
+        dry_run: bool,
+        scope: str,
+        current_state: LabelState,
+        desired_state: LabelState,
+    ) -> None:
+        # we iterate via the current state because it refers to the clusters we can act on
+        for label_owner_ref, current_labels in current_state.items():
+            ocm_api = self.ocm_apis[label_owner_ref.ocm_env]
+            desired_labels = desired_state.get(label_owner_ref, {})
+            if current_labels == desired_labels:
+                continue
+
+            diff_result = diff_mappings(current_labels, desired_labels)
+
+            for label_to_add, value in diff_result.add.items():
+                logging.info(
+                    [
+                        f"create_{scope}_label",
+                        *label_owner_ref.identity_labels(),
+                        f"{label_to_add}={value}",
+                    ]
+                )
+                if not dry_run:
+                    add_label(
+                        ocm_api=ocm_api,
+                        label_container_href=label_owner_ref.required_label_container_href(),
+                        label=label_to_add,
+                        value=value,
+                    )
+            for label_to_rm, value in diff_result.delete.items():
+                logging.info(
+                    [
+                        f"delete_{scope}_label",
+                        *label_owner_ref.identity_labels(),
+                        f"{label_to_rm}={value}",
+                    ]
+                )
+                if not dry_run:
+                    delete_label(
+                        ocm_api=ocm_api,
+                        label_container_href=label_owner_ref.required_label_container_href(),
+                        label=label_to_rm,
+                    )
+            for label_to_update, diff_pair in diff_result.change.items():
+                value = diff_pair.desired
+                logging.info(
+                    [
+                        f"update_{scope}_label",
+                        *label_owner_ref.identity_labels(),
+                        f"{label_to_update}={value}",
+                    ]
+                )
+                if not dry_run:
+                    update_label(
+                        ocm_api=ocm_api,
+                        label_container_href=label_owner_ref.required_label_container_href(),
+                        label=label_to_update,
+                        value=value,
+                    )
+
+
+def init_cluster_subscription_label_source(
+    clusters: list[ClusterV1],
+) -> ClusterSubscriptionLabelSource:
+    return ClusterSubscriptionLabelSource(
+        clusters=[
+            c
+            for c in clusters or []
+            if c.ocm is not None and integration_is_enabled(QONTRACT_INTEGRATION, c)
+        ],
+    )
+
+
+class ClusterSubscriptionLabelSource(LabelSource):
+    def __init__(self, clusters: Iterable[ClusterV1]) -> None:
+        self.clusters = clusters
+
+    def get_labels(self) -> LabelState:
+        return {
+            ClusterRef(
+                cluster_id=cluster.spec.q_id,
+                org_id=cluster.ocm.org_id,
+                ocm_env=cluster.ocm.environment.name,
+                name=cluster.name,
+                label_container_href=None,
+            ): flatten(cluster.ocm_subscription_labels or {})
+            for cluster in self.clusters
+            if cluster.ocm and cluster.spec and cluster.spec.q_id
+        }

reconcile/ocm_labels/label_sources.py

@@ -0,0 +1,76 @@
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class LabelOwnerRef(ABC):
+    ocm_env: str
+    label_container_href: Optional[str]
+
+    @abstractmethod
+    def identity_labels(self) -> list[str]:
+        pass
+
+    def required_label_container_href(self) -> str:
+        if self.label_container_href is None:
+            raise ValueError(
+                "label_container_href is missing - this method should probably not be called in this state"
+            )
+        return self.label_container_href
+
+
+@dataclass(frozen=True)
+class OrgRef(LabelOwnerRef):
+    org_id: str
+    name: str
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, OrgRef)
+            and self.org_id == other.org_id
+            and self.ocm_env == other.ocm_env
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.org_id, self.ocm_env))
+
+    def identity_labels(self) -> list[str]:
+        return [f"org_id={self.org_id}", f"org_name={self.name}"]
+
+
+@dataclass(frozen=True)
+class ClusterRef(LabelOwnerRef):
+    cluster_id: str
+    org_id: str
+    name: str
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, ClusterRef)
+            and self.cluster_id == other.cluster_id
+            and self.org_id == other.org_id
+            and self.ocm_env == other.ocm_env
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.cluster_id, self.org_id, self.ocm_env))
+
+    def identity_labels(self) -> list[str]:
+        return [
+            f"org_id={self.org_id}",
+            f"cluster_id={self.cluster_id}",
+            f"cluster_name={self.name}",
+        ]
+
+
+LabelState = dict[LabelOwnerRef, dict[str, str]]
+
+
+class LabelSource(ABC):
+    @abstractmethod
+    def get_labels(self) -> LabelState:
+        pass

reconcile/terraform_resources.py

@@ -41,6 +41,7 @@ from reconcile.utils.external_resources import (
     PROVIDER_AWS,
     get_external_resource_specs,
     managed_external_resources,
+    publish_metrics,
 )
 from reconcile.utils.oc import StatusCodeError
 from reconcile.utils.oc_map import (
@@ -399,6 +400,7 @@ def run(
         account_names,
         exclude_accounts,
     )
+    publish_metrics(resource_specs, QONTRACT_INTEGRATION.replace("_", "-"))
 
     if not dry_run and oc_map and defer:
         defer(oc_map.cleanup)

reconcile/utils/external_resource_spec.py

@@ -12,9 +12,11 @@ from typing import (
 )
 
 import yaml
+from pydantic import BaseModel
 from pydantic.dataclasses import dataclass
 
 from reconcile import openshift_resources_base
+from reconcile.utils.metrics import GaugeMetric
 from reconcile.utils.openshift_resource import (
     SECRET_MAX_KEY_LENGTH,
     OpenshiftResource,
@@ -188,6 +190,24 @@ class ExternalResourceUniqueKey:
         )
 
 
+class ExternalResourceBaseMetric(BaseModel):
+    "Base class External Resource metrics"
+
+    integration: str
+
+
+class ExternalResourceInventoryGauge(ExternalResourceBaseMetric, GaugeMetric):
+    "Inventory Gauge"
+
+    provision_provider: str
+    provisioner_name: str
+    provider: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "external_resource_inventory"
+
+
 ExternalResourceSpecInventory = MutableMapping[
     ExternalResourceUniqueKey, ExternalResourceSpec
 ]

reconcile/utils/external_resources.py

@@ -1,4 +1,5 @@
 import json
+from collections import Counter
 from collections.abc import (
     Mapping,
     MutableMapping,
@@ -10,9 +11,16 @@ from typing import (
 
 import anymarkup
 
-from reconcile.utils import gql
+from reconcile.utils import (
+    gql,
+    metrics,
+)
 from reconcile.utils.exceptions import FetchResourceError
-from reconcile.utils.external_resource_spec import ExternalResourceSpec
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceInventoryGauge,
+    ExternalResourceSpec,
+    ExternalResourceSpecInventory,
+)
 
 PROVIDER_AWS = "aws"
 PROVIDER_CLOUDFLARE = "cloudflare"
@@ -61,6 +69,29 @@ def managed_external_resources(namespace_info: Mapping[str, Any]) -> bool:
     return False
 
 
+def get_inventory_count_combinations(
+    inventory: ExternalResourceSpecInventory,
+) -> Counter[tuple]:
+    return Counter(
+        (k.provision_provider, k.provisioner_name, k.provider) for k in inventory
+    )
+
+
+def publish_metrics(inventory: ExternalResourceSpecInventory, integration: str) -> None:
+    count_combinations = get_inventory_count_combinations(inventory)
+    for combination, count in count_combinations.items():
+        provision_provider, provisioner_name, provider = combination
+        metrics.set_gauge(
+            ExternalResourceInventoryGauge(
+                integration=integration,
+                provision_provider=provision_provider,
+                provisioner_name=provisioner_name,
+                provider=provider,
+            ),
+            count,
+        )
+
+
 class ResourceValueResolver:
     """
     ExternalResourceSpec have data that's contained in different fields including the

reconcile/utils/ocm/labels.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from collections.abc import Generator
 from typing import (
     Any,
@@ -11,6 +12,7 @@ from reconcile.utils.ocm.base import (
     OCMLabel,
     OCMOrganizationLabel,
     OCMSubscriptionLabel,
+    build_label_container,
 )
 from reconcile.utils.ocm.search_filters import Filter
 from reconcile.utils.ocm_base_client import OCMBaseClient
@@ -34,10 +36,25 @@ def add_subscription_label(
     ocm_cluster: OCMCluster,
     label: str,
     value: str,
+) -> None:
+    """Add the given label to the cluster subscription."""
+    add_label(
+        ocm_api=ocm_api,
+        label_container_href=f"{ocm_cluster.subscription.href}/labels",
+        label=label,
+        value=value,
+    )
+
+
+def add_label(
+    ocm_api: OCMBaseClient,
+    label_container_href: str,
+    label: str,
+    value: str,
 ) -> None:
     """Add the given label to the cluster subscription."""
     ocm_api.post(
-        api_path=f"{ocm_cluster.subscription.href}/labels",
+        api_path=label_container_href,
         data={"kind": "Label", "key": label, "value": value},
     )
 
@@ -54,11 +71,29 @@ def update_ocm_label(
     )
 
 
+def update_label(
+    ocm_api: OCMBaseClient,
+    label_container_href: str,
+    label: str,
+    value: str,
+) -> None:
+    """Update the label value in the given OCM label."""
+    ocm_api.patch(
+        api_path=f"{label_container_href}/{label}",
+        data={"kind": "Label", "key": label, "value": value},
+    )
+
+
 def delete_ocm_label(ocm_api: OCMBaseClient, ocm_label: OCMLabel) -> None:
     """Delete the given OCM label."""
     ocm_api.delete(api_path=ocm_label.href)
 
 
+def delete_label(ocm_api: OCMBaseClient, label_container_href: str, label: str) -> None:
+    """Delete the given OCM label."""
+    ocm_api.delete(api_path=f"{label_container_href}/{label}")
+
+
 def subscription_label_filter() -> Filter:
     """
     Returns a filter that can be used to find only subscription labels.
@@ -145,3 +180,28 @@ def label_filter(key: str, value: Optional[str] = None) -> Filter:
     if value:
         return lf.eq("value", value)
     return lf
+
+
+def get_org_labels(
+    ocm_api: OCMBaseClient, org_ids: set[str], label_filter: Optional[Filter]
+) -> dict[str, LabelContainer]:
+    """
+    Fetch all labels from organizations. Optionally, label filtering can be
+    performed via the `label_filter` parameter.
+
+    The result is a dict with organization IDs as keys and label containers as values.
+    """
+    filter = Filter().is_in("organization_id", org_ids)
+    if label_filter:
+        filter &= label_filter
+    labels_by_org: dict[str, list[OCMOrganizationLabel]] = defaultdict(list)
+    for label in get_organization_labels(ocm_api, filter):
+        labels_by_org[label.organization_id].append(label)
+    return {
+        org_id: build_label_container(labels)
+        for org_id, labels in labels_by_org.items()
+    }
+
+
+def build_organization_labels_href(org_id: str) -> str:
+    return f"/api/accounts_mgmt/v1/organizations/{org_id}/labels"