qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/utils/ocm/clusters.py

@@ -1,86 +1,45 @@
 from collections import defaultdict
-from enum import Enum
-from typing import (
+from collections.abc import (
     Generator,
-    Optional,
+    Iterable,
 )
-
-from pydantic import BaseModel
-
-from reconcile.utils.ocm.base import OCMModelLink
-from reconcile.utils.ocm.labels import (
-    OCMLabel,
+from typing import Any
+
+from reconcile.utils.ocm.base import (
+    ACTIVE_SUBSCRIPTION_STATES,
+    PRODUCT_ID_OSD,
+    PRODUCT_ID_ROSA,
+    ClusterDetails,
+    FleetManagerServiceCluster,
+    OCMCluster,
+    OCMClusterState,
     OCMOrganizationLabel,
     OCMSubscriptionLabel,
+    ProvisionShard,
+    build_label_container,
+)
+from reconcile.utils.ocm.labels import (
     get_labels,
     get_organization_labels,
 )
 from reconcile.utils.ocm.search_filters import Filter
 from reconcile.utils.ocm.subscriptions import (
-    OCMCapability,
     build_subscription_filter,
     get_subscriptions,
 )
 from reconcile.utils.ocm_base_client import OCMBaseClient
 
-
-class OCMClusterState(Enum):
-    ERROR = "error"
-    HIBERNATING = "hibernating"
-    INSTALLING = "installing"
-    PENING = "pending"
-    POWERING_DOWN = "powering_down"
-    READY = "ready"
-    RESUMING = "resuming"
-    UNINSTALLING = "uninstalling"
-    UNKNOWN = "unknown"
-    VALIDATING = "validating"
-    WAITING = "waiting"
-
-
-class OCMCluster(BaseModel):
-
-    kind: str = "Cluster"
-    id: str
-    external_id: str
-    """
-    This is sometimes also called the cluster UUID.
-    """
-
-    name: str
-    display_name: str
-
-    managed: bool
-
-    openshift_version: str
-    state: OCMClusterState
-
-    subscription: OCMModelLink
-    region: OCMModelLink
-    cloud_provider: OCMModelLink
-    product: OCMModelLink
-
-
-class ClusterDetails(BaseModel):
-
-    ocm_cluster: OCMCluster
-
-    organization_id: str
-    capabilities: dict[str, OCMCapability]
-    """
-    The capabilities of a cluster. They represent feature flags and are
-    found on the subscription of a cluster.
-    """
-
-    subscription_labels: dict[str, OCMSubscriptionLabel]
-    organization_labels: dict[str, OCMOrganizationLabel]
-
-    def get_label(self, name: str) -> Optional[OCMLabel]:
-        if name in self.subscription_labels:
-            return self.subscription_labels[name]
-        if name in self.organization_labels:
-            return self.organization_labels[name]
-        return None
+NODE_POOL_DESIRED_KEYS = {
+    "id",
+    "instance_type",
+    "replicas",
+    "autoscaling",
+    "labels",
+    "taints",
+    "aws_node_pool",
+    "subnet",
+    "version",
+}
 
 
 def discover_clusters_by_labels(
@@ -90,31 +49,47 @@ def discover_clusters_by_labels(
     Discover clusters in OCM by their subscription and organization labels.
     The discovery labels are defined via the label_filter argument.
     """
-    subscription_ids = set()
-    organization_ids = set()
+    subscription_labels: dict[str, list[OCMSubscriptionLabel]] = defaultdict(list)
+    organization_labels: dict[str, list[OCMOrganizationLabel]] = defaultdict(list)
+
     for label in get_labels(ocm_api=ocm_api, filter=label_filter):
         if isinstance(label, OCMSubscriptionLabel):
-            subscription_ids.add(label.subscription_id)
+            subscription_labels[label.subscription_id].append(label)
         elif isinstance(label, OCMOrganizationLabel):
-            organization_ids.add(label.organization_id)
-    sub_id_filter = Filter().is_in("id", subscription_ids)
-    org_id_filter = Filter().is_in("organization_id", organization_ids)
+            organization_labels[label.organization_id].append(label)
+    if not subscription_labels and not organization_labels:
+        return []
+
+    sub_id_filter = Filter().is_in("id", subscription_labels.keys())
+    org_id_filter = Filter().is_in("organization_id", organization_labels.keys())
     subscription_filter = (
         sub_id_filter | org_id_filter  # pylint: disable=unsupported-binary-operation
     )
     # the pylint ignore above is because of a bug in pylint - https://github.com/PyCQA/pylint/issues/7381
-    return list(
+
+    clusters = list(
         get_cluster_details_for_subscriptions(
             ocm_api=ocm_api,
             subscription_filter=subscription_filter,
         )
     )
 
+    # fill in labels
+    for cluster in clusters:
+        cluster.subscription_labels = build_label_container(
+            subscription_labels[cluster.ocm_cluster.subscription.id]
+        )
+        cluster.organization_labels = build_label_container(
+            organization_labels[cluster.organization_id]
+        )
+
+    return clusters
+
 
 def discover_clusters_for_subscriptions(
     ocm_api: OCMBaseClient,
     subscription_ids: list[str],
-    cluster_filter: Optional[Filter] = None,
+    cluster_filter: Filter | None = None,
 ) -> list[ClusterDetails]:
     """
     Discover clusters by filtering on their subscription IDs.
@@ -135,8 +110,8 @@ def discover_clusters_for_subscriptions(
 
 def discover_clusters_for_organizations(
     ocm_api: OCMBaseClient,
-    organization_ids: list[str],
-    cluster_filter: Optional[Filter] = None,
+    organization_ids: Iterable[str],
+    cluster_filter: Filter | None = None,
 ) -> list[ClusterDetails]:
     """
     Discover clusters by filtering on their organization IDs.
@@ -161,16 +136,37 @@ def get_ocm_clusters(
 ) -> Generator[OCMCluster, None, None]:
     for cluster_dict in ocm_api.get_paginated(
         api_path="/api/clusters_mgmt/v1/clusters",
-        params={"search": cluster_filter.render()},
+        params={"search": cluster_filter.render(), "order": "creation_timestamp"},
         max_page_size=100,
     ):
         yield OCMCluster(**cluster_dict)
 
 
+def get_provision_shard_for_cluster_id(
+    ocm_api: OCMBaseClient,
+    id: str,
+) -> ProvisionShard:
+    data = ocm_api.get(api_path=f"/api/clusters_mgmt/v1/clusters/{id}/provision_shard")
+    return ProvisionShard(**data)
+
+
+def get_service_clusters(
+    ocm_api: OCMBaseClient,
+) -> Generator[FleetManagerServiceCluster, None, None]:
+    for cluster_dict in ocm_api.get_paginated(
+        api_path="/api/osd_fleet_mgmt/v1/service_clusters",
+        max_page_size=100,
+    ):
+        if not cluster_dict.get("provision_shard_reference"):
+            continue
+        yield FleetManagerServiceCluster(**cluster_dict)
+
+
 def get_cluster_details_for_subscriptions(
     ocm_api: OCMBaseClient,
-    subscription_filter: Optional[Filter] = None,
-    cluster_filter: Optional[Filter] = None,
+    subscription_filter: Filter | None = None,
+    cluster_filter: Filter | None = None,
+    init_labels: bool = False,
 ) -> Generator[ClusterDetails, None, None]:
     """
     Discover clusters by filtering on their subscriptions. The subscription_filter
@@ -180,21 +176,23 @@ def get_cluster_details_for_subscriptions(
     # get subscription details
     subscriptions = get_subscriptions(
         ocm_api=ocm_api,
-        filter=(subscription_filter or Filter()) & build_subscription_filter(),
+        filter=(subscription_filter or Filter())
+        & build_subscription_filter(states=ACTIVE_SUBSCRIPTION_STATES, managed=True),
     )
     if not subscriptions:
         return
 
     # get organization labels
     organization_labels: dict[str, list[OCMOrganizationLabel]] = defaultdict(list)
-    for label in get_organization_labels(
-        ocm_api=ocm_api,
-        filter=Filter().is_in(
-            "organization_id",
-            {s.organization_id for s in subscriptions.values()},
-        ),
-    ):
-        organization_labels[label.organization_id].append(label)
+    if init_labels:
+        for label in get_organization_labels(
+            ocm_api=ocm_api,
+            filter=Filter().is_in(
+                "organization_id",
+                {s.organization_id for s in subscriptions.values()},
+            ),
+        ):
+            organization_labels[label.organization_id].append(label)
 
     cluster_search_filter = (cluster_ready_for_app_interface() & cluster_filter).is_in(
         "subscription.id", subscriptions.keys()
@@ -211,14 +209,10 @@ def get_cluster_details_for_subscriptions(
                     capability.name: capability
                     for capability in subscription.capabilities or []
                 },
-                subscription_labels={
-                    label.key: label for label in subscription.labels or []
-                },
-                organization_labels={
-                    label.key: label
-                    for label in organization_labels.get(subscription.organization_id)
-                    or []
-                },
+                organization_labels=build_label_container(
+                    organization_labels.get(subscription.organization_id) or []
+                ),
+                subscription_labels=build_label_container(subscription.labels or []),
             )
 
 
@@ -231,5 +225,23 @@ def cluster_ready_for_app_interface() -> Filter:
         Filter()
         .eq("managed", "true")
         .eq("state", OCMClusterState.READY.value)
-        .is_in("product.id", ["osd", "rosa"])
+        .is_in("product.id", [PRODUCT_ID_OSD, PRODUCT_ID_ROSA])
     )
+
+
+def get_node_pools(ocm_api: OCMBaseClient, cluster_id: str) -> list[dict[str, Any]]:
+    results = []
+    api = f"/api/clusters_mgmt/v1/clusters/{cluster_id}/node_pools"
+
+    for item in ocm_api.get_paginated(api):
+        result = {k: v for k, v in item.items() if k in NODE_POOL_DESIRED_KEYS}
+        results.append(result)
+
+    return results
+
+
+def get_provisioning_shard_id(ocm_api: OCMBaseClient, cluster_id: str) -> str:
+    api = f"/api/clusters_mgmt/v1/clusters/{cluster_id}/provision_shard"
+
+    item = ocm_api.get(api)
+    return item["id"]

reconcile/utils/ocm/identity_providers.py

@@ -0,0 +1,66 @@
+from collections.abc import Generator
+
+from reconcile.utils.ocm.base import (
+    OCMCluster,
+    OCMOIdentityProvider,
+    OCMOIdentityProviderGithub,
+    OCMOIdentityProviderOidc,
+)
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+
+def get_identity_providers(
+    ocm_api: OCMBaseClient, ocm_cluster: OCMCluster
+) -> Generator[
+    OCMOIdentityProvider | OCMOIdentityProviderOidc | OCMOIdentityProviderGithub,
+    None,
+    None,
+]:
+    """Get all identity providers."""
+    if ocm_cluster.identity_providers.href:
+        for idp_dict in ocm_api.get_paginated(
+            api_path=ocm_cluster.identity_providers.href
+        ):
+            match idp_dict["type"]:
+                case "OpenIDIdentityProvider":
+                    yield OCMOIdentityProviderOidc(**idp_dict)
+                case "GithubIdentityProvider":
+                    yield OCMOIdentityProviderGithub(**idp_dict)
+                case _:
+                    yield OCMOIdentityProvider(**idp_dict)
+
+
+def add_identity_provider(
+    ocm_api: OCMBaseClient,
+    ocm_cluster: OCMCluster,
+    idp: OCMOIdentityProviderOidc,  # no other IDP types are tested and supported yet
+) -> None:
+    """Creates a new identity provider."""
+    if not ocm_cluster.identity_providers.href:
+        raise ValueError(
+            f"Cluster {ocm_cluster.name} does not support identity providers."
+        )
+    ocm_api.post(
+        api_path=ocm_cluster.identity_providers.href,
+        data=idp.dict(by_alias=True, exclude_none=True),
+    )
+
+
+def update_identity_provider(
+    ocm_api: OCMBaseClient,
+    idp: OCMOIdentityProviderOidc,  # no other IDP types are tested and supported yet
+) -> None:
+    """Creates a new identity provider."""
+    if not idp.href:
+        raise ValueError(f"IDP {idp.name} does not have a href!")
+    ocm_api.patch(
+        api_path=idp.href,
+        data=idp.dict(by_alias=True, exclude_none=True, exclude={"name"}),
+    )
+
+
+def delete_identity_provider(ocm_api: OCMBaseClient, idp: OCMOIdentityProvider) -> None:
+    """Delete a identity provider."""
+    if not idp.href:
+        raise ValueError(f"IDP {idp.name} does not have a href!")
+    ocm_api.delete(api_path=idp.href)

reconcile/utils/ocm/label_sources.py

@@ -0,0 +1,75 @@
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class LabelOwnerRef(ABC):
+    ocm_env: str
+    label_container_href: str | None
+
+    @abstractmethod
+    def identity_labels(self) -> list[str]:
+        pass
+
+    def required_label_container_href(self) -> str:
+        if self.label_container_href is None:
+            raise ValueError(
+                "label_container_href is missing - this method should probably not be called in this state"
+            )
+        return self.label_container_href
+
+
+@dataclass(frozen=True)
+class OrgRef(LabelOwnerRef):
+    org_id: str
+    name: str
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, OrgRef)
+            and self.org_id == other.org_id
+            and self.ocm_env == other.ocm_env
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.org_id, self.ocm_env))
+
+    def identity_labels(self) -> list[str]:
+        return [f"org_id={self.org_id}", f"org_name={self.name}"]
+
+
+@dataclass(frozen=True)
+class ClusterRef(LabelOwnerRef):
+    cluster_id: str
+    org_id: str
+    name: str
+
+    def __eq__(self, other: object) -> bool:
+        return (
+            isinstance(other, ClusterRef)
+            and self.cluster_id == other.cluster_id
+            and self.org_id == other.org_id
+            and self.ocm_env == other.ocm_env
+        )
+
+    def __hash__(self) -> int:
+        return hash((self.cluster_id, self.org_id, self.ocm_env))
+
+    def identity_labels(self) -> list[str]:
+        return [
+            f"org_id={self.org_id}",
+            f"cluster_id={self.cluster_id}",
+            f"cluster_name={self.name}",
+        ]
+
+
+LabelState = dict[LabelOwnerRef, dict[str, str]]
+
+
+class LabelSource(ABC):
+    @abstractmethod
+    def get_labels(self) -> LabelState:
+        pass

reconcile/utils/ocm/labels.py

@@ -1,56 +1,137 @@
-from datetime import datetime
-from typing import (
-    Any,
-    Generator,
-    Optional,
+from collections import defaultdict
+from collections.abc import Generator
+from typing import Any
+
+from reconcile.utils.ocm.base import (
+    LabelContainer,
+    OCMAccountLabel,
+    OCMCluster,
+    OCMLabel,
+    OCMOrganizationLabel,
+    OCMSubscriptionLabel,
+    build_label_container,
 )
-
-from pydantic import BaseModel
-
 from reconcile.utils.ocm.search_filters import Filter
 from reconcile.utils.ocm_base_client import OCMBaseClient
 
 
 def get_subscription_labels(
     ocm_api: OCMBaseClient, filter: Filter
-) -> Generator["OCMSubscriptionLabel", None, None]:
+) -> Generator[OCMSubscriptionLabel, None, None]:
     """
     Finds all subscription labels that match the given filter.
     """
     for subscription_label in get_labels(
-        ocm_api=ocm_api, filter=filter.eq("type", "Subscription")
+        ocm_api=ocm_api, filter=filter & subscription_label_filter()
     ):
         if isinstance(subscription_label, OCMSubscriptionLabel):
             yield subscription_label
 
 
+def add_subscription_label(
+    ocm_api: OCMBaseClient,
+    ocm_cluster: OCMCluster,
+    label: str,
+    value: str,
+) -> None:
+    """Add the given label to the cluster subscription."""
+    add_label(
+        ocm_api=ocm_api,
+        label_container_href=f"{ocm_cluster.subscription.href}/labels",
+        label=label,
+        value=value,
+    )
+
+
+def add_label(
+    ocm_api: OCMBaseClient,
+    label_container_href: str,
+    label: str,
+    value: str,
+) -> None:
+    """Add the given label to the cluster subscription."""
+    ocm_api.post(
+        api_path=label_container_href,
+        data={"kind": "Label", "key": label, "value": value},
+    )
+
+
+def update_ocm_label(
+    ocm_api: OCMBaseClient,
+    ocm_label: OCMLabel,
+    value: str,
+) -> None:
+    """Update the label value in the given OCM label."""
+    ocm_api.patch(
+        api_path=ocm_label.href,
+        data={"kind": "Label", "key": ocm_label.key, "value": value},
+    )
+
+
+def update_label(
+    ocm_api: OCMBaseClient,
+    label_container_href: str,
+    label: str,
+    value: str,
+) -> None:
+    """Update the label value in the given OCM label."""
+    ocm_api.patch(
+        api_path=f"{label_container_href}/{label}",
+        data={"kind": "Label", "key": label, "value": value},
+    )
+
+
+def delete_ocm_label(ocm_api: OCMBaseClient, ocm_label: OCMLabel) -> None:
+    """Delete the given OCM label."""
+    ocm_api.delete(api_path=ocm_label.href)
+
+
+def delete_label(ocm_api: OCMBaseClient, label_container_href: str, label: str) -> None:
+    """Delete the given OCM label."""
+    ocm_api.delete(api_path=f"{label_container_href}/{label}")
+
+
+def subscription_label_filter() -> Filter:
+    """
+    Returns a filter that can be used to find only subscription labels.
+    """
+    return Filter().eq("type", "Subscription")
+
+
 def get_organization_labels(
     ocm_api: OCMBaseClient, filter: Filter
-) -> Generator["OCMOrganizationLabel", None, None]:
+) -> Generator[OCMOrganizationLabel, None, None]:
     """
     Finds all organization labels that match the given filter.
     """
     for org_label in get_labels(
-        ocm_api=ocm_api, filter=filter.eq("type", "Organization")
+        ocm_api=ocm_api, filter=filter & organization_label_filter()
     ):
         if isinstance(org_label, OCMOrganizationLabel):
             yield org_label
 
 
+def organization_label_filter() -> Filter:
+    """
+    Returns a filter that can be used to find only organization labels.
+    """
+    return Filter().eq("type", "Organization")
+
+
 def get_labels(
     ocm_api: OCMBaseClient, filter: Filter
-) -> Generator["OCMLabel", None, None]:
+) -> Generator[OCMLabel, None, None]:
     """
     Finds all labels that match the given filter.
     """
     for label_dict in ocm_api.get_paginated(
         api_path="/api/accounts_mgmt/v1/labels",
-        params={"search": filter.render()},
+        params={"search": filter.render(), "orderBy": "created_at"},
     ):
         yield build_label_from_dict(label_dict)
 
 
-def build_label_from_dict(label_dict: dict[str, Any]) -> "OCMLabel":
+def build_label_from_dict(label_dict: dict[str, Any]) -> OCMLabel:
     """
     Translates a label dict into a type specific label object.
     """
@@ -64,51 +145,30 @@ def build_label_from_dict(label_dict: dict[str, Any]) -> "OCMLabel":
     raise ValueError(f"Unknown label type: {label_dict['type']}")
 
 
-class OCMLabel(BaseModel):
+def build_container_for_prefix(
+    container: LabelContainer, key_prefix: str, strip_key_prefix: bool = False
+) -> "LabelContainer":
     """
-    Represents a general label without any type specific information.
-    See subclasses for type specific information.
+    Builds a new label container with all labels that have the given prefix.
     """
 
-    id: str
-    internal: bool
-    updated_at: datetime
-    created_at: datetime
-    href: str
-    key: str
-    value: str
-    type: str
-    """
-    The type of the label, e.g. Subscription, Organization, Account.
-    See subclasses.
-    """
-
-
-class OCMOrganizationLabel(OCMLabel):
-    """
-    Represents a label attached to an organization.
-    """
-
-    organization_id: str
-
-
-class OCMSubscriptionLabel(OCMLabel):
-    """
-    Represents a label attached to a subscription.
-    """
-
-    subscription_id: str
+    def strip_prefix_if_needed(key: str) -> str:
+        if strip_key_prefix:
+            return key[len(key_prefix) :]
+        return key
 
+    return LabelContainer(
+        labels={
+            strip_prefix_if_needed(label.key): label.copy(
+                update={"key": strip_prefix_if_needed(label.key)}
+            )
+            for label in container.labels.values()
+            if label.key.startswith(key_prefix)
+        }
+    )
 
-class OCMAccountLabel(OCMLabel):
-    """
-    Represents a label attached to an account.
-    """
-
-    account_id: str
 
-
-def label_filter(key: str, value: Optional[str] = None) -> Filter:
+def label_filter(key: str, value: str | None = None) -> Filter:
     """
     Creates a filter that matches a label with the given key and
     optionally a value.
@@ -117,3 +177,28 @@ def label_filter(key: str, value: Optional[str] = None) -> Filter:
     if value:
         return lf.eq("value", value)
     return lf
+
+
+def get_org_labels(
+    ocm_api: OCMBaseClient, org_ids: set[str], label_filter: Filter | None
+) -> dict[str, LabelContainer]:
+    """
+    Fetch all labels from organizations. Optionally, label filtering can be
+    performed via the `label_filter` parameter.
+
+    The result is a dict with organization IDs as keys and label containers as values.
+    """
+    filter = Filter().is_in("organization_id", org_ids)
+    if label_filter:
+        filter &= label_filter
+    labels_by_org: dict[str, list[OCMOrganizationLabel]] = defaultdict(list)
+    for label in get_organization_labels(ocm_api, filter):
+        labels_by_org[label.organization_id].append(label)
+    return {
+        org_id: build_label_container(labels)
+        for org_id, labels in labels_by_org.items()
+    }
+
+
+def build_organization_labels_href(org_id: str) -> str:
+    return f"/api/accounts_mgmt/v1/organizations/{org_id}/labels"