qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/{utils → aus}/cluster_version_data.py

@@ -1,8 +1,8 @@
 import json
+from collections.abc import Iterable
 from datetime import datetime
 from typing import (
     Any,
-    Iterable,
     Optional,
 )
 
@@ -11,6 +11,7 @@ from pydantic import (
     Field,
 )
 
+from reconcile.aus.models import OrganizationUpgradeSpec
 from reconcile.utils.semver_helper import parse_semver
 from reconcile.utils.state import State
 
@@ -92,9 +93,9 @@ class VersionData(BaseModel):
     upgrade policies.
     """
 
-    check_in: Optional[datetime]
+    check_in: datetime | None
     versions: dict[str, VersionHistory] = Field(default_factory=dict)
-    stats: Optional[Stats]
+    stats: Stats | None
 
     def jsondict(self) -> dict[str, Any]:
         return json.loads(self.json(exclude_none=True))
@@ -103,7 +104,7 @@ class VersionData(BaseModel):
         state.add(ocm_name, self.jsondict(), force=True)
 
     def workload_history(
-        self, version: str, workload: str, default: Optional[WorkloadHistory] = None
+        self, version: str, workload: str, default: WorkloadHistory | None = None
     ) -> WorkloadHistory:
         if not default:
             vh = self.versions.get(version, VersionHistory())
@@ -117,12 +118,12 @@ class VersionData(BaseModel):
             workloads.update(v.workloads.keys())
         return workloads
 
-    def update_stats(self, upgrade_policies: list[dict[str, str]]) -> None:
+    def update_stats(self, org_upgrade_spec: OrganizationUpgradeSpec) -> None:
         """Update the versiondata stats with the provided upgrade_policies info"""
         min_version_per_workload: dict[str, str] = {}
-        for item in upgrade_policies:
-            current_version = item["current_version"]
-            for w in item["workloads"]:
+        for spec in org_upgrade_spec.specs:
+            current_version = spec.current_version
+            for w in spec.upgrade_policy.workloads:
                 min_ver = min_version_per_workload.setdefault(w, current_version)
                 if parse_semver(current_version) < parse_semver(min_ver):
                     min_version_per_workload[w] = current_version
@@ -134,7 +135,7 @@ class VersionData(BaseModel):
                 min_version_per_workload=min_version_per_workload,
             )
 
-    def aggregate(self, added: "VersionData", added_org_name: str) -> None:
+    def aggregate(self, added: "VersionData", added_scope: str) -> None:
         """aggregate an other version data with this one.
         this adds new value and merges the ones we we already have
         """
@@ -147,7 +148,7 @@ class VersionData(BaseModel):
                 w = self.workload_history(version, workload, WorkloadHistory())
                 w.soak_days += workload_data.soak_days
                 ocm_clusters = [
-                    f"{added_org_name}/{cluster}" for cluster in workload_data.reporting
+                    f"{added_scope}/{cluster}" for cluster in workload_data.reporting
                 ]
                 w.reporting += ocm_clusters
         if self.stats and added.stats:
@@ -159,6 +160,20 @@ class VersionData(BaseModel):
         return self.stats.validate_against_inherited(version, workloads)
 
 
-def get_version_data(state: State, ocm_name: str) -> VersionData:
-    vd = state.get(ocm_name, {})
+class VersionDataMap:
+    def __init__(self) -> None:
+        self._data: dict[str, VersionData] = {}
+
+    def add(self, ocm_env: str, org_id: str, version_data: VersionData) -> None:
+        self._data[f"{ocm_env}/{org_id}"] = version_data
+
+    def get(self, ocm_env: str, org_id: str) -> VersionData:
+        return self._data[f"{ocm_env}/{org_id}"]
+
+    def items(self) -> Iterable[tuple[str, VersionData]]:
+        return self._data.items()
+
+
+def get_version_data(state: State, key: str) -> VersionData:
+    vd = state.get(key, {})
     return VersionData(**vd)

reconcile/aus/healthchecks.py

@@ -0,0 +1,77 @@
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.fragments.aus_organization import AUSOCMOrganization
+from reconcile.utils.clusterhealth.providerbase import ClusterHealthProvider
+
+
+class AUSHealthError(BaseModel):
+    source: str
+    error: str
+    enforce: bool
+
+
+class AUSClusterHealth(BaseModel):
+    state: dict[str, list[AUSHealthError]]
+
+    def get_errors(self, only_enforced: bool = False) -> list[AUSHealthError]:
+        return [
+            e
+            for errors in self.state.values()
+            for e in errors
+            if not only_enforced or e.enforce
+        ]
+
+    def has_health_errors(self, only_enforced: bool = False) -> bool:
+        return bool(self.get_errors(only_enforced=only_enforced))
+
+    def health_errors_by_source(
+        self, only_enforced: bool = False
+    ) -> dict[str, list[AUSHealthError]]:
+        return {
+            source: [e for e in errors if not only_enforced or e.enforce]
+            for source, errors in self.state.items()
+        }
+
+
+class AUSClusterHealthCheckProvider:
+    def __init__(self) -> None:
+        self.providers: dict[str, tuple[ClusterHealthProvider, bool]] = {}
+
+    def add_provider(
+        self, name: str, provider: ClusterHealthProvider, enforce: bool
+    ) -> "AUSClusterHealthCheckProvider":
+        self.providers[name] = (provider, enforce)
+        return self
+
+    def cluster_health(self, cluster_external_id: str, org_id: str) -> AUSClusterHealth:
+        state: dict[str, list[AUSHealthError]] = {}
+        for provider_name in self.providers:
+            state[provider_name] = []
+            provider, enforce = self.providers[provider_name]
+            health = provider.cluster_health(cluster_external_id, org_id)
+            for error in health.errors or {}:
+                state[provider_name].append(
+                    AUSHealthError(source=provider_name, error=error, enforce=enforce)
+                )
+        return AUSClusterHealth(state=state)
+
+
+def build_cluster_health_providers_for_organization(
+    org: AUSOCMOrganization,
+    providers: dict[str, ClusterHealthProvider],
+) -> AUSClusterHealthCheckProvider:
+    provider = AUSClusterHealthCheckProvider()
+
+    for health_check in org.aus_cluster_health_checks or []:
+        if health_check.provider not in providers:
+            raise Exception(
+                f"organization {org.name} ({org.org_id}) requests health data "
+                f"from an unavailable health check provider {health_check.provider}"
+            )
+        provider.add_provider(
+            name=health_check.provider,
+            provider=providers[health_check.provider],
+            enforce=health_check.enforced,
+        )
+
+    return provider

reconcile/aus/metrics.py

@@ -0,0 +1,158 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    CounterMetric,
+    ErrorRateMetricSet,
+    GaugeMetric,
+    InfoMetric,
+)
+
+
+class AUSBaseMetric(BaseModel):
+    "Base class for AUS metrics"
+
+    integration: str
+    ocm_env: str
+
+
+UPGRADE_SCHEDULED_METRIC_VALUE = -1.0
+UPGRADE_STARTED_METRIC_VALUE = -2.0
+UPGRADE_LONG_RUNNING_METRIC_VALUE = -3.0
+UPGRADE_BLOCKED_METRIC_VALUE = -4.0
+
+CLUSTER_HEALTH_HEALTHY_METRIC_VALUE = 1.0
+CLUSTER_HEALTH_UNHEALTHY_METRIC_VALUE = 0.0
+
+
+class AUSOCMEnvironmentError(AUSBaseMetric, CounterMetric):
+    "Error metric for failed OCM environment interactions"
+
+    ocm_env: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_ocm_environment_errors"
+
+
+class AUSClusterVersionRemainingSoakDaysGauge(AUSBaseMetric, GaugeMetric):
+    "Remaining days a version needs to soak for a cluster"
+
+    cluster_uuid: str
+    soaking_version: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_cluster_version_remaining_soak_days"
+
+
+class AUSClusterHealthStateGauge(AUSBaseMetric, GaugeMetric):
+    "The health state of a cluster determined by AUS cluster health checks.1 is healthy, 0 is unhealthy."
+
+    cluster_uuid: str
+    health_source: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_cluster_health_state"
+
+
+class AUSAddonVersionRemainingSoakDaysGauge(AUSClusterVersionRemainingSoakDaysGauge):
+    "Remaining days a version needs to soak for an addon on a cluster"
+
+    addon: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_addon_version_remaining_soak_days"
+
+
+class AUSClusterUpgradePolicyInfoMetric(AUSBaseMetric, InfoMetric):
+    "Info metric for clusters under AUS upgrade control"
+
+    cluster_uuid: str
+    org_id: str
+    org_name: str
+    channel: str
+    current_version: str
+    cluster_name: str
+    schedule: str
+    sector: str
+    mutexes: str
+    soak_days: str
+    workloads: str
+    product: str
+    hypershift: bool
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_cluster_upgrade_policy_info"
+
+
+class AUSAddonUpgradePolicyInfoMetric(AUSClusterUpgradePolicyInfoMetric):  # pylint: disable=R0901
+    "Info metric for cluster addons under AUS upgrade control"
+
+    addon: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_addon_upgrade_policy_info"
+
+
+class AUSOrganizationValidationErrorsGauge(AUSBaseMetric, GaugeMetric):
+    "Current validation errors within an OCM organization"
+
+    org_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_organization_validation_errors"
+
+
+class AUSOrganizationReconcileCounter(AUSBaseMetric, CounterMetric):
+    "Counter for the number of times an OCM organization was reconciled"
+
+    org_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_organization_reconciled"
+
+
+class AUSOrganizationReconcileErrorCounter(AUSBaseMetric, CounterMetric):
+    "Counter for the failed reconcile runs for an OCM organization"
+
+    org_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_organization_reconcile_errors"
+
+
+class AUSOrganizationErrorRate(ErrorRateMetricSet):
+    "Collection of AUS metrics for an OCM organization"
+
+    def __init__(self, integration: str, org_id: str, ocm_env: str) -> None:
+        super().__init__(
+            counter=AUSOrganizationReconcileCounter(
+                integration=integration,
+                ocm_env=ocm_env,
+                org_id=org_id,
+            ),
+            error_counter=AUSOrganizationReconcileErrorCounter(
+                integration=integration,
+                ocm_env=ocm_env,
+                org_id=org_id,
+            ),
+        )
+
+
+class AUSOrganizationVersionDataGauge(AUSBaseMetric, GaugeMetric):
+    """Gauge for the version data for an OCM organization"""
+
+    org_id: str
+    version: str
+    workload: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "aus_organization_version_data"

reconcile/aus/models.py

@@ -1,18 +1,258 @@
+from __future__ import annotations
+
+import logging
+import re
+from collections.abc import (
+    Iterable,
+    Mapping,
+    Sequence,
+)
+
 from pydantic import (
     BaseModel,
     Field,
+    PrivateAttr,
 )
 
+from reconcile.aus.healthchecks import AUSClusterHealth
 from reconcile.gql_definitions.fragments.aus_organization import AUSOCMOrganization
-from reconcile.gql_definitions.fragments.upgrade_policy import ClusterUpgradePolicy
+from reconcile.gql_definitions.fragments.upgrade_policy import ClusterUpgradePolicyV1
+from reconcile.utils.ocm.addons import OCMAddonInstallation
+from reconcile.utils.ocm.clusters import OCMCluster
+from reconcile.utils.semver_helper import parse_semver
+
+
+class NodePoolSpec(BaseModel):
+    id: str
+    version: str
 
 
 class ClusterUpgradeSpec(BaseModel):
-    name: str
-    ocm: AUSOCMOrganization
-    upgrade_policy: ClusterUpgradePolicy = Field(..., alias="upgradePolicy")
+    """
+    An upgrade spec for a cluster.
+    """
+
+    org: AUSOCMOrganization
+    cluster: OCMCluster
+    upgrade_policy: ClusterUpgradePolicyV1 = Field(..., alias="upgradePolicy")
+    health: AUSClusterHealth
+    node_pools: list[NodePoolSpec] = Field(default_factory=list, alias="nodePools")
+
+    @property
+    def name(self) -> str:
+        return self.cluster.name
+
+    @property
+    def cluster_uuid(self) -> str:
+        return self.cluster.external_id
+
+    @property
+    def current_version(self) -> str:
+        return self.cluster.version.raw_id
+
+    @property
+    def oldest_current_version(self) -> str:
+        """
+        Consider versions in node pools and the cluster itself, find the oldest one.
+        """
+        versions = [np.version for np in self.node_pools] + [self.current_version]
+        return min(versions, key=parse_semver)
+
+    @property
+    def blocked_versions(self) -> set[str]:
+        return set(self.org.blocked_versions or []) | set(
+            self.upgrade_policy.conditions.blocked_versions or []
+        )
+
+    def version_blocked(self, version: str) -> bool:
+        return any(re.search(b, version) for b in self.blocked_versions)
+
+    def get_available_upgrades(self) -> list[str]:
+        return self.cluster.available_upgrades()
+
+    @property
+    def effective_mutexes(self) -> set[str]:
+        mutexes = set(self.upgrade_policy.conditions.mutexes or [])
+        mutexes.add(self.cluster.id)
+        return mutexes
+
+
+class ClusterAddonUpgradeSpec(ClusterUpgradeSpec):
+    addon: OCMAddonInstallation
+
+    def get_available_upgrades(self) -> list[str]:
+        return self.addon.addon_version.available_upgrades
+
+    @property
+    def current_version(self) -> str:
+        return self.addon.addon_version.id
+
+
+class ClusterValidationError(BaseModel):
+    """
+    A validation error for a cluster.
+    """
+
+    cluster_uuid: str
+    messages: list[str]
+
+
+class OrganizationValidationError(BaseModel):
+    """
+    A validation error for a cluster.
+    """
+
+    message: str
 
 
 class OrganizationUpgradeSpec(BaseModel):
+    """
+    Represents all cluster upgrade specs for an OCM organization.
+    """
+
     org: AUSOCMOrganization
-    specs: list[ClusterUpgradeSpec]
+    _specs: list[ClusterUpgradeSpec] = PrivateAttr(default_factory=list)
+    _cluster_errors: dict[str, ClusterValidationError] = PrivateAttr(
+        default_factory=dict
+    )
+    _org_errors: list[OrganizationValidationError] = PrivateAttr(default_factory=list)
+    _sectors: dict[str, Sector] = PrivateAttr(default_factory=dict)
+
+    def __init__(
+        self,
+        org: AUSOCMOrganization,
+        specs: Iterable[ClusterUpgradeSpec] | None = None,
+    ) -> None:
+        super().__init__(org=org)
+
+        # extract sectors
+        self._sectors = {
+            s.name: Sector(org_id=self.org.org_id, name=s.name)
+            for s in self.org.sectors or []
+        }
+
+        # link sector dependencies
+        for s in self.org.sectors or []:
+            self._sectors[s.name].dependencies = [
+                self._sectors[d.name] for d in s.dependencies or []
+            ]
+
+        # validate sectors
+        for sector in self._sectors.values():
+            sector.validate_dependencies()
+
+        # register specs
+        if specs:
+            for spec in specs:
+                self.add_spec(spec)
+
+    @property
+    def sectors(self) -> Mapping[str, Sector]:
+        return self._sectors
+
+    def add_spec(self, spec: ClusterUpgradeSpec) -> None:
+        # add clusters to their sectors
+        if spec.upgrade_policy.conditions.sector:
+            if spec.upgrade_policy.conditions.sector not in self._sectors:
+                self.add_organization_error(
+                    f"sector {spec.upgrade_policy.conditions.sector} not found in organization"
+                )
+                self.add_cluster_error(
+                    spec.cluster_uuid,
+                    f"cluster {spec.cluster_uuid} references unknown sector {spec.upgrade_policy.conditions.sector}",
+                )
+                return
+            self._sectors[spec.upgrade_policy.conditions.sector].add_spec(spec)
+        self._specs.append(spec)
+        self._specs.sort(key=upgrade_spec_sort_key)
+
+    @property
+    def specs(self) -> Sequence[ClusterUpgradeSpec]:
+        return self._specs
+
+    @property
+    def has_validation_errors(self) -> bool:
+        return self.nr_of_validation_errors > 0
+
+    @property
+    def nr_of_validation_errors(self) -> int:
+        return len(self._cluster_errors) + len(self._org_errors)
+
+    @property
+    def cluster_errors(self) -> list[ClusterValidationError]:
+        return list(self._cluster_errors.values())
+
+    @property
+    def organization_errors(self) -> list[OrganizationValidationError]:
+        return list(self._org_errors)
+
+    def add_cluster_error(self, cluster_uuid: str, message: str) -> None:
+        if cluster_uuid not in self._cluster_errors:
+            self._cluster_errors[cluster_uuid] = ClusterValidationError(
+                cluster_uuid=cluster_uuid, messages=[]
+            )
+        self._cluster_errors[cluster_uuid].messages.append(message)
+
+    def add_organization_error(self, message: str) -> None:
+        self._org_errors.append(OrganizationValidationError(message=message))
+
+
+def upgrade_spec_sort_key(spec: ClusterUpgradeSpec) -> tuple:
+    """
+    consider first lower versions and lower soakdays (when versions are equal)
+    """
+    return (
+        parse_semver(spec.current_version),
+        spec.upgrade_policy.conditions.soak_days or 0,
+    )
+
+
+class SectorConfigError(Exception):
+    pass
+
+
+class Sector(BaseModel):
+    name: str
+    dependencies: list[Sector] = Field(default_factory=list)
+    _specs: dict[str, ClusterUpgradeSpec] = PrivateAttr(default_factory=dict)
+
+    def __key(self) -> str:
+        return self.name
+
+    def __hash__(self) -> int:
+        return hash(self.__key())
+
+    def __str__(self) -> str:
+        return self.name
+
+    def add_spec(self, spec: ClusterUpgradeSpec) -> None:
+        self._specs[spec.name] = spec
+
+    @property
+    def specs(self) -> Sequence[ClusterUpgradeSpec]:
+        return list(self._specs.values())
+
+    def set_specs(self, specs: Sequence[ClusterUpgradeSpec]) -> None:
+        self._specs = {spec.name: spec for spec in specs}
+
+    def _iter_dependencies(self) -> Iterable[Sector]:
+        """
+        iterate recursively over all the sector dependencies
+        """
+        logging.debug(f"[{self}] checking dependencies")
+        for dep in self.dependencies or []:
+            if self.name == dep.name:
+                raise SectorConfigError(
+                    f"[{self}] infinite sector dependency loop detected: depending on itself"
+                )
+            yield dep
+            for d in dep._iter_dependencies():
+                if self.name == d.name:
+                    raise SectorConfigError(
+                        f"[{self}] infinite sector dependency loop detected under {dep} dependencies"
+                    )
+                yield d
+
+    def validate_dependencies(self) -> bool:
+        list(self._iter_dependencies())
+        return True

reconcile/aus/node_pool_spec.py

@@ -0,0 +1,35 @@
+from collections.abc import Iterable, Mapping
+
+from reconcile.aus.models import NodePoolSpec
+from reconcile.utils.ocm import get_node_pools
+from reconcile.utils.ocm.base import ClusterDetails
+from reconcile.utils.ocm_base_client import OCMBaseClient
+
+
+def get_node_pool_specs(ocm_api: OCMBaseClient, cluster_id: str) -> list[NodePoolSpec]:
+    node_pools = get_node_pools(ocm_api, cluster_id)
+    return [
+        NodePoolSpec(
+            id=p["id"],
+            version=p["version"]["raw_id"],
+        )
+        for p in node_pools
+    ]
+
+
+def get_node_pool_specs_by_org_cluster(
+    ocm_api: OCMBaseClient,
+    clusters_by_org: Mapping[str, Iterable[ClusterDetails]],
+) -> dict[str, dict[str, list[NodePoolSpec]]]:
+    """
+    Fetch node pool specs for all rosa hypershift clusters
+    Returns a dict with org IDs as keys, the values are dicts with cluster id as key
+    """
+    return {
+        org_id: {
+            c.ocm_cluster.id: get_node_pool_specs(ocm_api, c.ocm_cluster.id)
+            for c in clusters
+            if c.ocm_cluster.is_rosa_hypershift()
+        }
+        for org_id, clusters in clusters_by_org.items()
+    }