qontract-reconcile 0.10.2.dev131__tar.gz → 0.10.2.dev133__tar.gz

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev131
+Version: 0.10.2.dev133
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/aus/advanced_upgrade_service.py

@@ -289,6 +289,10 @@ class OrganizationLabelSet(BaseModel):
 
     blocked_versions: CSV | None = Field(alias=aus_label_key("blocked-versions"))
 
+    sector_max_parallel_upgrades: dict[str, str] = labelset_groupfield(
+        group_prefix=aus_label_key("sector-max-parallel-upgrades.")
+    )
+
     sector_deps: dict[str, CSV] = labelset_groupfield(
         group_prefix=aus_label_key("sector-deps.")
     )
@@ -311,6 +315,7 @@ class OrganizationLabelSet(BaseModel):
         return [
             OpenShiftClusterManagerSectorV1(
                 name=s,
+                maxParallelUpgrades=self.sector_max_parallel_upgrades.get(s),
                 dependencies=[
                     OpenShiftClusterManagerSectorDependenciesV1(name=d, ocm=None)
                     for d in self.sector_deps.get(s, [])

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/aus/aus_label_source.py

@@ -88,8 +88,12 @@ class AUSOrganizationLabelSource(LabelSource):
             )
         # sector dependencies
         for sector in organization.sectors or []:
-            if sector.dependencies:
-                labels[aus_label_key(f"sectors.{sector.name}")] = ",".join(
+            if sector.max_parallel_upgrades:
+                labels[aus_label_key(f"sector-max-parallel-upgrades.{sector.name}")] = (
+                    sector.max_parallel_upgrades
+                )
+            if sector.dependencies is not None:
+                labels[aus_label_key(f"sector-deps.{sector.name}")] = ",".join(
                     sorted([dep.name for dep in sector.dependencies])
                 )
         # version-data sharing

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/aus/base.py

@@ -5,6 +5,7 @@ from abc import (
     ABC,
     abstractmethod,
 )
+from collections import defaultdict
 from collections.abc import Callable, Sequence
 from datetime import (
     datetime,
@@ -982,6 +983,47 @@ def verify_lock_should_skip(
     return False
 
 
+def verify_max_upgrades_should_skip(
+    desired: ClusterUpgradeSpec,
+    sector_upgrades: dict[str, set[str]],
+    sector: Sector | None,
+) -> bool:
+    if sector is None:
+        return False
+
+    current_upgrades = sector_upgrades[sector.name]
+    # Allow at least one upgrade
+    if len(current_upgrades) == 0:
+        return False
+
+    # if sector.max_parallel_upgrades is not set, we allow all upgrades
+    if sector.max_parallel_upgrades is None:
+        return False
+
+    sector_cluster_count = len(sector.specs)
+
+    if sector.max_parallel_upgrades.endswith("%"):
+        max_parallel_upgrades_percent = int(sector.max_parallel_upgrades[:-1])
+        max_parallel_upgrades = round(
+            sector_cluster_count * max_parallel_upgrades_percent / 100
+        )
+    else:
+        max_parallel_upgrades = int(sector.max_parallel_upgrades)
+
+    # we allow at least one upgrade
+    if max_parallel_upgrades == 0:
+        max_parallel_upgrades = 1
+
+    if len(current_upgrades) >= max_parallel_upgrades:
+        logging.debug(
+            f"[{desired.org.org_id}/{desired.org.name}/{desired.cluster.name}] skipping cluster: "
+            f"sector '{sector.name}' has reached max parallel upgrades {sector.max_parallel_upgrades}"
+        )
+        return True
+
+    return False
+
+
 def _create_upgrade_policy(
     next_schedule: str, spec: ClusterUpgradeSpec, version: str
 ) -> AbstractUpgradePolicy:
@@ -1042,25 +1084,32 @@ def calculate_diff(
         list: upgrade policies to be applied
     """
 
-    def set_mutex(
-        locked: dict[str, str], cluster_id: str, mutexes: set[str] | None = None
+    locked: dict[str, str] = {}
+    sector_upgrades: dict[str, set[str]] = defaultdict(set)
+
+    def set_upgrading(
+        cluster_id: str, mutexes: set[str] | None, sector_name: str | None
     ) -> None:
         for mutex in mutexes or set():
             locked[mutex] = cluster_id
+        if sector_name:
+            sector_upgrades[sector_name].add(cluster_id)
 
     diffs: list[UpgradePolicyHandler] = []
 
     # all clusters IDs with a current upgradePolicy are considered locked
-    locked: dict[str, str] = {}
     for spec in desired_state.specs:
         if spec.cluster.id in [s.cluster.id for s in current_state]:
-            for mutex in spec.effective_mutexes:
-                locked[mutex] = spec.cluster.id
+            sector_name = spec.upgrade_policy.conditions.sector
+            set_upgrading(spec.cluster.id, spec.effective_mutexes, sector_name)
 
     addon_service = init_addon_service(desired_state.org.environment)
     now = datetime.utcnow()
     gates = get_version_gates(ocm_api)
     for spec in desired_state.specs:
+        sector_name = spec.upgrade_policy.conditions.sector
+        sector = desired_state.sectors[sector_name] if sector_name else None
+
         # Upgrading node pools, only required for Hypershift clusters
         # do this in the same loop, to skip cluster on node pool upgrade
         if spec.cluster.is_rosa_hypershift():
@@ -1070,7 +1119,7 @@ def calculate_diff(
             node_pool_update = _calculate_node_pool_diffs(spec, now)
             if node_pool_update:  # node pool update policy not yet created
                 diffs.append(node_pool_update)
-                set_mutex(locked, spec.cluster.id, spec.effective_mutexes)
+                set_upgrading(spec.cluster.id, spec.effective_mutexes, sector_name)
                 continue
 
         # ignore clusters with an existing upgrade policy
@@ -1089,10 +1138,9 @@ def calculate_diff(
         if verify_lock_should_skip(spec, locked):
             continue
 
-        sector_name = spec.upgrade_policy.conditions.sector
-        sector = None
-        if sector_name:
-            sector = desired_state.sectors[sector_name]
+        if verify_max_upgrades_should_skip(spec, sector_upgrades, sector):
+            continue
+
         version = upgradeable_version(spec, version_data, sector)
         if version:
             if addon_id:
@@ -1139,7 +1187,7 @@ def calculate_diff(
                         policy=_create_upgrade_policy(next_schedule, spec, version),
                     )
                 )
-            set_mutex(locked, spec.cluster.id, spec.effective_mutexes)
+            set_upgrading(spec.cluster.id, spec.effective_mutexes, sector_name)
 
     return diffs
 

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/aus/models.py

@@ -137,7 +137,11 @@ class OrganizationUpgradeSpec(BaseModel):
 
         # extract sectors
         self._sectors = {
-            s.name: Sector(org_id=self.org.org_id, name=s.name)
+            s.name: Sector(
+                org_id=self.org.org_id,
+                name=s.name,
+                max_parallel_upgrades=s.max_parallel_upgrades,
+            )
             for s in self.org.sectors or []
         }
 
@@ -223,6 +227,7 @@ class SectorConfigError(Exception):
 
 class Sector(BaseModel):
     name: str
+    max_parallel_upgrades: str | None
     dependencies: list[Sector] = Field(default_factory=list)
     _specs: dict[str, ClusterUpgradeSpec] = PrivateAttr(default_factory=dict)
 

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/external_resources/aws.py

@@ -168,7 +168,9 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
             and (target := blue_green_deployment.get("target"))
             and (parameter_group := target.get("parameter_group"))
         ):
-            target["parameter_group"] = rvr._get_values(parameter_group)
+            data["blue_green_deployment"] = blue_green_deployment | {
+                "target": target | {"parameter_group": rvr._get_values(parameter_group)}
+            }
         if "replica_source" in data:
             sourcedb_spec = self._get_source_db_spec(
                 spec.provisioner_name, data["replica_source"]

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/gql_definitions/advanced_upgrade_service/aus_clusters.py

@@ -66,6 +66,7 @@ fragment AUSOCMOrganization on OpenShiftClusterManager_v1 {
   }
   sectors {
     name
+    maxParallelUpgrades
     dependencies {
       name
       ocm {

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/gql_definitions/advanced_upgrade_service/aus_organization.py

@@ -65,6 +65,7 @@ fragment AUSOCMOrganization on OpenShiftClusterManager_v1 {
   }
   sectors {
     name
+    maxParallelUpgrades
     dependencies {
       name
       ocm {

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/gql_definitions/common/clusters.py

@@ -148,6 +148,7 @@ query Clusters($name: String) {
       }
       sectors {
         name
+        maxParallelUpgrades
         dependencies {
           name
           ocm {
@@ -388,6 +389,7 @@ class OpenShiftClusterManagerSectorDependenciesV1(ConfiguredBaseModel):
 
 class OpenShiftClusterManagerSectorV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    max_parallel_upgrades: Optional[str] = Field(..., alias="maxParallelUpgrades")
     dependencies: Optional[list[OpenShiftClusterManagerSectorDependenciesV1]] = Field(..., alias="dependencies")
 
 

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/gql_definitions/fragments/aus_organization.py

@@ -67,6 +67,7 @@ class OpenShiftClusterManagerSectorDependenciesV1(ConfiguredBaseModel):
 
 class OpenShiftClusterManagerSectorV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    max_parallel_upgrades: Optional[str] = Field(..., alias="maxParallelUpgrades")
     dependencies: Optional[list[OpenShiftClusterManagerSectorDependenciesV1]] = Field(..., alias="dependencies")
 
 

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/gql_definitions/introspection.json

@@ -8079,6 +8079,18 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
+                        {
+                            "name": "maxParallelUpgrades",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "SCALAR",
+                                "name": "String",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
                         {
                             "name": "dependencies",
                             "description": null,

{qontract_reconcile-0.10.2.dev131 → qontract_reconcile-0.10.2.dev133}/reconcile/queries.py

@@ -739,6 +739,7 @@ CLUSTERS_QUERY = """
       }
       sectors {
         name
+        maxParallelUpgrades
         dependencies {
           name
         }
@@ -1348,6 +1349,7 @@ OCM_QUERY = """
     }
     sectors {
       name
+      maxParallelUpgrades
       dependencies {
         name
         ocm {