qontract-reconcile 0.10.2.dev90__py3-none-any.whl → 0.10.2.dev92__py3-none-any.whl

{qontract_reconcile-0.10.2.dev90.dist-info → qontract_reconcile-0.10.2.dev92.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev90
+Version: 0.10.2.dev92
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev90.dist-info → qontract_reconcile-0.10.2.dev92.dist-info}/RECORD

@@ -208,11 +208,11 @@ reconcile/external_resources/secrets_sync.py,sha256=ZDxzGZ6wC4zxLhA7-L39xDRH6rzU
 reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
 reconcile/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/fleet_labeler/dependencies.py,sha256=ZOpmtwgxEPBWU2yHRc6rhPwlvqhwYSsNMJQ_jVq1dLI,2993
-reconcile/fleet_labeler/integration.py,sha256=GR-LUDfDp29VNy1aSLun1UwEOQ1AAXXC18ZdngdCsRg,8976
+reconcile/fleet_labeler/integration.py,sha256=IeDUDlIIZREQRbsODB9xsCMoNrMk5738OF_fP7lUGFs,13128
 reconcile/fleet_labeler/merge_request.py,sha256=SfGxXInxeJzVnsTtO0ZC9-PesUJMdpKxKY9eCB6ms-g,1538
-reconcile/fleet_labeler/meta.py,sha256=DF7O4T9wvQ7-xTWXvuNw1OG_F0SBmRrjFBtVy9wWh9U,146
+reconcile/fleet_labeler/meta.py,sha256=lWnpH2U0PHCPXu9Ok_CPmO494qQJQ5pOuqo28s0jzIQ,146
 reconcile/fleet_labeler/metrics.py,sha256=wx9BmXLsN67m-aSsf81iB7Ehj5SzUsS2WB75isUReZg,662
-reconcile/fleet_labeler/ocm.py,sha256=25UVRQDcbcJFQgg4WDXvUI5M8w9yqr0noiy90-sbQjY,2699
+reconcile/fleet_labeler/ocm.py,sha256=qcg1_p7nKlZG7-MQeOZos3rz6YSPAPh-HKxE3OVJwe0,4165
 reconcile/fleet_labeler/validate.py,sha256=gzc2tt7h9F60h7dcyJfEmsnjnfuux5Jtc_WzrIqr-5k,2541
 reconcile/fleet_labeler/vcs.py,sha256=6UHUQ08AGAHXF7629I6X-T_E1pvx96LxjS66EeOzve4,1108
 reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
@@ -786,7 +786,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev90.dist-info/METADATA,sha256=ITNNwFXdctUPEAooDO8HTV-cc3Z7izzfNugSWuoAFwU,24565
-qontract_reconcile-0.10.2.dev90.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev90.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev90.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev92.dist-info/METADATA,sha256=jLJhRpJR82Z3TVBJRPKXiwDJF_rs9_Vz9L9aaajBipo,24565
+qontract_reconcile-0.10.2.dev92.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev92.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev92.dist-info/RECORD,,

reconcile/fleet_labeler/integration.py

@@ -26,6 +26,7 @@ from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
 from reconcile.utils import (
     metrics,
 )
+from reconcile.utils.differ import diff_mappings
 from reconcile.utils.jinja2.utils import process_jinja2_template
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.runtime.integration import (
@@ -34,6 +35,18 @@ from reconcile.utils.runtime.integration import (
 )
 
 
+class ClusterData(BaseModel):
+    """
+    Helper structure for synching process
+    """
+
+    name: str
+    server_url: str
+    subscription_id: str
+    desired_label_default: FleetLabelDefaultV1
+    current_subscription_labels: dict[str, str]
+
+
 class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
     def __init__(self) -> None:
         super().__init__(NoParams())
@@ -59,13 +72,61 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
     def reconcile(self, dependencies: Dependencies) -> None:
         validate_label_specs(specs=dependencies.label_specs_by_name)
         for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
+            spec = dependencies.label_specs_by_name[spec_name]
+            discovered_clusters = self._discover_desired_clusters(
+                spec=spec,
+                ocm=ocm,
+            )
+            all_desired_clusters = {
+                k: v[0] for k, v in discovered_clusters.items() if len(v) == 1
+            }
+            clusters_with_duplicate_matches = {
+                k: v for k, v in discovered_clusters.items() if len(v) > 1
+            }
             self._sync_cluster_inventory(
                 ocm=ocm,
                 spec=dependencies.label_specs_by_name[spec_name],
                 vcs=dependencies.vcs,
+                all_desired_clusters=all_desired_clusters,
+                clusters_with_duplicate_matches=clusters_with_duplicate_matches,
+                dry_run=dependencies.dry_run,
+            )
+            self._sync_subscription_labels(
+                spec=spec,
+                desired_clusters=all_desired_clusters,
+                ocm=ocm,
                 dry_run=dependencies.dry_run,
             )
 
+    def _discover_desired_clusters(
+        self, spec: FleetLabelsSpecV1, ocm: OCMClient
+    ) -> dict[str, list[ClusterData]]:
+        clusters: dict[str, list[ClusterData]] = defaultdict(list)
+        for label_default in spec.label_defaults:
+            match_subscription_labels = dict(label_default.match_subscription_labels)
+            for cluster in ocm.discover_clusters_by_labels(
+                labels=match_subscription_labels,
+                managed_prefix=spec.managed_subscription_label_prefix,
+            ):
+                # Note, due to the nature of how our label filtering works (see ocm.py), we
+                # also fetch clusters that do not match the filter label.
+                # Here, we filter the clusters on client side.
+                # TODO: move this into utils.ocm module
+                if (
+                    match_subscription_labels.items()
+                    <= cluster.subscription_labels.items()
+                ):
+                    clusters[cluster.cluster_id].append(
+                        ClusterData(
+                            subscription_id=cluster.subscription_id,
+                            desired_label_default=label_default,
+                            name=cluster.name,
+                            server_url=cluster.server_url,
+                            current_subscription_labels=cluster.subscription_labels,
+                        )
+                    )
+        return clusters
+
     def _render_default_labels(
         self,
         template: FleetSubscriptionLabelTemplateV1,
@@ -115,63 +176,91 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
             yml.dump(content, stream)
             return stream.getvalue()
 
+    def _sync_subscription_labels(
+        self,
+        spec: FleetLabelsSpecV1,
+        desired_clusters: dict[str, ClusterData],
+        ocm: OCMClient,
+        dry_run: bool,
+    ) -> None:
+        """
+        Synchronize subscription labels for clusters in the spec's inventory.
+        Note, that we only update labels for clusters which are also part of the
+        discovered desired state.
+        I.e., we only operate on clusters that are in both, current state and desired state.
+        That way we ensure we do not work on deleted clusters and still synch only on
+        whats written in the rendered spec yet.
+        """
+        for cluster in spec.clusters:
+            if not desired_clusters.get(cluster.cluster_id):
+                # The cluster is not part of the desired inventory (will be updated with MR soon)
+                continue
+            # Ensure we only handle labels for our managed prefix
+            current_subscription_labels = {
+                k: v
+                for k, v in desired_clusters[
+                    cluster.cluster_id
+                ].current_subscription_labels.items()
+                if k.startswith(spec.managed_subscription_label_prefix)
+            }
+            desired_subscription_labels = {
+                f"{spec.managed_subscription_label_prefix}.{k}": v
+                for k, v in dict(cluster.subscription_labels).items()
+            }
+            diff = diff_mappings(
+                current=current_subscription_labels,
+                desired=desired_subscription_labels,
+            )
+            for key in diff.add:
+                value = desired_subscription_labels[key]
+                logging.info(
+                    f"[{spec.name}] Adding label '{key}={value}' for cluster '{cluster.cluster_id}' in subscription '{cluster.subscription_id}'."
+                )
+                if not dry_run:
+                    ocm.add_subscription_label(
+                        subscription_id=cluster.subscription_id,
+                        key=key,
+                        value=value,
+                    )
+            for key in diff.change:
+                value = desired_subscription_labels[key]
+                logging.info(
+                    f"[{spec.name}] Updating label '{key}={value}' for cluster '{cluster.cluster_id}' in subscription '{cluster.subscription_id}'."
+                )
+                if not dry_run:
+                    ocm.update_subscription_label(
+                        subscription_id=cluster.subscription_id,
+                        key=key,
+                        value=value,
+                    )
+            # Note, we dont want to enable removal for now - its too dangerous on a broad managed prefix
+            # However, if it is needed in the future, we could easily add it here.
+
     def _sync_cluster_inventory(
         self,
         ocm: OCMClient,
         spec: FleetLabelsSpecV1,
         vcs: VCS,
+        clusters_with_duplicate_matches: dict[str, list[ClusterData]],
+        all_desired_clusters: dict[str, ClusterData],
         dry_run: bool,
     ) -> None:
-        class ClusterData(BaseModel):
-            """
-            Helper structure for synching process
-            """
-
-            name: str
-            server_url: str
-            subscription_id: str
-            label_default: FleetLabelDefaultV1
-
         all_current_cluster_ids = {cluster.cluster_id for cluster in spec.clusters}
-        clusters: dict[str, list[ClusterData]] = defaultdict(list)
-        for label_default in spec.label_defaults:
-            match_subscription_labels = dict(label_default.match_subscription_labels)
-            for cluster in ocm.discover_clusters_by_labels(
-                labels=match_subscription_labels
-            ):
-                # TODO: ideally we filter on server side - see TODO in ocm.py
-                if (
-                    match_subscription_labels.items()
-                    <= cluster.subscription_labels.items()
-                ):
-                    clusters[cluster.cluster_id].append(
-                        ClusterData(
-                            label_default=label_default,
-                            subscription_id=cluster.subscription_id,
-                            name=cluster.name,
-                            server_url=cluster.server_url,
-                        )
-                    )
-
-        cluster_with_duplicate_matches = {
-            k: v for k, v in clusters.items() if len(v) > 1
-        }
-        for cluster_id, matches in cluster_with_duplicate_matches.items():
+        for cluster_id, matches in clusters_with_duplicate_matches.items():
             label_matches = "\n".join(
-                str(m.label_default.match_subscription_labels) for m in matches
+                str(m.desired_label_default.match_subscription_labels) for m in matches
             )
             logging.error(
-                f"Spec '{spec.name}': Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
+                f"[{spec.name}] Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
             )
         metrics.set_gauge(
             FleetLabelerDuplicateClusterMatchesGauge(
                 integration=self.name,
                 ocm_name=spec.ocm.name,
             ),
-            len(cluster_with_duplicate_matches),
+            len(clusters_with_duplicate_matches),
         )
 
-        all_desired_clusters = {k: v[0] for k, v in clusters.items() if len(v) == 1}
         clusters_to_add = [
             YamlCluster(
                 cluster_id=cluster_id,
@@ -179,7 +268,7 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
                 name=cluster_info.name,
                 server_url=cluster_info.server_url,
                 subscription_labels_content=self._render_default_labels(
-                    template=cluster_info.label_default.subscription_label_template,
+                    template=cluster_info.desired_label_default.subscription_label_template,
                     labels=ocm.get_cluster_labels(cluster_id=cluster_id),
                 ),
             )
@@ -223,4 +312,5 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
         desired_content = self._render_yaml_file(
             current_content, cluster_ids_to_delete, sorted_clusters_to_add
         )
-        vcs.open_merge_request(path=f"data{spec.path}", content=desired_content)
+        if not dry_run:
+            vcs.open_merge_request(path=f"data{spec.path}", content=desired_content)

reconcile/fleet_labeler/meta.py

@@ -1,4 +1,4 @@
 from reconcile.utils.semver_helper import make_semver
 
 QONTRACT_INTEGRATION = "fleet-labeler"
-QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)

reconcile/fleet_labeler/ocm.py

@@ -10,7 +10,9 @@ from reconcile.utils.ocm.clusters import (
     discover_clusters_by_labels,
 )
 from reconcile.utils.ocm.labels import (
+    add_label,
     get_cluster_labels_for_cluster_id,
+    update_label,
 )
 from reconcile.utils.ocm.search_filters import Filter, FilterMode
 from reconcile.utils.ocm_base_client import (
@@ -24,9 +26,9 @@ Thin abstractions of reconcile.ocm module to reduce coupling.
 
 class Cluster(BaseModel):
     cluster_id: str
+    subscription_id: str
     server_url: str
     name: str
-    subscription_id: str
     subscription_labels: dict[str, str]
 
     @staticmethod
@@ -69,19 +71,56 @@ class OCMClient:
     def __init__(self, ocm_client: OCMBaseClient):
         self._ocm_client = ocm_client
 
-    def discover_clusters_by_labels(self, labels: Mapping[str, str]) -> list[Cluster]:
+    def discover_clusters_by_labels(
+        self, labels: Mapping[str, str], managed_prefix: str
+    ) -> list[Cluster]:
         label_filter = Filter(mode=FilterMode.AND).eq("type", "Subscription")
         for key in labels:
             label_filter = label_filter.eq("Key", key)
-        # TODO: This throws 400 bad request
-        # for k, v in labels.items():
-        #     label_filter = label_filter.eq(k, v)
+        desired_labels_filter = (
+            Filter(mode=FilterMode.AND)
+            .eq("type", "Subscription")
+            .like(
+                "key",
+                f"{managed_prefix}.%",
+            )
+        )
+        # Note, that discover_clusters_by_labels() only fetches labels that are matching the filter!
+        # However, to understand current state, we also want to know the labels under managed_prefix.
+        # I.e., we need a fetch condition such as:
+        # ("{managed_prefix}.%") OR ("{key1_filter}" AND "{key2_filter}" ....)
+        # The above returns also clusters that do not fit the match label.
+        # I.e., we must filter the result on client side.
+        # TODO: do this in utils.ocm module
+        desired_filter = label_filter | desired_labels_filter
         return [
             Cluster.from_cluster_details(cluster)
             for cluster in discover_clusters_by_labels(
-                ocm_api=self._ocm_client, label_filter=label_filter
+                ocm_api=self._ocm_client, label_filter=desired_filter
             )
         ]
 
     def get_cluster_labels(self, cluster_id: str) -> dict[str, str]:
         return get_cluster_labels_for_cluster_id(self._ocm_client, cluster_id)
+
+    def add_subscription_label(
+        self, subscription_id: str, key: str, value: str
+    ) -> None:
+        # TODO: move href into a utils function
+        add_label(
+            ocm_api=self._ocm_client,
+            label_container_href=f"/api/accounts_mgmt/v1/subscriptions/{subscription_id}/labels",
+            label=key,
+            value=value,
+        )
+
+    def update_subscription_label(
+        self, subscription_id: str, key: str, value: str
+    ) -> None:
+        # TODO: move href into a utils function
+        update_label(
+            ocm_api=self._ocm_client,
+            label_container_href=f"/api/accounts_mgmt/v1/subscriptions/{subscription_id}/labels",
+            label=key,
+            value=value,
+        )