qontract-reconcile 0.10.2.dev89__py3-none-any.whl → 0.10.2.dev91__py3-none-any.whl

{qontract_reconcile-0.10.2.dev89.dist-info → qontract_reconcile-0.10.2.dev91.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev89
+Version: 0.10.2.dev91
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev89.dist-info → qontract_reconcile-0.10.2.dev91.dist-info}/RECORD

@@ -208,11 +208,11 @@ reconcile/external_resources/secrets_sync.py,sha256=ZDxzGZ6wC4zxLhA7-L39xDRH6rzU
 reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
 reconcile/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/fleet_labeler/dependencies.py,sha256=ZOpmtwgxEPBWU2yHRc6rhPwlvqhwYSsNMJQ_jVq1dLI,2993
-reconcile/fleet_labeler/integration.py,sha256=jm2wvkxsxqGLsX_vE874crgce_YsDWyggL3Q3olMNJM,8753
-reconcile/fleet_labeler/merge_request.py,sha256=VA_XSIob4LWYb02dRUr9coXjIa8_0prObI1v5Gqi_mY,1513
-reconcile/fleet_labeler/meta.py,sha256=DF7O4T9wvQ7-xTWXvuNw1OG_F0SBmRrjFBtVy9wWh9U,146
+reconcile/fleet_labeler/integration.py,sha256=x-vfUcWRCI8-_2O1mdEdSbqGM51IwEBgUUKQPlu5_TM,12849
+reconcile/fleet_labeler/merge_request.py,sha256=SfGxXInxeJzVnsTtO0ZC9-PesUJMdpKxKY9eCB6ms-g,1538
+reconcile/fleet_labeler/meta.py,sha256=lWnpH2U0PHCPXu9Ok_CPmO494qQJQ5pOuqo28s0jzIQ,146
 reconcile/fleet_labeler/metrics.py,sha256=wx9BmXLsN67m-aSsf81iB7Ehj5SzUsS2WB75isUReZg,662
-reconcile/fleet_labeler/ocm.py,sha256=GGsz-bq1g8BJVVMCfI2kSwZCyngbQoZ3i3k8fO608KA,2506
+reconcile/fleet_labeler/ocm.py,sha256=1VkUZtgcoqg-F9r--VtTg2LReumkOcRBIPhj-yTCKE0,3470
 reconcile/fleet_labeler/validate.py,sha256=gzc2tt7h9F60h7dcyJfEmsnjnfuux5Jtc_WzrIqr-5k,2541
 reconcile/fleet_labeler/vcs.py,sha256=6UHUQ08AGAHXF7629I6X-T_E1pvx96LxjS66EeOzve4,1108
 reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
@@ -224,7 +224,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
 reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
 reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/introspection.json,sha256=fhnOfg-0gFKOL6yrnI5JJE0w9khgA8ivmEzH_gmDajw,2238549
+reconcile/gql_definitions/introspection.json,sha256=nBoWZM_ilrKlbCUF9cbRe7D1jsTqHqfGkLQW2gbleBM,2239254
 reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
 reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -314,7 +314,7 @@ reconcile/gql_definitions/external_resources/external_resources_settings.py,sha2
 reconcile/gql_definitions/external_resources/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py,sha256=T_qWCRtzU8F9frebBXG9TkeQdrKGt3R9YinSngPoFqM,1262
 reconcile/gql_definitions/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=TGpc-NYm2qnURHigCppUZRY1WWaIqA3E_69BWyni1RQ,4323
+reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=rII_VzEfjGibY1lJXMBMKdCeV7kaNyRXk9-J9-neBoU,4412
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
 reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -786,7 +786,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev89.dist-info/METADATA,sha256=qs4cK6oBZn2zwikJef7d6Z1khsXcc-k_Izwl1NWgO6o,24565
-qontract_reconcile-0.10.2.dev89.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev89.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev89.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev91.dist-info/METADATA,sha256=Zb9JTF0crcCR37ioM_rENLPJWgXh1_1IyFQGxR8kqx8,24565
+qontract_reconcile-0.10.2.dev91.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev91.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev91.dist-info/RECORD,,

reconcile/fleet_labeler/integration.py

@@ -26,6 +26,7 @@ from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
 from reconcile.utils import (
     metrics,
 )
+from reconcile.utils.differ import diff_mappings
 from reconcile.utils.jinja2.utils import process_jinja2_template
 from reconcile.utils.ruamel import create_ruamel_instance
 from reconcile.utils.runtime.integration import (
@@ -34,6 +35,18 @@ from reconcile.utils.runtime.integration import (
 )
 
 
+class ClusterData(BaseModel):
+    """
+    Helper structure for synching process
+    """
+
+    name: str
+    server_url: str
+    subscription_id: str
+    desired_label_default: FleetLabelDefaultV1
+    current_subscription_labels: dict[str, str]
+
+
 class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
     def __init__(self) -> None:
         super().__init__(NoParams())
@@ -59,13 +72,57 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
     def reconcile(self, dependencies: Dependencies) -> None:
         validate_label_specs(specs=dependencies.label_specs_by_name)
         for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
+            spec = dependencies.label_specs_by_name[spec_name]
+            discovered_clusters = self._discover_desired_clusters(
+                spec=spec,
+                ocm=ocm,
+            )
+            all_desired_clusters = {
+                k: v[0] for k, v in discovered_clusters.items() if len(v) == 1
+            }
+            clusters_with_duplicate_matches = {
+                k: v for k, v in discovered_clusters.items() if len(v) > 1
+            }
             self._sync_cluster_inventory(
                 ocm=ocm,
                 spec=dependencies.label_specs_by_name[spec_name],
                 vcs=dependencies.vcs,
+                all_desired_clusters=all_desired_clusters,
+                clusters_with_duplicate_matches=clusters_with_duplicate_matches,
+                dry_run=dependencies.dry_run,
+            )
+            self._sync_subscription_labels(
+                spec=spec,
+                desired_clusters=all_desired_clusters,
+                ocm=ocm,
                 dry_run=dependencies.dry_run,
             )
 
+    def _discover_desired_clusters(
+        self, spec: FleetLabelsSpecV1, ocm: OCMClient
+    ) -> dict[str, list[ClusterData]]:
+        clusters: dict[str, list[ClusterData]] = defaultdict(list)
+        for label_default in spec.label_defaults:
+            match_subscription_labels = dict(label_default.match_subscription_labels)
+            for cluster in ocm.discover_clusters_by_labels(
+                labels=match_subscription_labels
+            ):
+                # TODO: ideally we filter on server side - see TODO in ocm.py
+                if (
+                    match_subscription_labels.items()
+                    <= cluster.subscription_labels.items()
+                ):
+                    clusters[cluster.cluster_id].append(
+                        ClusterData(
+                            subscription_id=cluster.subscription_id,
+                            desired_label_default=label_default,
+                            name=cluster.name,
+                            server_url=cluster.server_url,
+                            current_subscription_labels=cluster.subscription_labels,
+                        )
+                    )
+        return clusters
+
     def _render_default_labels(
         self,
         template: FleetSubscriptionLabelTemplateV1,
@@ -104,6 +161,7 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
             {
                 "name": cluster.name,
                 "clusterId": cluster.cluster_id,
+                "subscriptionId": cluster.subscription_id,
                 "serverUrl": cluster.server_url,
                 "subscriptionLabels": cluster.subscription_labels_content,
             }
@@ -114,68 +172,99 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
             yml.dump(content, stream)
             return stream.getvalue()
 
+    def _sync_subscription_labels(
+        self,
+        spec: FleetLabelsSpecV1,
+        desired_clusters: dict[str, ClusterData],
+        ocm: OCMClient,
+        dry_run: bool,
+    ) -> None:
+        """
+        Synchronize subscription labels for clusters in the spec's inventory.
+        Note, that we only update labels for clusters which are also part of the
+        discovered desired state.
+        I.e., we only operate on clusters that are in both, current state and desired state.
+        That way we ensure we do not work on deleted clusters and still synch only on
+        whats written in the rendered spec yet.
+        """
+        for cluster in spec.clusters:
+            if not desired_clusters.get(cluster.cluster_id):
+                # The cluster is not part of the desired inventory (will be updated with MR soon)
+                continue
+            # Ensure we only handle labels for our managed prefix
+            current_subscription_labels = {
+                k: v
+                for k, v in desired_clusters[
+                    cluster.cluster_id
+                ].current_subscription_labels.items()
+                if k.startswith(spec.managed_subscription_label_prefix)
+            }
+            desired_subscription_labels = {
+                f"{spec.managed_subscription_label_prefix}.{k}": v
+                for k, v in dict(cluster.subscription_labels).items()
+            }
+            diff = diff_mappings(
+                current=current_subscription_labels,
+                desired=desired_subscription_labels,
+            )
+            for key in diff.add:
+                value = desired_subscription_labels[key]
+                logging.info(
+                    f"[{spec.name}] Adding label '{key}={value}' for cluster '{cluster.cluster_id}' in subscription '{cluster.subscription_id}'."
+                )
+                if not dry_run:
+                    ocm.add_subscription_label(
+                        subscription_id=cluster.subscription_id,
+                        key=key,
+                        value=value,
+                    )
+            for key in diff.change:
+                value = desired_subscription_labels[key]
+                logging.info(
+                    f"[{spec.name}] Updating label '{key}={value}' for cluster '{cluster.cluster_id}' in subscription '{cluster.subscription_id}'."
+                )
+                if not dry_run:
+                    ocm.update_subscription_label(
+                        subscription_id=cluster.subscription_id,
+                        key=key,
+                        value=value,
+                    )
+            # Note, we dont want to enable removal for now - its too dangerous on a broad managed prefix
+            # However, if it is needed in the future, we could easily add it here.
+
     def _sync_cluster_inventory(
         self,
         ocm: OCMClient,
         spec: FleetLabelsSpecV1,
         vcs: VCS,
+        clusters_with_duplicate_matches: dict[str, list[ClusterData]],
+        all_desired_clusters: dict[str, ClusterData],
         dry_run: bool,
     ) -> None:
-        class ClusterData(BaseModel):
-            """
-            Helper structure for synching process
-            """
-
-            name: str
-            server_url: str
-            label_default: FleetLabelDefaultV1
-
         all_current_cluster_ids = {cluster.cluster_id for cluster in spec.clusters}
-        clusters: dict[str, list[ClusterData]] = defaultdict(list)
-        for label_default in spec.label_defaults:
-            match_subscription_labels = dict(label_default.match_subscription_labels)
-            for cluster in ocm.discover_clusters_by_labels(
-                labels=match_subscription_labels
-            ):
-                # TODO: ideally we filter on server side - see TODO in ocm.py
-                if (
-                    match_subscription_labels.items()
-                    <= cluster.subscription_labels.items()
-                ):
-                    clusters[cluster.cluster_id].append(
-                        ClusterData(
-                            label_default=label_default,
-                            name=cluster.name,
-                            server_url=cluster.server_url,
-                        )
-                    )
-
-        cluster_with_duplicate_matches = {
-            k: v for k, v in clusters.items() if len(v) > 1
-        }
-        for cluster_id, matches in cluster_with_duplicate_matches.items():
+        for cluster_id, matches in clusters_with_duplicate_matches.items():
             label_matches = "\n".join(
-                str(m.label_default.match_subscription_labels) for m in matches
+                str(m.desired_label_default.match_subscription_labels) for m in matches
             )
             logging.error(
-                f"Spec '{spec.name}': Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
+                f"[{spec.name}] Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
             )
         metrics.set_gauge(
             FleetLabelerDuplicateClusterMatchesGauge(
                 integration=self.name,
                 ocm_name=spec.ocm.name,
             ),
-            len(cluster_with_duplicate_matches),
+            len(clusters_with_duplicate_matches),
         )
 
-        all_desired_clusters = {k: v[0] for k, v in clusters.items() if len(v) == 1}
         clusters_to_add = [
             YamlCluster(
                 cluster_id=cluster_id,
+                subscription_id=cluster_info.subscription_id,
                 name=cluster_info.name,
                 server_url=cluster_info.server_url,
                 subscription_labels_content=self._render_default_labels(
-                    template=cluster_info.label_default.subscription_label_template,
+                    template=cluster_info.desired_label_default.subscription_label_template,
                     labels=ocm.get_cluster_labels(cluster_id=cluster_id),
                 ),
             )
@@ -219,4 +308,5 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
         desired_content = self._render_yaml_file(
             current_content, cluster_ids_to_delete, sorted_clusters_to_add
         )
-        vcs.open_merge_request(path=f"data{spec.path}", content=desired_content)
+        if not dry_run:
+            vcs.open_merge_request(path=f"data{spec.path}", content=desired_content)

reconcile/fleet_labeler/merge_request.py

@@ -13,6 +13,7 @@ class YamlCluster(BaseModel):
     name: str
     server_url: str
     cluster_id: str
+    subscription_id: str
     subscription_labels_content: Any
 
 

reconcile/fleet_labeler/meta.py

@@ -1,4 +1,4 @@
 from reconcile.utils.semver_helper import make_semver
 
 QONTRACT_INTEGRATION = "fleet-labeler"
-QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 0)

reconcile/fleet_labeler/ocm.py

@@ -10,7 +10,9 @@ from reconcile.utils.ocm.clusters import (
     discover_clusters_by_labels,
 )
 from reconcile.utils.ocm.labels import (
+    add_label,
     get_cluster_labels_for_cluster_id,
+    update_label,
 )
 from reconcile.utils.ocm.search_filters import Filter, FilterMode
 from reconcile.utils.ocm_base_client import (
@@ -24,18 +26,22 @@ Thin abstractions of reconcile.ocm module to reduce coupling.
 
 class Cluster(BaseModel):
     cluster_id: str
+    subscription_id: str
     server_url: str
     name: str
     subscription_labels: dict[str, str]
 
     @staticmethod
     def from_cluster_details(cluster: ClusterDetails) -> Cluster:
-        server_url = (
+        console_url = (
             cluster.ocm_cluster.console.url if cluster.ocm_cluster.console else ""
         )
+        api_url = cluster.ocm_cluster.api_url or ""
+        server_url = api_url or console_url or ""
 
         return Cluster(
             cluster_id=cluster.ocm_cluster.id,
+            subscription_id=cluster.ocm_cluster.subscription.id,
             server_url=server_url,
             name=cluster.ocm_cluster.name,
             subscription_labels={
@@ -81,3 +87,25 @@ class OCMClient:
 
     def get_cluster_labels(self, cluster_id: str) -> dict[str, str]:
         return get_cluster_labels_for_cluster_id(self._ocm_client, cluster_id)
+
+    def add_subscription_label(
+        self, subscription_id: str, key: str, value: str
+    ) -> None:
+        # TODO: move href into a utils function
+        add_label(
+            ocm_api=self._ocm_client,
+            label_container_href=f"/api/accounts_mgmt/v1/subscriptions/{subscription_id}/labels",
+            label=key,
+            value=value,
+        )
+
+    def update_subscription_label(
+        self, subscription_id: str, key: str, value: str
+    ) -> None:
+        # TODO: move href into a utils function
+        update_label(
+            ocm_api=self._ocm_client,
+            label_container_href=f"/api/accounts_mgmt/v1/subscriptions/{subscription_id}/labels",
+            label=key,
+            value=value,
+        )

reconcile/gql_definitions/fleet_labeler/fleet_labels.py

@@ -58,6 +58,7 @@ query FleetLabelSpecs {
         clusters {
             name
             serverUrl
+            subscriptionId
             clusterId
             subscriptionLabels
         }
@@ -103,6 +104,7 @@ class FleetLabelDefaultV1(ConfiguredBaseModel):
 class FleetClusterV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
     server_url: str = Field(..., alias="serverUrl")
+    subscription_id: str = Field(..., alias="subscriptionId")
     cluster_id: str = Field(..., alias="clusterId")
     subscription_labels: Json = Field(..., alias="subscriptionLabels")
 

reconcile/gql_definitions/introspection.json

@@ -34632,6 +34632,22 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
+                        {
+                            "name": "subscriptionId",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "SCALAR",
+                                    "name": "String",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
+                        },
                         {
                             "name": "subscriptionLabels",
                             "description": null,