qontract-reconcile 0.10.2.dev74__py3-none-any.whl → 0.10.2.dev76__py3-none-any.whl

{qontract_reconcile-0.10.2.dev74.dist-info → qontract_reconcile-0.10.2.dev76.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev74
+Version: 0.10.2.dev76
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev74.dist-info → qontract_reconcile-0.10.2.dev76.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=q96mwr2KeEQ5bpNniGlgIMZTxiuLSodcYfX-t
 reconcile/aws_support_cases_sos.py,sha256=hl_9L53yQYRQxKs3IWrd69Cc60XK067g_bJRM9B0udo,2975
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
 reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
-reconcile/cli.py,sha256=bLfpfCJqlpSOtcpFVJExYY9DYa4n156zj9SOCbPWfPw,107417
+reconcile/cli.py,sha256=MRdXMNRMBk0e4kv-ctHyYtDD5kAKnT8j6_wiTlWkrQo,107736
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=MvGKBqH9PdHWdMjhLuptze-dk0Tifhp3-0SZdI-7Fmo,4862
 reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
 reconcile/dashdotdb_base.py,sha256=83ZWIf5JJk3P_D69y2TmXRcQr6ELJGlv10OM0h7fJVs,4767
@@ -155,7 +155,7 @@ reconcile/aws_saml_idp/integration.py,sha256=Z2JtUx2YIbkn0KVrVa2CoAErPB8vTykOOkW
 reconcile/aws_saml_roles/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_saml_roles/integration.py,sha256=inU10Yu0lZpJw_00vVe2bytJrecjLtu2hR7kQQIAbx0,11234
 reconcile/aws_version_sync/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/aws_version_sync/integration.py,sha256=7tfOZx9uY4bLcPFf0i6o9cu1Hmk9ByoGPTZ66Iv5fYQ,17885
+reconcile/aws_version_sync/integration.py,sha256=uqIJzek-6pcoB22wmXyfFvmifL-2xODQ5rZ_o-vJGZQ,17899
 reconcile/aws_version_sync/utils.py,sha256=x-45QT7zAwdNvCg7w_qJNwLaksFcfz1_6KQoD_0IVuA,1727
 reconcile/aws_version_sync/merge_request_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/aws_version_sync/merge_request_manager/merge_request.py,sha256=2FbqLLdqxycWNvX1eNbwMjWSVBb7q0p-8t5Db0m7b4Q,4842
@@ -206,6 +206,15 @@ reconcile/external_resources/model.py,sha256=dxwiyI3J9xyLeue8_W9NJoap-CkKLMAoY0S
 reconcile/external_resources/reconciler.py,sha256=-0trp1K-iUgOQn3mm1ZUSmfaReRrUT0eHzPkUhNPolQ,9583
 reconcile/external_resources/secrets_sync.py,sha256=50fK4fzgSz-K8uy5_DQQWA_ju_rTDYAC2HRymgfY7TA,16344
 reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
+reconcile/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/fleet_labeler/dependencies.py,sha256=Ta-SLnrHRN4OBAmhE_mTk1P7y1X7AInIiQsIYaY6hY0,2910
+reconcile/fleet_labeler/integration.py,sha256=5etCwHDTOEeOXHPr9FEB5TaVPphI3sef5fiHq7BR-1Y,7253
+reconcile/fleet_labeler/merge_request.py,sha256=j6RFAr5lujeL73fzhshoBe1JS4dK-EsfROKr9fENoAw,1131
+reconcile/fleet_labeler/meta.py,sha256=DF7O4T9wvQ7-xTWXvuNw1OG_F0SBmRrjFBtVy9wWh9U,146
+reconcile/fleet_labeler/metrics.py,sha256=wx9BmXLsN67m-aSsf81iB7Ehj5SzUsS2WB75isUReZg,662
+reconcile/fleet_labeler/ocm.py,sha256=GGsz-bq1g8BJVVMCfI2kSwZCyngbQoZ3i3k8fO608KA,2506
+reconcile/fleet_labeler/validate.py,sha256=gzc2tt7h9F60h7dcyJfEmsnjnfuux5Jtc_WzrIqr-5k,2541
+reconcile/fleet_labeler/vcs.py,sha256=v4e_3l8F6aquVfe-ItLv2WJtS0kjMiRZ6DQ4mzCLOlE,726
 reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=vCyg8W4ZUGxjU8tB1Gkre_auSpzo83n05mmO8_-7al0,8263
@@ -215,7 +224,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
 reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
 reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/introspection.json,sha256=p3gP25rfNSUn7J4tzfihZ8yyQQU0yutdB15uP9VLlGg,2239836
+reconcile/gql_definitions/introspection.json,sha256=dhwQvjxcy9059f2opu1-Nv2KWO2givEsMzEcoAZhwB0,2238565
 reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
 reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -305,7 +314,7 @@ reconcile/gql_definitions/external_resources/external_resources_settings.py,sha2
 reconcile/gql_definitions/external_resources/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py,sha256=T_qWCRtzU8F9frebBXG9TkeQdrKGt3R9YinSngPoFqM,1262
 reconcile/gql_definitions/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=4echDp7gADwyr-c5aJm4SqnUbkjWCEpN430kBHwIr1Q,4168
+reconcile/gql_definitions/fleet_labeler/fleet_labels.py,sha256=TGpc-NYm2qnURHigCppUZRY1WWaIqA3E_69BWyni1RQ,4323
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
 reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -526,7 +535,7 @@ reconcile/typed_queries/dynatrace.py,sha256=8vXDXDIDf9_vN_efYwysDr4gLN7SCx4I2bOo
 reconcile/typed_queries/dynatrace_environments.py,sha256=jnW1GwIopNjvssEkQMYB1pJR9moYe9hUo6SAGk_-tVA,404
 reconcile/typed_queries/dynatrace_token_provider_token_specs.py,sha256=51kbjfVwzz3n7zTLFi6Lrvotl_iRYKq21tREWTVUIXM,424
 reconcile/typed_queries/external_resources.py,sha256=AT5md8nH5gX56UrdWnU4T3_aVF_FanHorXNFkU6s6KY,1573
-reconcile/typed_queries/fleet_labels.py,sha256=1gTJPHywuRo6R4xZVzAKKvsYvyXlTIP7qhgw_DSjOyc,345
+reconcile/typed_queries/fleet_labels.py,sha256=6yBuAEPKGLMrU-g0yVm8FIDgGAYP9DUh7H2GQxJaB8Q,364
 reconcile/typed_queries/get_state_aws_account.py,sha256=CSJjVPWsUZ2rkGIt8ehoQt7hokFqrUDgG9HFlg2lVD8,492
 reconcile/typed_queries/github_orgs.py,sha256=UZhoPl8qvA_tcO7CZlN8GuMKckt3ywd47Suu61rgHsc,258
 reconcile/typed_queries/gitlab_instances.py,sha256=ZVQHy2W9xIp53f5qYkjKLHLHgOVtQpxTfcmM1C2046g,291
@@ -777,7 +786,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev74.dist-info/METADATA,sha256=TD6ZaK3hyP88I13ucRR7xu-w1np1nDqlv8Q0qRmr_II,24565
-qontract_reconcile-0.10.2.dev74.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev74.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev74.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev76.dist-info/METADATA,sha256=s6eKwa3m2zrcw94LpEgwZaygpv33tsluTXLefR2jdzM,24565
+qontract_reconcile-0.10.2.dev76.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev76.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev76.dist-info/RECORD,,

reconcile/aws_version_sync/integration.py

@@ -359,8 +359,8 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
             desired=external_resources_aws,
             key=lambda r: r.key,
             equal=lambda external_resources_app_interface,
-            external_resources_aws: external_resources_app_interface.resource_engine_version
-            == external_resources_aws.resource_engine_version,
+            external_resources_aws: external_resources_app_interface.resource_engine_version_string
+            == external_resources_aws.resource_engine_version_string,
         )
         for diff_pair in diff.change.values():
             aws_resource = diff_pair.desired

reconcile/cli.py

@@ -3106,6 +3106,19 @@ def dynatrace_token_provider(ctx):
     )
 
 
+@integration.command(short_help="Manage labels across cluster fleets in OCM")
+@click.pass_context
+def fleet_labeler(ctx):
+    from reconcile.fleet_labeler.integration import (
+        FleetLabelerIntegration,
+    )
+
+    run_class_integration(
+        integration=FleetLabelerIntegration(),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage additional routers in OCM.")
 @click.pass_context
 def ocm_additional_routers(ctx):

reconcile/fleet_labeler/dependencies.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from reconcile.fleet_labeler.ocm import OCMClient, OCMClientConfig
+from reconcile.fleet_labeler.vcs import VCS
+from reconcile.gql_definitions.fleet_labeler.fleet_labels import FleetLabelsSpecV1
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils.ocm_base_client import (
+    init_ocm_base_client,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.vcs import VCS as VCSBase
+
+
+class Dependencies:
+    """
+    Depenedencies class to hold all the dependencies (API clients, Specs) for the Fleet Labeler.
+    Dependency inversion simplifies setting up tests.
+    """
+
+    def __init__(
+        self,
+        label_specs_by_name: Mapping[str, FleetLabelsSpecV1],
+        ocm_clients_by_label_spec_name: Mapping[str, OCMClient],
+        vcs: VCS,
+    ):
+        self.label_specs_by_name = label_specs_by_name
+        self.ocm_clients_by_label_spec_name = ocm_clients_by_label_spec_name
+        self.vcs = vcs
+
+    @classmethod
+    def create(
+        cls,
+        secret_reader: SecretReaderBase,
+        dry_run: bool = True,
+    ) -> Dependencies:
+        return Dependencies(
+            label_specs_by_name=_label_specs(),
+            ocm_clients_by_label_spec_name=_ocm_clients(secret_reader=secret_reader),
+            vcs=_vcs(secret_reader=secret_reader, dry_run=dry_run),
+        )
+
+
+def _label_specs() -> dict[str, FleetLabelsSpecV1]:
+    return {spec.name: spec for spec in get_fleet_label_specs()}
+
+
+def _ocm_clients(secret_reader: SecretReaderBase) -> dict[str, OCMClient]:
+    ocm_clients_by_label_spec_name: dict[str, OCMClient] = {}
+    for spec in get_fleet_label_specs():
+        ocm_base_client = init_ocm_base_client(
+            cfg=OCMClientConfig(
+                url=spec.ocm.environment.url,
+                access_token_client_id=spec.ocm.access_token_client_id,
+                access_token_url=spec.ocm.access_token_url,
+                access_token_client_secret=spec.ocm.access_token_client_secret,
+            ),
+            secret_reader=secret_reader,
+        )
+        ocm_clients_by_label_spec_name[spec.name] = OCMClient(ocm_base_client)
+    return ocm_clients_by_label_spec_name
+
+
+def _vcs(secret_reader: SecretReaderBase, dry_run: bool = True) -> VCS:
+    return VCS(
+        vcs=VCSBase(
+            secret_reader=secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+            allow_deleting_mrs=False,
+            allow_opening_mrs=True,
+        )
+    )

reconcile/fleet_labeler/integration.py

@@ -0,0 +1,190 @@
+import logging
+from collections import defaultdict
+from collections.abc import Iterable
+from typing import Any
+
+import yaml
+from pydantic import BaseModel
+from ruamel.yaml.compat import StringIO
+
+from reconcile.fleet_labeler.dependencies import Dependencies
+from reconcile.fleet_labeler.merge_request import YamlCluster
+from reconcile.fleet_labeler.meta import (
+    QONTRACT_INTEGRATION,
+    QONTRACT_INTEGRATION_VERSION,
+)
+from reconcile.fleet_labeler.metrics import FleetLabelerDuplicateClusterMatchesGauge
+from reconcile.fleet_labeler.ocm import OCMClient
+from reconcile.fleet_labeler.validate import validate_label_specs
+from reconcile.fleet_labeler.vcs import VCS
+from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
+    FleetLabelDefaultV1,
+    FleetLabelsSpecV1,
+    FleetSubscriptionLabelTemplateV1,
+)
+from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
+from reconcile.utils import (
+    metrics,
+)
+from reconcile.utils.jinja2.utils import process_jinja2_template
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.runtime.integration import (
+    NoParams,
+    QontractReconcileIntegration,
+)
+
+
+class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
+    def __init__(self) -> None:
+        super().__init__(NoParams())
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_early_exit_desired_state(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        """Return the desired state for early exit."""
+        return {
+            "version": QONTRACT_INTEGRATION_VERSION,
+            "specs": {spec.name: spec.dict() for spec in get_fleet_label_specs()},
+        }
+
+    def run(self, dry_run: bool) -> None:
+        dependencies = Dependencies.create(
+            secret_reader=self.secret_reader,
+            dry_run=dry_run,
+        )
+        self.reconcile(dependencies=dependencies)
+
+    def reconcile(self, dependencies: Dependencies) -> None:
+        validate_label_specs(specs=dependencies.label_specs_by_name)
+        for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
+            self._sync_cluster_inventory(
+                ocm, dependencies.label_specs_by_name[spec_name], dependencies.vcs
+            )
+
+    def _render_default_labels(
+        self,
+        template: FleetSubscriptionLabelTemplateV1,
+        labels: dict[str, str],
+    ) -> dict[str, Any]:
+        if not template.path:
+            # Make mypy happy
+            raise ValueError("path is required for subscription label template")
+        body = template.path.content
+        type = template.q_type or "jinja2"
+        extra_curly = type == "extracurlyjinja2"
+        vars = dict(template.variables or {})
+        vars["labels"] = labels
+        rendered = process_jinja2_template(
+            body,
+            vars,
+            extra_curly=extra_curly,
+        )
+        return yaml.safe_load(rendered)
+
+    def _render_yaml_file(
+        self,
+        current_content: str,
+        ids_to_delete: Iterable[str],
+        clusters_to_add: Iterable[YamlCluster],
+    ) -> str:
+        yml = create_ruamel_instance(pure=True)
+        content = yml.load(current_content)
+        current_clusters = content.get("clusters", [])
+        desired_clusters = [
+            cluster
+            for cluster in current_clusters
+            if cluster.get("clusterId") not in ids_to_delete
+        ]
+        desired_clusters.extend([
+            {
+                "name": cluster.name,
+                "clusterId": cluster.cluster_id,
+                "serverUrl": cluster.server_url,
+                "subscriptionLabels": cluster.subscription_labels_content,
+            }
+            for cluster in clusters_to_add
+        ])
+        content["clusters"] = desired_clusters
+        with StringIO() as stream:
+            yml.dump(content, stream)
+            return stream.getvalue()
+
+    def _sync_cluster_inventory(
+        self, ocm: OCMClient, spec: FleetLabelsSpecV1, vcs: VCS
+    ) -> None:
+        class ClusterData(BaseModel):
+            """
+            Helper structure for synching process
+            """
+
+            name: str
+            server_url: str
+            label_default: FleetLabelDefaultV1
+
+        all_current_cluster_ids = {cluster.cluster_id for cluster in spec.clusters}
+        clusters: dict[str, list[ClusterData]] = defaultdict(list)
+        for label_default in spec.label_defaults:
+            match_subscription_labels = dict(label_default.match_subscription_labels)
+            for cluster in ocm.discover_clusters_by_labels(
+                labels=match_subscription_labels
+            ):
+                # TODO: ideally we filter on server side - see TODO in ocm.py
+                if (
+                    match_subscription_labels.items()
+                    <= cluster.subscription_labels.items()
+                ):
+                    clusters[cluster.cluster_id].append(
+                        ClusterData(
+                            label_default=label_default,
+                            name=cluster.name,
+                            server_url=cluster.server_url,
+                        )
+                    )
+
+        cluster_with_duplicate_matches = {
+            k: v for k, v in clusters.items() if len(v) > 1
+        }
+        for cluster_id, matches in cluster_with_duplicate_matches.items():
+            label_matches = "\n".join(
+                str(m.label_default.match_subscription_labels) for m in matches
+            )
+            logging.error(
+                f"Spec '{spec.name}': Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
+            )
+        metrics.set_gauge(
+            FleetLabelerDuplicateClusterMatchesGauge(
+                integration=self.name,
+                ocm_name=spec.ocm.name,
+            ),
+            len(cluster_with_duplicate_matches),
+        )
+
+        all_desired_clusters = {k: v[0] for k, v in clusters.items() if len(v) == 1}
+        clusters_to_add = [
+            YamlCluster(
+                cluster_id=cluster_id,
+                name=cluster_info.name,
+                server_url=cluster_info.server_url,
+                subscription_labels_content=self._render_default_labels(
+                    template=cluster_info.label_default.subscription_label_template,
+                    labels=ocm.get_cluster_labels(cluster_id=cluster_id),
+                ),
+            )
+            for cluster_id, cluster_info in all_desired_clusters.items()
+            if cluster_id not in all_current_cluster_ids
+        ]
+        cluster_ids_to_delete = all_current_cluster_ids - all_desired_clusters.keys()
+
+        if not (cluster_ids_to_delete or clusters_to_add):
+            return
+
+        current_content = vcs.get_file_content_from_main(path=spec.path)
+        # Lets make sure we are deterministic when adding new clusters
+        # The overhead is neglectable and it makes testing easier
+        sorted_clusters_to_add = sorted(clusters_to_add, key=lambda c: c.name)
+        desired_content = self._render_yaml_file(
+            current_content, cluster_ids_to_delete, sorted_clusters_to_add
+        )
+        vcs.open_merge_request(path=spec.path, content=desired_content)

reconcile/fleet_labeler/merge_request.py

@@ -0,0 +1,48 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+
+FLEET_LABELER_LABEL = "FleetLabeler"
+
+
+class YamlCluster(BaseModel):
+    name: str
+    server_url: str
+    cluster_id: str
+    subscription_labels_content: Any
+
+
+class FleetLabelerUpdates(MergeRequestBase):
+    def __init__(
+        self,
+        path: str,
+        content: str,
+    ):
+        self._path = path
+        self._content = content
+        self.name = f"[Fleet Labeler] Update cluster inventory for {path}"
+
+        super().__init__()
+
+        self.labels = [AUTO_MERGE, FLEET_LABELER_LABEL]
+
+    @property
+    def title(self) -> str:
+        return self.name
+
+    @property
+    def description(self) -> str:
+        return self.name
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        msg = "update cluster inventory"
+        gitlab_cli.update_file(
+            branch_name=self.branch,
+            file_path=self._path,
+            commit_message=msg,
+            content=self._content,
+        )

reconcile/fleet_labeler/meta.py

@@ -0,0 +1,4 @@
+from reconcile.utils.semver_helper import make_semver
+
+QONTRACT_INTEGRATION = "fleet-labeler"
+QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)

reconcile/fleet_labeler/metrics.py

@@ -0,0 +1,23 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    GaugeMetric,
+)
+
+
+class FleetLabelerBaseMetric(BaseModel):
+    integration: str
+    ocm_name: str
+
+
+class FleetLabelerDuplicateClusterMatchesGauge(FleetLabelerBaseMetric, GaugeMetric):
+    """
+    Gauge for the number of clusters that have duplicate matches. Clusters with
+    duplicate matches are being ignored by fleet labeler, as it cannot clearly
+    determine which default label to apply for the cluster. Check the logs to
+    identify the clusters with duplicate matches.
+    """
+
+    @classmethod
+    def name(cls) -> str:
+        return "fleet_labeler_duplicate_cluster_matches"

reconcile/fleet_labeler/ocm.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.utils.ocm.clusters import (
+    ClusterDetails,
+    discover_clusters_by_labels,
+)
+from reconcile.utils.ocm.labels import (
+    get_cluster_labels_for_cluster_id,
+)
+from reconcile.utils.ocm.search_filters import Filter, FilterMode
+from reconcile.utils.ocm_base_client import (
+    OCMBaseClient,
+)
+
+"""
+Thin abstractions of reconcile.ocm module to reduce coupling.
+"""
+
+
+class Cluster(BaseModel):
+    cluster_id: str
+    server_url: str
+    name: str
+    subscription_labels: dict[str, str]
+
+    @staticmethod
+    def from_cluster_details(cluster: ClusterDetails) -> Cluster:
+        server_url = (
+            cluster.ocm_cluster.console.url if cluster.ocm_cluster.console else ""
+        )
+
+        return Cluster(
+            cluster_id=cluster.ocm_cluster.id,
+            server_url=server_url,
+            name=cluster.ocm_cluster.name,
+            subscription_labels={
+                label.key: label.value
+                for label in cluster.subscription_labels.labels.values()
+            },
+        )
+
+
+class OCMClientConfig(BaseModel):
+    """
+    OCMOrg does not have the required structure to comply with OCMBaseClient Protocol.
+    This class provides a concrete implementation for the required Protocol.
+    """
+
+    url: str
+    access_token_client_id: str
+    access_token_url: str
+    access_token_client_secret: VaultSecret
+
+
+class OCMClient:
+    """
+    Thin OOP wrapper around OCMBaseClient to avoid function mocking in tests and reduce coupling.
+    """
+
+    def __init__(self, ocm_client: OCMBaseClient):
+        self._ocm_client = ocm_client
+
+    def discover_clusters_by_labels(self, labels: Mapping[str, str]) -> list[Cluster]:
+        label_filter = Filter(mode=FilterMode.AND).eq("type", "Subscription")
+        for key in labels:
+            label_filter = label_filter.eq("Key", key)
+        # TODO: This throws 400 bad request
+        # for k, v in labels.items():
+        #     label_filter = label_filter.eq(k, v)
+        return [
+            Cluster.from_cluster_details(cluster)
+            for cluster in discover_clusters_by_labels(
+                ocm_api=self._ocm_client, label_filter=label_filter
+            )
+        ]
+
+    def get_cluster_labels(self, cluster_id: str) -> dict[str, str]:
+        return get_cluster_labels_for_cluster_id(self._ocm_client, cluster_id)

reconcile/fleet_labeler/validate.py

@@ -0,0 +1,83 @@
+from collections import Counter
+from collections.abc import Mapping
+
+from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
+    FleetLabelsSpecV1,
+    OpenShiftClusterManagerV1,
+)
+
+
+class OCMAccessTokenClientIdMissing(Exception):
+    pass
+
+
+class OCMAccessTokenClientSecretMissing(Exception):
+    pass
+
+
+class OCMAccessTokenUrlMissing(Exception):
+    pass
+
+
+class MatchLabelsNotUniqueError(Exception):
+    pass
+
+
+def validate_label_specs(specs: Mapping[str, FleetLabelsSpecV1]) -> None:
+    """
+    We cannot catch all potential errors through json schema definition.
+    """
+    for spec in specs.values():
+        _validate_ocm_token_spec(spec.ocm)
+        _validate_match_labels(spec)
+        _validate_unique_ocm_managed_label_combo(spec)
+
+
+def _validate_unique_ocm_managed_label_combo(spec: FleetLabelsSpecV1) -> None:
+    """
+    Every fleet labeler spec is pinned to one OCM client and manages a single
+    label prefix. We must be sure, that the label prefixes are not overlapping
+    for the same OCM client, as that would mean to default label specs will be
+    competing.
+    """
+    # TODO: implement
+    pass
+
+
+def _validate_match_labels(spec: FleetLabelsSpecV1) -> None:
+    """
+    Match labels should be unique within the same spec.
+    """
+    for label_default in spec.label_defaults:
+        keys = (
+            ".".join(
+                f"{k}:{v}"
+                for k, v in sorted(
+                    dict(label_default.match_subscription_labels).items()
+                )
+            )
+            for label_default in spec.label_defaults
+        )
+        duplicates = [key for key, count in Counter(keys).items() if count > 1]
+        if duplicates:
+            raise MatchLabelsNotUniqueError(
+                f"The 'matchSubscriptionLabels' combinations must be unique within a spec. Found duplicates in spec {spec.name} for matchers: {duplicates}"
+            )
+
+
+def _validate_ocm_token_spec(ocm: OpenShiftClusterManagerV1) -> None:
+    """
+    OCM tokens are optional in the schema. Lets verify they exist.
+    """
+    if not ocm.access_token_client_id:
+        raise OCMAccessTokenClientIdMissing(
+            f"accessTokenClientId missing in ocm spec '{ocm.name}'"
+        )
+    if not ocm.access_token_client_secret:
+        raise OCMAccessTokenClientSecretMissing(
+            f"accessTokenClientSecret missing in ocm spec '{ocm.name}'"
+        )
+    if not ocm.access_token_url:
+        raise OCMAccessTokenUrlMissing(
+            f"accessTokenUrl missing in ocm spec '{ocm.name}'"
+        )

reconcile/fleet_labeler/vcs.py

@@ -0,0 +1,21 @@
+from reconcile.fleet_labeler.merge_request import FleetLabelerUpdates
+from reconcile.utils.vcs import VCS as VCSBase
+
+
+class VCS:
+    """
+    Thin abstractions of reconcile.utils.vcs module to reduce coupling and simplify tests.
+    """
+
+    def __init__(self, vcs: VCSBase):
+        self._vcs = vcs
+
+    def get_file_content_from_main(self, path: str) -> str:
+        return self._vcs.get_file_content_from_app_interface_ref(
+            file_path=path, ref="main"
+        )
+
+    def open_merge_request(self, path: str, content: str) -> None:
+        mr = FleetLabelerUpdates(path=path, content=content)
+        # Note, that VCS is initialized with dry-run flag already
+        self._vcs.open_app_interface_merge_request(mr)

reconcile/gql_definitions/fleet_labeler/fleet_labels.py

@@ -28,11 +28,13 @@ fragment VaultSecret on VaultSecret_v1 {
     format
 }
 
-query FleetLabels {
-    fleet_labels: fleet_labels_v1 {
+query FleetLabelSpecs {
+    fleet_labels_specs: fleet_labels_specs_v1 {
         name
+        path
         managedSubscriptionLabelPrefix
-        ocms {
+        ocm {
+            name
             environment {
                 url
             }
@@ -75,6 +77,7 @@ class OpenShiftClusterManagerEnvironmentV1(ConfiguredBaseModel):
 
 
 class OpenShiftClusterManagerV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
     environment: OpenShiftClusterManagerEnvironmentV1 = Field(..., alias="environment")
     access_token_client_id: Optional[str] = Field(..., alias="accessTokenClientId")
     access_token_client_secret: Optional[VaultSecret] = Field(..., alias="accessTokenClientSecret")
@@ -104,19 +107,20 @@ class FleetClusterV1(ConfiguredBaseModel):
     subscription_labels: Json = Field(..., alias="subscriptionLabels")
 
 
-class FleetLabelsV1(ConfiguredBaseModel):
+class FleetLabelsSpecV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    path: str = Field(..., alias="path")
     managed_subscription_label_prefix: str = Field(..., alias="managedSubscriptionLabelPrefix")
-    ocms: list[OpenShiftClusterManagerV1] = Field(..., alias="ocms")
+    ocm: OpenShiftClusterManagerV1 = Field(..., alias="ocm")
     label_defaults: list[FleetLabelDefaultV1] = Field(..., alias="labelDefaults")
     clusters: list[FleetClusterV1] = Field(..., alias="clusters")
 
 
-class FleetLabelsQueryData(ConfiguredBaseModel):
-    fleet_labels: Optional[list[FleetLabelsV1]] = Field(..., alias="fleet_labels")
+class FleetLabelSpecsQueryData(ConfiguredBaseModel):
+    fleet_labels_specs: Optional[list[FleetLabelsSpecV1]] = Field(..., alias="fleet_labels_specs")
 
 
-def query(query_func: Callable, **kwargs: Any) -> FleetLabelsQueryData:
+def query(query_func: Callable, **kwargs: Any) -> FleetLabelSpecsQueryData:
     """
     This is a convenience function which queries and parses the data into
     concrete types. It should be compatible with most GQL clients.
@@ -129,7 +133,7 @@ def query(query_func: Callable, **kwargs: Any) -> FleetLabelsQueryData:
         kwargs: optional arguments that will be passed to the query function
 
     Returns:
-        FleetLabelsQueryData: queried data parsed into generated classes
+        FleetLabelSpecsQueryData: queried data parsed into generated classes
     """
     raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return FleetLabelsQueryData(**raw_data)
+    return FleetLabelSpecsQueryData(**raw_data)

reconcile/gql_definitions/introspection.json

@@ -3757,7 +3757,7 @@
                             "deprecationReason": null
                         },
                         {
-                            "name": "fleet_labels_v1",
+                            "name": "fleet_labels_specs_v1",
                             "description": null,
                             "args": [
                                 {
@@ -3799,7 +3799,7 @@
                                     "name": null,
                                     "ofType": {
                                         "kind": "OBJECT",
-                                        "name": "FleetLabels_v1",
+                                        "name": "FleetLabelsSpec_v1",
                                         "ofType": null
                                     }
                                 }
@@ -5621,7 +5621,7 @@
                         },
                         {
                             "kind": "OBJECT",
-                            "name": "FleetLabels_v1",
+                            "name": "FleetLabelsSpec_v1",
                             "ofType": null
                         },
                         {
@@ -10426,6 +10426,26 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "namespaces",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "LIST",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "NON_NULL",
+                                    "name": null,
+                                    "ofType": {
+                                        "kind": "OBJECT",
+                                        "name": "Namespace_v1",
+                                        "ofType": null
+                                    }
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -33880,22 +33900,6 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
-                        {
-                            "name": "module_default_resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "NON_NULL",
-                                "name": null,
-                                "ofType": {
-                                    "kind": "OBJECT",
-                                    "name": "DeployResources_v1",
-                                    "ofType": null
-                                }
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
                         {
                             "name": "module_default_resources",
                             "description": null,
@@ -34113,18 +34117,6 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
-                        {
-                            "name": "resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "OBJECT",
-                                "name": "DeployResources_v1",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
                         {
                             "name": "resources",
                             "description": null,
@@ -34338,7 +34330,7 @@
                 },
                 {
                     "kind": "OBJECT",
-                    "name": "FleetLabels_v1",
+                    "name": "FleetLabelsSpec_v1",
                     "description": null,
                     "fields": [
                         {
@@ -34402,24 +34394,16 @@
                             "deprecationReason": null
                         },
                         {
-                            "name": "ocms",
+                            "name": "ocm",
                             "description": null,
                             "args": [],
                             "type": {
                                 "kind": "NON_NULL",
                                 "name": null,
                                 "ofType": {
-                                    "kind": "LIST",
-                                    "name": null,
-                                    "ofType": {
-                                        "kind": "NON_NULL",
-                                        "name": null,
-                                        "ofType": {
-                                            "kind": "OBJECT",
-                                            "name": "OpenShiftClusterManager_v1",
-                                            "ofType": null
-                                        }
-                                    }
+                                    "kind": "OBJECT",
+                                    "name": "OpenShiftClusterManager_v1",
+                                    "ofType": null
                                 }
                             },
                             "isDeprecated": false,
@@ -43406,18 +43390,6 @@
                             "isDeprecated": false,
                             "deprecationReason": null
                         },
-                        {
-                            "name": "resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "OBJECT",
-                                "name": "DeployResources_v1",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
-                        },
                         {
                             "name": "resources",
                             "description": null,

reconcile/typed_queries/fleet_labels.py

@@ -1,14 +1,14 @@
 from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
-    FleetLabelsV1,
+    FleetLabelsSpecV1,
     query,
 )
 from reconcile.utils import gql
 from reconcile.utils.gql import GqlApi
 
 
-def get_fleet_labels(
+def get_fleet_label_specs(
     api: GqlApi | None = None,
-) -> list[FleetLabelsV1]:
+) -> list[FleetLabelsSpecV1]:
     api = api or gql.get_api()
     data = query(api.query)
-    return data.fleet_labels or []
+    return data.fleet_labels_specs or []