qontract-reconcile 0.10.2.dev74__tar.gz → 0.10.2.dev76__tar.gz

{qontract_reconcile-0.10.2.dev74 → qontract_reconcile-0.10.2.dev76}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev74
+Version: 0.10.2.dev76
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev74 → qontract_reconcile-0.10.2.dev76}/reconcile/aws_version_sync/integration.py

@@ -359,8 +359,8 @@ class AVSIntegration(QontractReconcileIntegration[AVSIntegrationParams]):
             desired=external_resources_aws,
             key=lambda r: r.key,
             equal=lambda external_resources_app_interface,
-            external_resources_aws: external_resources_app_interface.resource_engine_version
-            == external_resources_aws.resource_engine_version,
+            external_resources_aws: external_resources_app_interface.resource_engine_version_string
+            == external_resources_aws.resource_engine_version_string,
         )
         for diff_pair in diff.change.values():
             aws_resource = diff_pair.desired

{qontract_reconcile-0.10.2.dev74 → qontract_reconcile-0.10.2.dev76}/reconcile/cli.py

@@ -3106,6 +3106,19 @@ def dynatrace_token_provider(ctx):
     )
 
 
+@integration.command(short_help="Manage labels across cluster fleets in OCM")
+@click.pass_context
+def fleet_labeler(ctx):
+    from reconcile.fleet_labeler.integration import (
+        FleetLabelerIntegration,
+    )
+
+    run_class_integration(
+        integration=FleetLabelerIntegration(),
+        ctx=ctx.obj,
+    )
+
+
 @integration.command(short_help="Manage additional routers in OCM.")
 @click.pass_context
 def ocm_additional_routers(ctx):

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/dependencies.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from reconcile.fleet_labeler.ocm import OCMClient, OCMClientConfig
+from reconcile.fleet_labeler.vcs import VCS
+from reconcile.gql_definitions.fleet_labeler.fleet_labels import FleetLabelsSpecV1
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils.ocm_base_client import (
+    init_ocm_base_client,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.vcs import VCS as VCSBase
+
+
+class Dependencies:
+    """
+    Depenedencies class to hold all the dependencies (API clients, Specs) for the Fleet Labeler.
+    Dependency inversion simplifies setting up tests.
+    """
+
+    def __init__(
+        self,
+        label_specs_by_name: Mapping[str, FleetLabelsSpecV1],
+        ocm_clients_by_label_spec_name: Mapping[str, OCMClient],
+        vcs: VCS,
+    ):
+        self.label_specs_by_name = label_specs_by_name
+        self.ocm_clients_by_label_spec_name = ocm_clients_by_label_spec_name
+        self.vcs = vcs
+
+    @classmethod
+    def create(
+        cls,
+        secret_reader: SecretReaderBase,
+        dry_run: bool = True,
+    ) -> Dependencies:
+        return Dependencies(
+            label_specs_by_name=_label_specs(),
+            ocm_clients_by_label_spec_name=_ocm_clients(secret_reader=secret_reader),
+            vcs=_vcs(secret_reader=secret_reader, dry_run=dry_run),
+        )
+
+
+def _label_specs() -> dict[str, FleetLabelsSpecV1]:
+    return {spec.name: spec for spec in get_fleet_label_specs()}
+
+
+def _ocm_clients(secret_reader: SecretReaderBase) -> dict[str, OCMClient]:
+    ocm_clients_by_label_spec_name: dict[str, OCMClient] = {}
+    for spec in get_fleet_label_specs():
+        ocm_base_client = init_ocm_base_client(
+            cfg=OCMClientConfig(
+                url=spec.ocm.environment.url,
+                access_token_client_id=spec.ocm.access_token_client_id,
+                access_token_url=spec.ocm.access_token_url,
+                access_token_client_secret=spec.ocm.access_token_client_secret,
+            ),
+            secret_reader=secret_reader,
+        )
+        ocm_clients_by_label_spec_name[spec.name] = OCMClient(ocm_base_client)
+    return ocm_clients_by_label_spec_name
+
+
+def _vcs(secret_reader: SecretReaderBase, dry_run: bool = True) -> VCS:
+    return VCS(
+        vcs=VCSBase(
+            secret_reader=secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+            allow_deleting_mrs=False,
+            allow_opening_mrs=True,
+        )
+    )

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/integration.py

@@ -0,0 +1,190 @@
+import logging
+from collections import defaultdict
+from collections.abc import Iterable
+from typing import Any
+
+import yaml
+from pydantic import BaseModel
+from ruamel.yaml.compat import StringIO
+
+from reconcile.fleet_labeler.dependencies import Dependencies
+from reconcile.fleet_labeler.merge_request import YamlCluster
+from reconcile.fleet_labeler.meta import (
+    QONTRACT_INTEGRATION,
+    QONTRACT_INTEGRATION_VERSION,
+)
+from reconcile.fleet_labeler.metrics import FleetLabelerDuplicateClusterMatchesGauge
+from reconcile.fleet_labeler.ocm import OCMClient
+from reconcile.fleet_labeler.validate import validate_label_specs
+from reconcile.fleet_labeler.vcs import VCS
+from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
+    FleetLabelDefaultV1,
+    FleetLabelsSpecV1,
+    FleetSubscriptionLabelTemplateV1,
+)
+from reconcile.typed_queries.fleet_labels import get_fleet_label_specs
+from reconcile.utils import (
+    metrics,
+)
+from reconcile.utils.jinja2.utils import process_jinja2_template
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.runtime.integration import (
+    NoParams,
+    QontractReconcileIntegration,
+)
+
+
+class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
+    def __init__(self) -> None:
+        super().__init__(NoParams())
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_early_exit_desired_state(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        """Return the desired state for early exit."""
+        return {
+            "version": QONTRACT_INTEGRATION_VERSION,
+            "specs": {spec.name: spec.dict() for spec in get_fleet_label_specs()},
+        }
+
+    def run(self, dry_run: bool) -> None:
+        dependencies = Dependencies.create(
+            secret_reader=self.secret_reader,
+            dry_run=dry_run,
+        )
+        self.reconcile(dependencies=dependencies)
+
+    def reconcile(self, dependencies: Dependencies) -> None:
+        validate_label_specs(specs=dependencies.label_specs_by_name)
+        for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
+            self._sync_cluster_inventory(
+                ocm, dependencies.label_specs_by_name[spec_name], dependencies.vcs
+            )
+
+    def _render_default_labels(
+        self,
+        template: FleetSubscriptionLabelTemplateV1,
+        labels: dict[str, str],
+    ) -> dict[str, Any]:
+        if not template.path:
+            # Make mypy happy
+            raise ValueError("path is required for subscription label template")
+        body = template.path.content
+        type = template.q_type or "jinja2"
+        extra_curly = type == "extracurlyjinja2"
+        vars = dict(template.variables or {})
+        vars["labels"] = labels
+        rendered = process_jinja2_template(
+            body,
+            vars,
+            extra_curly=extra_curly,
+        )
+        return yaml.safe_load(rendered)
+
+    def _render_yaml_file(
+        self,
+        current_content: str,
+        ids_to_delete: Iterable[str],
+        clusters_to_add: Iterable[YamlCluster],
+    ) -> str:
+        yml = create_ruamel_instance(pure=True)
+        content = yml.load(current_content)
+        current_clusters = content.get("clusters", [])
+        desired_clusters = [
+            cluster
+            for cluster in current_clusters
+            if cluster.get("clusterId") not in ids_to_delete
+        ]
+        desired_clusters.extend([
+            {
+                "name": cluster.name,
+                "clusterId": cluster.cluster_id,
+                "serverUrl": cluster.server_url,
+                "subscriptionLabels": cluster.subscription_labels_content,
+            }
+            for cluster in clusters_to_add
+        ])
+        content["clusters"] = desired_clusters
+        with StringIO() as stream:
+            yml.dump(content, stream)
+            return stream.getvalue()
+
+    def _sync_cluster_inventory(
+        self, ocm: OCMClient, spec: FleetLabelsSpecV1, vcs: VCS
+    ) -> None:
+        class ClusterData(BaseModel):
+            """
+            Helper structure for synching process
+            """
+
+            name: str
+            server_url: str
+            label_default: FleetLabelDefaultV1
+
+        all_current_cluster_ids = {cluster.cluster_id for cluster in spec.clusters}
+        clusters: dict[str, list[ClusterData]] = defaultdict(list)
+        for label_default in spec.label_defaults:
+            match_subscription_labels = dict(label_default.match_subscription_labels)
+            for cluster in ocm.discover_clusters_by_labels(
+                labels=match_subscription_labels
+            ):
+                # TODO: ideally we filter on server side - see TODO in ocm.py
+                if (
+                    match_subscription_labels.items()
+                    <= cluster.subscription_labels.items()
+                ):
+                    clusters[cluster.cluster_id].append(
+                        ClusterData(
+                            label_default=label_default,
+                            name=cluster.name,
+                            server_url=cluster.server_url,
+                        )
+                    )
+
+        cluster_with_duplicate_matches = {
+            k: v for k, v in clusters.items() if len(v) > 1
+        }
+        for cluster_id, matches in cluster_with_duplicate_matches.items():
+            label_matches = "\n".join(
+                str(m.label_default.match_subscription_labels) for m in matches
+            )
+            logging.error(
+                f"Spec '{spec.name}': Cluster ID {cluster_id} is matched multiple times by different label matchers:\n{label_matches}"
+            )
+        metrics.set_gauge(
+            FleetLabelerDuplicateClusterMatchesGauge(
+                integration=self.name,
+                ocm_name=spec.ocm.name,
+            ),
+            len(cluster_with_duplicate_matches),
+        )
+
+        all_desired_clusters = {k: v[0] for k, v in clusters.items() if len(v) == 1}
+        clusters_to_add = [
+            YamlCluster(
+                cluster_id=cluster_id,
+                name=cluster_info.name,
+                server_url=cluster_info.server_url,
+                subscription_labels_content=self._render_default_labels(
+                    template=cluster_info.label_default.subscription_label_template,
+                    labels=ocm.get_cluster_labels(cluster_id=cluster_id),
+                ),
+            )
+            for cluster_id, cluster_info in all_desired_clusters.items()
+            if cluster_id not in all_current_cluster_ids
+        ]
+        cluster_ids_to_delete = all_current_cluster_ids - all_desired_clusters.keys()
+
+        if not (cluster_ids_to_delete or clusters_to_add):
+            return
+
+        current_content = vcs.get_file_content_from_main(path=spec.path)
+        # Lets make sure we are deterministic when adding new clusters
+        # The overhead is neglectable and it makes testing easier
+        sorted_clusters_to_add = sorted(clusters_to_add, key=lambda c: c.name)
+        desired_content = self._render_yaml_file(
+            current_content, cluster_ids_to_delete, sorted_clusters_to_add
+        )
+        vcs.open_merge_request(path=spec.path, content=desired_content)

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/merge_request.py

@@ -0,0 +1,48 @@
+from typing import Any
+
+from pydantic import BaseModel
+
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+
+FLEET_LABELER_LABEL = "FleetLabeler"
+
+
+class YamlCluster(BaseModel):
+    name: str
+    server_url: str
+    cluster_id: str
+    subscription_labels_content: Any
+
+
+class FleetLabelerUpdates(MergeRequestBase):
+    def __init__(
+        self,
+        path: str,
+        content: str,
+    ):
+        self._path = path
+        self._content = content
+        self.name = f"[Fleet Labeler] Update cluster inventory for {path}"
+
+        super().__init__()
+
+        self.labels = [AUTO_MERGE, FLEET_LABELER_LABEL]
+
+    @property
+    def title(self) -> str:
+        return self.name
+
+    @property
+    def description(self) -> str:
+        return self.name
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        msg = "update cluster inventory"
+        gitlab_cli.update_file(
+            branch_name=self.branch,
+            file_path=self._path,
+            commit_message=msg,
+            content=self._content,
+        )

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/meta.py

@@ -0,0 +1,4 @@
+from reconcile.utils.semver_helper import make_semver
+
+QONTRACT_INTEGRATION = "fleet-labeler"
+QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/metrics.py

@@ -0,0 +1,23 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    GaugeMetric,
+)
+
+
+class FleetLabelerBaseMetric(BaseModel):
+    integration: str
+    ocm_name: str
+
+
+class FleetLabelerDuplicateClusterMatchesGauge(FleetLabelerBaseMetric, GaugeMetric):
+    """
+    Gauge for the number of clusters that have duplicate matches. Clusters with
+    duplicate matches are being ignored by fleet labeler, as it cannot clearly
+    determine which default label to apply for the cluster. Check the logs to
+    identify the clusters with duplicate matches.
+    """
+
+    @classmethod
+    def name(cls) -> str:
+        return "fleet_labeler_duplicate_cluster_matches"

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/ocm.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
+from reconcile.utils.ocm.clusters import (
+    ClusterDetails,
+    discover_clusters_by_labels,
+)
+from reconcile.utils.ocm.labels import (
+    get_cluster_labels_for_cluster_id,
+)
+from reconcile.utils.ocm.search_filters import Filter, FilterMode
+from reconcile.utils.ocm_base_client import (
+    OCMBaseClient,
+)
+
+"""
+Thin abstractions of reconcile.ocm module to reduce coupling.
+"""
+
+
+class Cluster(BaseModel):
+    cluster_id: str
+    server_url: str
+    name: str
+    subscription_labels: dict[str, str]
+
+    @staticmethod
+    def from_cluster_details(cluster: ClusterDetails) -> Cluster:
+        server_url = (
+            cluster.ocm_cluster.console.url if cluster.ocm_cluster.console else ""
+        )
+
+        return Cluster(
+            cluster_id=cluster.ocm_cluster.id,
+            server_url=server_url,
+            name=cluster.ocm_cluster.name,
+            subscription_labels={
+                label.key: label.value
+                for label in cluster.subscription_labels.labels.values()
+            },
+        )
+
+
+class OCMClientConfig(BaseModel):
+    """
+    OCMOrg does not have the required structure to comply with OCMBaseClient Protocol.
+    This class provides a concrete implementation for the required Protocol.
+    """
+
+    url: str
+    access_token_client_id: str
+    access_token_url: str
+    access_token_client_secret: VaultSecret
+
+
+class OCMClient:
+    """
+    Thin OOP wrapper around OCMBaseClient to avoid function mocking in tests and reduce coupling.
+    """
+
+    def __init__(self, ocm_client: OCMBaseClient):
+        self._ocm_client = ocm_client
+
+    def discover_clusters_by_labels(self, labels: Mapping[str, str]) -> list[Cluster]:
+        label_filter = Filter(mode=FilterMode.AND).eq("type", "Subscription")
+        for key in labels:
+            label_filter = label_filter.eq("Key", key)
+        # TODO: This throws 400 bad request
+        # for k, v in labels.items():
+        #     label_filter = label_filter.eq(k, v)
+        return [
+            Cluster.from_cluster_details(cluster)
+            for cluster in discover_clusters_by_labels(
+                ocm_api=self._ocm_client, label_filter=label_filter
+            )
+        ]
+
+    def get_cluster_labels(self, cluster_id: str) -> dict[str, str]:
+        return get_cluster_labels_for_cluster_id(self._ocm_client, cluster_id)

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/validate.py

@@ -0,0 +1,83 @@
+from collections import Counter
+from collections.abc import Mapping
+
+from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
+    FleetLabelsSpecV1,
+    OpenShiftClusterManagerV1,
+)
+
+
+class OCMAccessTokenClientIdMissing(Exception):
+    pass
+
+
+class OCMAccessTokenClientSecretMissing(Exception):
+    pass
+
+
+class OCMAccessTokenUrlMissing(Exception):
+    pass
+
+
+class MatchLabelsNotUniqueError(Exception):
+    pass
+
+
+def validate_label_specs(specs: Mapping[str, FleetLabelsSpecV1]) -> None:
+    """
+    We cannot catch all potential errors through json schema definition.
+    """
+    for spec in specs.values():
+        _validate_ocm_token_spec(spec.ocm)
+        _validate_match_labels(spec)
+        _validate_unique_ocm_managed_label_combo(spec)
+
+
+def _validate_unique_ocm_managed_label_combo(spec: FleetLabelsSpecV1) -> None:
+    """
+    Every fleet labeler spec is pinned to one OCM client and manages a single
+    label prefix. We must be sure, that the label prefixes are not overlapping
+    for the same OCM client, as that would mean to default label specs will be
+    competing.
+    """
+    # TODO: implement
+    pass
+
+
+def _validate_match_labels(spec: FleetLabelsSpecV1) -> None:
+    """
+    Match labels should be unique within the same spec.
+    """
+    for label_default in spec.label_defaults:
+        keys = (
+            ".".join(
+                f"{k}:{v}"
+                for k, v in sorted(
+                    dict(label_default.match_subscription_labels).items()
+                )
+            )
+            for label_default in spec.label_defaults
+        )
+        duplicates = [key for key, count in Counter(keys).items() if count > 1]
+        if duplicates:
+            raise MatchLabelsNotUniqueError(
+                f"The 'matchSubscriptionLabels' combinations must be unique within a spec. Found duplicates in spec {spec.name} for matchers: {duplicates}"
+            )
+
+
+def _validate_ocm_token_spec(ocm: OpenShiftClusterManagerV1) -> None:
+    """
+    OCM tokens are optional in the schema. Lets verify they exist.
+    """
+    if not ocm.access_token_client_id:
+        raise OCMAccessTokenClientIdMissing(
+            f"accessTokenClientId missing in ocm spec '{ocm.name}'"
+        )
+    if not ocm.access_token_client_secret:
+        raise OCMAccessTokenClientSecretMissing(
+            f"accessTokenClientSecret missing in ocm spec '{ocm.name}'"
+        )
+    if not ocm.access_token_url:
+        raise OCMAccessTokenUrlMissing(
+            f"accessTokenUrl missing in ocm spec '{ocm.name}'"
+        )

qontract_reconcile-0.10.2.dev76/reconcile/fleet_labeler/vcs.py

@@ -0,0 +1,21 @@
+from reconcile.fleet_labeler.merge_request import FleetLabelerUpdates
+from reconcile.utils.vcs import VCS as VCSBase
+
+
+class VCS:
+    """
+    Thin abstractions of reconcile.utils.vcs module to reduce coupling and simplify tests.
+    """
+
+    def __init__(self, vcs: VCSBase):
+        self._vcs = vcs
+
+    def get_file_content_from_main(self, path: str) -> str:
+        return self._vcs.get_file_content_from_app_interface_ref(
+            file_path=path, ref="main"
+        )
+
+    def open_merge_request(self, path: str, content: str) -> None:
+        mr = FleetLabelerUpdates(path=path, content=content)
+        # Note, that VCS is initialized with dry-run flag already
+        self._vcs.open_app_interface_merge_request(mr)

{qontract_reconcile-0.10.2.dev74 → qontract_reconcile-0.10.2.dev76}/reconcile/gql_definitions/fleet_labeler/fleet_labels.py

@@ -28,11 +28,13 @@ fragment VaultSecret on VaultSecret_v1 {
     format
 }
 
-query FleetLabels {
-    fleet_labels: fleet_labels_v1 {
+query FleetLabelSpecs {
+    fleet_labels_specs: fleet_labels_specs_v1 {
         name
+        path
         managedSubscriptionLabelPrefix
-        ocms {
+        ocm {
+            name
             environment {
                 url
             }
@@ -75,6 +77,7 @@ class OpenShiftClusterManagerEnvironmentV1(ConfiguredBaseModel):
 
 
 class OpenShiftClusterManagerV1(ConfiguredBaseModel):
+    name: str = Field(..., alias="name")
     environment: OpenShiftClusterManagerEnvironmentV1 = Field(..., alias="environment")
     access_token_client_id: Optional[str] = Field(..., alias="accessTokenClientId")
     access_token_client_secret: Optional[VaultSecret] = Field(..., alias="accessTokenClientSecret")
@@ -104,19 +107,20 @@ class FleetClusterV1(ConfiguredBaseModel):
     subscription_labels: Json = Field(..., alias="subscriptionLabels")
 
 
-class FleetLabelsV1(ConfiguredBaseModel):
+class FleetLabelsSpecV1(ConfiguredBaseModel):
     name: str = Field(..., alias="name")
+    path: str = Field(..., alias="path")
     managed_subscription_label_prefix: str = Field(..., alias="managedSubscriptionLabelPrefix")
-    ocms: list[OpenShiftClusterManagerV1] = Field(..., alias="ocms")
+    ocm: OpenShiftClusterManagerV1 = Field(..., alias="ocm")
     label_defaults: list[FleetLabelDefaultV1] = Field(..., alias="labelDefaults")
     clusters: list[FleetClusterV1] = Field(..., alias="clusters")
 
 
-class FleetLabelsQueryData(ConfiguredBaseModel):
-    fleet_labels: Optional[list[FleetLabelsV1]] = Field(..., alias="fleet_labels")
+class FleetLabelSpecsQueryData(ConfiguredBaseModel):
+    fleet_labels_specs: Optional[list[FleetLabelsSpecV1]] = Field(..., alias="fleet_labels_specs")
 
 
-def query(query_func: Callable, **kwargs: Any) -> FleetLabelsQueryData:
+def query(query_func: Callable, **kwargs: Any) -> FleetLabelSpecsQueryData:
     """
     This is a convenience function which queries and parses the data into
     concrete types. It should be compatible with most GQL clients.
@@ -129,7 +133,7 @@ def query(query_func: Callable, **kwargs: Any) -> FleetLabelsQueryData:
         kwargs: optional arguments that will be passed to the query function
 
     Returns:
-        FleetLabelsQueryData: queried data parsed into generated classes
+        FleetLabelSpecsQueryData: queried data parsed into generated classes
     """
     raw_data: dict[Any, Any] = query_func(DEFINITION, **kwargs)
-    return FleetLabelsQueryData(**raw_data)
+    return FleetLabelSpecsQueryData(**raw_data)