qontract-reconcile 0.9.1rc298__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/dynatrace_token_provider/metrics.py

@@ -0,0 +1,62 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    CounterMetric,
+    ErrorRateMetricSet,
+    GaugeMetric,
+)
+
+
+class DTPBaseMetric(BaseModel):
+    integration: str
+    ocm_env: str
+
+
+class DTPOrganizationReconcileCounter(DTPBaseMetric, CounterMetric):
+    org_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "dtp_organization_reconciled"
+
+
+class DTPOrganizationReconcileErrorCounter(DTPBaseMetric, CounterMetric):
+    org_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "dtp_organization_reconcile_errors"
+
+
+class DTPOrganizationErrorRate(ErrorRateMetricSet):
+    def __init__(self, integration: str, org_id: str, ocm_env: str) -> None:
+        super().__init__(
+            counter=DTPOrganizationReconcileCounter(
+                integration=integration,
+                ocm_env=ocm_env,
+                org_id=org_id,
+            ),
+            error_counter=DTPOrganizationReconcileErrorCounter(
+                integration=integration,
+                ocm_env=ocm_env,
+                org_id=org_id,
+            ),
+        )
+
+
+class DTPClustersManagedGauge(DTPBaseMetric, GaugeMetric):
+    "Gauge for the number of clusters DTP manages"
+
+    @classmethod
+    def name(cls) -> str:
+        return "dtp_clusters_managed"
+
+
+class DTPTokensManagedGauge(DTPBaseMetric, GaugeMetric):
+    "Gauge for the number of tokens DTP manages"
+
+    dt_tenant_id: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "dtp_tokens_managed"

reconcile/dynatrace_token_provider/model.py

@@ -0,0 +1,14 @@
+from pydantic import BaseModel
+
+
+class DynatraceAPIToken(BaseModel):
+    token: str
+    id: str
+    name: str
+    secret_key: str
+
+
+class K8sSecret(BaseModel):
+    namespace_name: str
+    secret_name: str
+    tokens: list[DynatraceAPIToken]

reconcile/dynatrace_token_provider/ocm.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from datetime import timedelta
+from typing import Any
+
+from pydantic import BaseModel
+
+from reconcile.utils.ocm.base import (
+    OCMClusterServiceLogCreateModel,
+)
+from reconcile.utils.ocm.clusters import (
+    ClusterDetails,
+    discover_clusters_by_labels,
+)
+from reconcile.utils.ocm.labels import Filter
+from reconcile.utils.ocm.manifests import (
+    create_manifest,
+    get_manifest,
+    patch_manifest,
+)
+from reconcile.utils.ocm.service_log import create_service_log
+from reconcile.utils.ocm.sre_capability_labels import sre_capability_label_key
+from reconcile.utils.ocm.syncsets import (
+    create_syncset,
+    get_syncset,
+    patch_syncset,
+)
+from reconcile.utils.ocm_base_client import (
+    OCMBaseClient,
+)
+
+"""
+Thin abstractions of reconcile.ocm module to reduce coupling.
+"""
+
+DTP_TENANT_LABEL = sre_capability_label_key("dtp", "tenant")
+DTP_SPEC_LABEL = sre_capability_label_key("dtp", "token-spec")
+DTP_LABEL_SEARCH = sre_capability_label_key("dtp", "%")
+
+
+class Cluster(BaseModel):
+    id: str
+    external_id: str
+    organization_id: str
+    dt_tenant: str
+    token_spec_name: str
+    is_hcp: bool
+
+    @staticmethod
+    def from_cluster_details(cluster: ClusterDetails) -> Cluster:
+        dt_tenant = cluster.labels.get_label_value(DTP_TENANT_LABEL)
+        token_spec_name = cluster.labels.get_label_value(DTP_SPEC_LABEL)
+        if not token_spec_name:
+            """
+            We want to stay backwards compatible.
+            Earlier version of DTP did not set a value for the label.
+            We fall back to a default token in that case.
+
+            Long-term, we want to remove this behavior.
+            """
+            token_spec_name = "hypershift-management-cluster-v1"
+        return Cluster(
+            id=cluster.ocm_cluster.id,
+            external_id=cluster.ocm_cluster.external_id,
+            organization_id=cluster.organization_id,
+            dt_tenant=dt_tenant,
+            token_spec_name=token_spec_name,
+            is_hcp=cluster.ocm_cluster.is_rosa_hypershift(),
+        )
+
+
+class OCMClient:
+    """
+    Thin OOP wrapper around OCMBaseClient to avoid function mocking in tests
+    """
+
+    def __init__(self, ocm_client: OCMBaseClient):
+        self._ocm_client = ocm_client
+
+    def create_syncset(self, cluster_id: str, syncset_map: Mapping) -> None:
+        create_syncset(
+            ocm_client=self._ocm_client, cluster_id=cluster_id, syncset_map=syncset_map
+        )
+
+    def get_syncset(self, cluster_id: str, syncset_id: str) -> Any:
+        return get_syncset(
+            ocm_client=self._ocm_client, cluster_id=cluster_id, syncset_id=syncset_id
+        )
+
+    def patch_syncset(
+        self, cluster_id: str, syncset_id: str, syncset_map: Mapping
+    ) -> None:
+        patch_syncset(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            syncset_id=syncset_id,
+            syncset_map=syncset_map,
+        )
+
+    def create_manifest(self, cluster_id: str, manifest_map: Mapping) -> None:
+        create_manifest(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            manifest_map=manifest_map,
+        )
+
+    def get_manifest(self, cluster_id: str, manifest_id: str) -> Any:
+        return get_manifest(
+            ocm_client=self._ocm_client, cluster_id=cluster_id, manifest_id=manifest_id
+        )
+
+    def patch_manifest(
+        self, cluster_id: str, manifest_id: str, manifest_map: Mapping
+    ) -> None:
+        patch_manifest(
+            ocm_client=self._ocm_client,
+            cluster_id=cluster_id,
+            manifest_id=manifest_id,
+            manifest_map=manifest_map,
+        )
+
+    def discover_clusters_by_labels(self, label_filter: Filter) -> list[Cluster]:
+        return [
+            Cluster.from_cluster_details(cluster)
+            for cluster in discover_clusters_by_labels(
+                ocm_api=self._ocm_client, label_filter=label_filter
+            )
+        ]
+
+    def create_service_log(
+        self,
+        service_log: OCMClusterServiceLogCreateModel,
+        dedup_interval: timedelta | None,
+    ) -> None:
+        create_service_log(
+            ocm_api=self._ocm_client,
+            service_log=service_log,
+            dedup_interval=dedup_interval,
+        )

reconcile/dynatrace_token_provider/validate.py

@@ -0,0 +1,48 @@
+from collections.abc import Iterable
+
+from reconcile.gql_definitions.dynatrace_token_provider.token_specs import (
+    DynatraceTokenProviderTokenSpecV1,
+)
+
+
+class SecretNotUniqueError(Exception):
+    pass
+
+
+class TokenNameNotUniqueInSecretError(Exception):
+    pass
+
+
+class KeyNameNotUniqueInSecretError(Exception):
+    pass
+
+
+def validate_token_specs(specs: Iterable[DynatraceTokenProviderTokenSpecV1]) -> None:
+    """
+    We cannot catch all potential errors through json schema definition.
+    """
+    for spec in specs:
+        seen_secrets: set[str] = set()
+        for secret in spec.secrets:
+            secret_definition_key = f"{secret.namespace}/{secret.name}"
+            if secret_definition_key in seen_secrets:
+                raise SecretNotUniqueError(
+                    f"A secret cannot be re-defined and must be unique per spec. Secret '{secret.name}' in namespace '{secret.namespace}' is defined multiple times in spec '{spec.name}'."
+                )
+            seen_secrets.add(secret_definition_key)
+
+            seen_tokens: set[str] = set()
+            seen_key_names: set[str] = set()
+            for token in secret.tokens:
+                if token.name in seen_tokens:
+                    raise TokenNameNotUniqueInSecretError(
+                        f"A token name must be unique within a secret. Token name '{token.name}' is used more than once in secret '{secret.name}' in token spec '{spec.name}'."
+                    )
+                seen_tokens.add(token.name)
+
+                secret_key = token.key_name_in_secret or token.name
+                if secret_key in seen_key_names:
+                    raise KeyNameNotUniqueInSecretError(
+                        f"A key name must be unique within a secret. Key name '{secret_key}' is used more than once in secret '{secret.name}' in token spec '{spec.name}'."
+                    )
+                seen_key_names.add(secret_key)

reconcile/endpoints_discovery/integration.py

@@ -0,0 +1,348 @@
+import logging
+from collections import defaultdict
+from collections.abc import Callable, Iterable
+from typing import TypedDict
+
+import jinja2
+from pydantic import BaseModel
+
+from reconcile.endpoints_discovery.merge_request import Renderer, create_parser
+from reconcile.endpoints_discovery.merge_request_manager import (
+    App,
+    Endpoint,
+    EndpointsToAdd,
+    EndpointsToChange,
+    EndpointsToDelete,
+    MergeRequestManager,
+)
+from reconcile.gql_definitions.endpoints_discovery.namespaces import (
+    AppEndPointsV1,
+    NamespaceV1,
+)
+from reconcile.gql_definitions.endpoints_discovery.namespaces import (
+    query as namespaces_query,
+)
+from reconcile.typed_queries.app_interface_repo_url import get_app_interface_repo_url
+from reconcile.typed_queries.github_orgs import get_github_orgs
+from reconcile.typed_queries.gitlab_instances import get_gitlab_instances
+from reconcile.utils import gql
+from reconcile.utils.defer import defer
+from reconcile.utils.differ import diff_any_iterables
+from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.extended_early_exit import (
+    ExtendedEarlyExitRunnerResult,
+    extended_early_exit_run,
+)
+from reconcile.utils.oc_map import OCMap, init_oc_map_from_namespaces
+from reconcile.utils.ruamel import create_ruamel_instance
+from reconcile.utils.runtime.integration import (
+    PydanticRunParams,
+    QontractReconcileIntegration,
+)
+from reconcile.utils.semver_helper import make_semver
+from reconcile.utils.unleash import get_feature_toggle_state
+from reconcile.utils.vcs import VCS
+
+QONTRACT_INTEGRATION = "endpoints-discovery"
+QONTRACT_INTEGRATION_VERSION = make_semver(1, 0, 1)
+
+
+class EndpointsDiscoveryIntegrationParams(PydanticRunParams):
+    thread_pool_size: int = 10
+    internal: bool | None = None
+    use_jump_host: bool = True
+    cluster_name: set[str] | None = None
+    namespace_name: str | None = None
+    endpoint_tmpl_resource: str = "/endpoints-discovery/endpoint-template.yml"
+    # extended early exit parameters
+    enable_extended_early_exit: bool = False
+    extended_early_exit_cache_ttl_seconds: int = 7200  # run every 2 hours
+    log_cached_log_output: bool = False
+
+
+class Route(BaseModel):
+    name: str
+    host: str
+    tls: bool
+
+    @property
+    def url(self) -> str:
+        return f"{self.host}:{443 if self.tls else 80}"
+
+
+def endpoint_prefix(namespace: NamespaceV1) -> str:
+    return f"{QONTRACT_INTEGRATION}/{namespace.cluster.name}/{namespace.name}/"
+
+
+def compile_endpoint_name(endpoint_prefix: str, route: Route) -> str:
+    return f"{endpoint_prefix}{route.name}"
+
+
+def render_template(template: str, endpoint_name: str, route: Route) -> dict:
+    yml = create_ruamel_instance()
+    return yml.load(
+        jinja2.Template(
+            template,
+            undefined=jinja2.StrictUndefined,
+            trim_blocks=True,
+            lstrip_blocks=True,
+            keep_trailing_newline=True,
+        ).render({"endpoint_name": endpoint_name, "route": route})
+    )
+
+
+class RunnerParams(TypedDict):
+    oc_map: OCMap
+    merge_request_manager: MergeRequestManager
+    endpoint_template: str
+    namespaces: Iterable[NamespaceV1]
+
+
+class EndpointsDiscoveryIntegration(
+    QontractReconcileIntegration[EndpointsDiscoveryIntegrationParams]
+):
+    """Discover routes from all OpenShift clusters and update endPoints in app-interface."""
+
+    @property
+    def name(self) -> str:
+        return QONTRACT_INTEGRATION
+
+    def get_desired_state_shard_config(self) -> None:
+        """Sharding (per cluster) is not supported for this integration.
+
+        An application can have endpoints in multiple clusters and this may cause merge conflicts."""
+        return None
+
+    def get_namespaces(
+        self,
+        query_func: Callable,
+        cluster_names: Iterable[str] | None = None,
+        namespace_name: str | None = None,
+    ) -> list[NamespaceV1]:
+        """Return namespaces to consider for the integration."""
+        return [
+            ns
+            for ns in namespaces_query(query_func).namespaces or []
+            if integration_is_enabled(self.name, ns.cluster)
+            and (not cluster_names or ns.cluster.name in cluster_names)
+            and (not namespace_name or ns.name == namespace_name)
+            and not ns.delete
+        ]
+
+    def get_routes(self, oc_map: OCMap, namespace: NamespaceV1) -> list[Route]:
+        """Return the routes for the given namespace."""
+        oc = oc_map.get_cluster(namespace.cluster.name)
+        if not oc.project_exists(namespace.name):
+            logging.info(
+                f"{namespace.cluster.name}/{namespace.name}: Namespace does not exist (yet). Skipping for now!"
+            )
+            return []
+
+        routes = defaultdict(list)
+        for item in oc.get_items(kind="Route", namespace=namespace.name):
+            tls = bool(item["spec"].get("tls"))
+            host = item["spec"]["host"]
+            # group all routes with the same hostname/tls
+            routes[host, tls].append(item["metadata"]["name"])
+
+        # merge all routes with the same hostname into one and combine the names
+        return [
+            Route(
+                name="|".join(sorted(names)),
+                host=host,
+                tls=tls,
+            )
+            for (host, tls), names in routes.items()
+        ]
+
+    def get_endpoint_changes(
+        self,
+        app: str,
+        endpoint_prefix: str,
+        endpoint_template: str,
+        endpoints: Iterable[AppEndPointsV1],
+        routes: Iterable[Route],
+    ) -> tuple[EndpointsToAdd, EndpointsToChange, EndpointsToDelete]:
+        """Get all new/changed/deleted endpoints for the given namespace."""
+        if not routes and not endpoints:
+            # nothing to do
+            return [], [], []
+
+        diff = diff_any_iterables(
+            # exclude manual endpoints
+            current=[
+                endpoint
+                for endpoint in endpoints
+                if endpoint.name.startswith(endpoint_prefix)
+            ]
+            or [],
+            desired=routes,
+            # names are unique, so we can use them as keys
+            current_key=lambda endpoint: endpoint.name,
+            desired_key=lambda route: compile_endpoint_name(endpoint_prefix, route),
+            # compare the endpoint and route by url.
+            # we can't use other endpoint attributes because we don't want to query them.
+            # there is a note about that behavior in the template.
+            equal=lambda endpoint, route: endpoint.url == route.url,
+        )
+
+        endpoints_to_add = []
+        endpoints_to_change = []
+        endpoints_to_delete = []
+
+        for add in diff.add.values():
+            logging.info(f"{app}: Adding endpoint for route {add.name}")
+            endpoints_to_add.append(
+                Endpoint(
+                    name=compile_endpoint_name(endpoint_prefix, add),
+                    data=render_template(
+                        endpoint_template,
+                        endpoint_name=compile_endpoint_name(endpoint_prefix, add),
+                        route=add,
+                    ),
+                )
+            )
+
+        for pair in diff.change.values():
+            logging.info(
+                f"{app}: Changing endpoint {pair.current.name} for route {pair.desired.name}"
+            )
+            endpoints_to_change.append(
+                Endpoint(
+                    name=pair.current.name,
+                    data=render_template(
+                        endpoint_template,
+                        endpoint_name=compile_endpoint_name(
+                            endpoint_prefix, pair.desired
+                        ),
+                        route=pair.desired,
+                    ),
+                )
+            )
+        for delete in diff.delete.values():
+            logging.info(f"{app}: Deleting endpoint for route {delete.name}")
+            endpoints_to_delete.append(Endpoint(name=delete.name))
+        return endpoints_to_add, endpoints_to_change, endpoints_to_delete
+
+    def get_apps(
+        self, oc_map: OCMap, endpoint_template: str, namespaces: Iterable[NamespaceV1]
+    ) -> list[App]:
+        """Compile a list of apps with their endpoints to add, change and delete."""
+        apps: dict[str, App] = {}
+        for namespace in namespaces:
+            logging.debug(
+                f"Processing namespace {namespace.cluster.name}/{namespace.name}"
+            )
+            routes = self.get_routes(oc_map, namespace)
+            endpoints_to_add, endpoints_to_change, endpoints_to_delete = (
+                self.get_endpoint_changes(
+                    app=namespace.app.name,
+                    endpoint_prefix=endpoint_prefix(namespace),
+                    endpoint_template=endpoint_template,
+                    endpoints=namespace.app.end_points or [],
+                    routes=routes,
+                )
+            )
+            # update the app with the endpoints per namespace
+            app = apps.setdefault(
+                namespace.app.path,
+                App(name=namespace.app.name, path=namespace.app.path),
+            )
+            app.endpoints_to_add += endpoints_to_add
+            app.endpoints_to_change += endpoints_to_change
+            app.endpoints_to_delete += endpoints_to_delete
+
+        # return only apps endpoint changes
+        return [
+            app
+            for app in apps.values()
+            if app.endpoints_to_add
+            or app.endpoints_to_change
+            or app.endpoints_to_delete
+        ]
+
+    def runner(
+        self,
+        oc_map: OCMap,
+        merge_request_manager: MergeRequestManager,
+        endpoint_template: str,
+        namespaces: Iterable[NamespaceV1],
+    ) -> ExtendedEarlyExitRunnerResult:
+        """Reconcile the endpoints for all namespaces."""
+        apps = self.get_apps(oc_map, endpoint_template, namespaces)
+        merge_request_manager.create_merge_request(apps=apps)
+        return ExtendedEarlyExitRunnerResult(payload={}, applied_count=len(apps))
+
+    @defer
+    def run(self, dry_run: bool, defer: Callable | None = None) -> None:
+        """Run the integration."""
+        gql_api = gql.get_api()
+        namespaces = self.get_namespaces(
+            gql_api.query,
+            cluster_names=self.params.cluster_name,
+            namespace_name=self.params.namespace_name,
+        )
+        if not namespaces:
+            # nothing to do
+            return
+
+        oc_map = init_oc_map_from_namespaces(
+            namespaces=namespaces,
+            secret_reader=self.secret_reader,
+            integration=QONTRACT_INTEGRATION,
+            use_jump_host=self.params.use_jump_host,
+            thread_pool_size=self.params.thread_pool_size,
+            internal=self.params.internal,
+            init_projects=True,
+        )
+
+        if defer:
+            defer(oc_map.cleanup)
+
+        vcs = VCS(
+            secret_reader=self.secret_reader,
+            github_orgs=get_github_orgs(),
+            gitlab_instances=get_gitlab_instances(),
+            app_interface_repo_url=get_app_interface_repo_url(),
+            dry_run=dry_run,
+            allow_deleting_mrs=True,
+            allow_opening_mrs=True,
+        )
+        if defer:
+            defer(vcs.cleanup)
+        merge_request_manager = MergeRequestManager(
+            vcs=vcs,
+            renderer=Renderer(),
+            parser=create_parser(),
+            auto_merge_enabled=get_feature_toggle_state(
+                integration_name=f"{self.name}-allow-auto-merge-mrs", default=False
+            ),
+        )
+        endpoint_template = gql_api.get_resource(
+            path=self.params.endpoint_tmpl_resource
+        )["content"]
+
+        runner_params: RunnerParams = {
+            "oc_map": oc_map,
+            "merge_request_manager": merge_request_manager,
+            "endpoint_template": endpoint_template,
+            "namespaces": namespaces,
+        }
+
+        if self.params.enable_extended_early_exit and get_feature_toggle_state(
+            f"{QONTRACT_INTEGRATION}-extended-early-exit", default=True
+        ):
+            extended_early_exit_run(
+                integration=QONTRACT_INTEGRATION,
+                integration_version=QONTRACT_INTEGRATION_VERSION,
+                dry_run=dry_run,
+                cache_source=self.get_early_exit_desired_state(),
+                shard="",
+                ttl_seconds=self.params.extended_early_exit_cache_ttl_seconds,
+                logger=logging.getLogger(),
+                runner=self.runner,
+                runner_params=runner_params,
+                log_cached_log_output=self.params.log_cached_log_output,
+            )
+        else:
+            self.runner(**runner_params)