qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/endpoints_discovery/merge_request.py

@@ -0,0 +1,96 @@
+import re
+import string
+
+from pydantic import BaseModel
+from ruamel.yaml.compat import StringIO
+
+from reconcile.utils.merge_request_manager.parser import Parser
+from reconcile.utils.ruamel import create_ruamel_instance
+
+VERSION = "1.0.0"
+INTEGRATION = "endpoints-discovery"
+LABEL = "ENDPOINTS-DISCOVERY"
+
+INTEGRATION_REF = "integration"
+VERSION_REF = "endpoints-discover-version"
+HASH_REF = "hash"
+COMPILED_REGEXES = {
+    i: re.compile(rf".*{i}: (.*)$", re.MULTILINE)
+    for i in [INTEGRATION_REF, VERSION_REF, HASH_REF]
+}
+
+PROMOTION_DATA_SEPARATOR = "**DO NOT MANUALLY CHANGE ANYTHING BELOW THIS LINE**"
+DESC = string.Template(
+    f"""
+This MR is triggered by app-interface's [endpoints-discovery](https://github.com/app-sre/qontract-reconcile/tree/master/reconcile/endpoints_discovery).
+
+Please **do not remove the `{LABEL}` label** from this MR!
+
+Parts of this description are used to manage the MR.
+
+{PROMOTION_DATA_SEPARATOR}
+
+* {INTEGRATION_REF}: {INTEGRATION}
+* {VERSION_REF}: {VERSION}
+* {HASH_REF}: $hash
+"""
+)
+
+
+class EPDInfo(BaseModel):
+    integration: str = INTEGRATION
+    hash: str
+
+
+def create_parser() -> Parser:
+    return Parser[EPDInfo](
+        klass=EPDInfo,
+        compiled_regexes=COMPILED_REGEXES,
+        version_ref=VERSION_REF,
+        expected_version=VERSION,
+        data_separator=PROMOTION_DATA_SEPARATOR,
+    )
+
+
+class Renderer:
+    """
+    This class is only concerned with rendering text for MRs.
+    Most logic evolves around ruamel yaml modification.
+    This class makes testing for MergeRequestManager easier.
+
+    Note, that this class is very susceptible to schema changes
+    as it mainly works on raw dicts.
+    """
+
+    def render_merge_request_content(
+        self,
+        current_content: str,
+        endpoints_to_add: list[dict],
+        endpoints_to_change: dict[str, dict],
+        endpoints_to_delete: list[str],
+    ) -> str:
+        """Update the app-interface app file for a merge request."""
+        yml = create_ruamel_instance(explicit_start=True)
+        content = yml.load(current_content)
+        new_endpoints = []
+        for app_endpoint in content.setdefault("endPoints", []):
+            if app_endpoint["name"] in endpoints_to_delete:
+                continue
+            elif app_endpoint["name"] in endpoints_to_change:
+                new_endpoints.append(endpoints_to_change[app_endpoint["name"]])
+            else:
+                new_endpoints.append(app_endpoint)
+
+        new_endpoints.extend(endpoints_to_add)
+        content["endPoints"] = new_endpoints
+        with StringIO() as stream:
+            yml.dump(content, stream)
+            return stream.getvalue()
+
+    def render_description(self, hash: str) -> str:
+        """Render the description for a merge request."""
+        return DESC.safe_substitute(EPDInfo(hash=hash).dict())
+
+    def render_title(self) -> str:
+        """Render the title for a merge request."""
+        return f"[auto] {INTEGRATION}: update application endpoints"

reconcile/endpoints_discovery/merge_request_manager.py

@@ -0,0 +1,178 @@
+import hashlib
+import json
+import logging
+from collections.abc import Sequence
+from typing import Any, TypeAlias
+
+from gitlab.exceptions import GitlabGetError
+from pydantic import BaseModel
+
+from reconcile.endpoints_discovery.merge_request import (
+    INTEGRATION,
+    INTEGRATION_REF,
+    LABEL,
+    EPDInfo,
+    Parser,
+    Renderer,
+)
+from reconcile.utils.gitlab_api import GitLabApi
+from reconcile.utils.merge_request_manager.merge_request_manager import (
+    MergeRequestManagerBase,
+)
+from reconcile.utils.mr.base import MergeRequestBase
+from reconcile.utils.mr.labels import AUTO_MERGE
+from reconcile.utils.vcs import VCS
+
+
+class EndpointsDiscoveryMR(MergeRequestBase):
+    name = "endpoints-discovery"
+
+    def __init__(self, title: str, description: str, labels: list[str]):
+        super().__init__()
+        self._title = title
+        self._description = description
+        self.labels = labels
+        self._commits: list[tuple[str, str, str]] = []
+
+    @property
+    def title(self) -> str:
+        return self._title
+
+    @property
+    def description(self) -> str:
+        return self._description
+
+    def add_commit(self, path: str, content: str, msg: str) -> None:
+        self._commits.append((path, content, msg))
+
+    def process(self, gitlab_cli: GitLabApi) -> None:
+        for path, content, msg in self._commits:
+            gitlab_cli.update_file(
+                branch_name=self.branch,
+                file_path=path,
+                commit_message=msg,
+                content=content,
+            )
+
+
+class Endpoint(BaseModel):
+    # the current endpoint name to change or delete. It doesn't matter for new endpoints
+    name: str
+    # the endpoint data will be generated and rendered from the endpoint template resource
+    # see EndpointsDiscoveryIntegrationParams.endpoint_tmpl_resource
+    data: dict[str, Any] = {}
+
+    @property
+    def hash(self) -> str:
+        return hashlib.sha256(
+            json.dumps(self.dict(), sort_keys=True).encode()
+        ).hexdigest()
+
+
+EndpointsToAdd: TypeAlias = list[Endpoint]
+EndpointsToChange: TypeAlias = list[Endpoint]
+EndpointsToDelete: TypeAlias = list[Endpoint]
+
+
+class App(BaseModel):
+    name: str
+    path: str
+    endpoints_to_add: EndpointsToAdd = EndpointsToAdd()
+    endpoints_to_change: EndpointsToChange = EndpointsToChange()
+    endpoints_to_delete: EndpointsToDelete = EndpointsToDelete()
+
+    @property
+    def hash(self) -> str:
+        return hashlib.sha256(
+            f"""
+                {self.path}
+                {[i.hash for i in sorted(self.endpoints_to_add, key=lambda i: i.name)]}
+                {[i.hash for i in sorted(self.endpoints_to_change, key=lambda i: i.name)]}
+                {[i.hash for i in sorted(self.endpoints_to_delete, key=lambda i: i.name)]}
+            """.encode()
+        ).hexdigest()
+
+
+def hash_apps(apps: Sequence[App]) -> str:
+    return hashlib.sha256(
+        ",".join(app.hash for app in sorted(apps, key=lambda i: i.name)).encode()
+    ).hexdigest()
+
+
+class MergeRequestManager(MergeRequestManagerBase[EPDInfo]):
+    """
+    Manager for AVS merge requests. This class
+    is responsible for housekeeping (closing old/bad MRs) and
+    opening new MRs for external resources that have new versions.
+
+    For each external resource, there are exist just one MR to update
+    the version number in the App-Interface. Old or obsolete MRs are
+    closed automatically.
+    """
+
+    def __init__(
+        self, vcs: VCS, renderer: Renderer, parser: Parser, auto_merge_enabled: bool
+    ):
+        super().__init__(vcs, parser, LABEL)
+        self._renderer = renderer
+        self._auto_merge_enabled = auto_merge_enabled
+
+    def create_merge_request(self, apps: Sequence[App]) -> None:
+        """Open new MR (if not already present) for apps and close any outdated before."""
+        if not self._housekeeping_ran:
+            self.housekeeping()
+
+        apps_hash = hash_apps(apps)
+        # we support only one MR at a time for all apps
+        if mr := self._merge_request_already_exists({INTEGRATION_REF: INTEGRATION}):
+            if mr.mr_info.hash == apps_hash:
+                logging.info(
+                    f"Found an open MR for {INTEGRATION} and it's up-to-date - doing nothing."
+                )
+                return None
+            logging.info(f"Found an outdated MR for {INTEGRATION} - closing it.")
+            self._vcs.close_app_interface_mr(
+                mr.raw, "Closing this MR because it's outdated."
+            )
+            # don't open a new MR right now, because the deletion of the old MRs could be
+            # disabled. In this case, we would end up with multiple open MRs for the
+            # same external resource.
+            return None
+
+        if not apps:
+            return None
+
+        endpoints_discovery_mr = EndpointsDiscoveryMR(
+            title=self._renderer.render_title(),
+            description=self._renderer.render_description(hash=apps_hash),
+            labels=[LABEL] + ([AUTO_MERGE] if self._auto_merge_enabled else []),
+        )
+        for app in apps:
+            try:
+                content = self._vcs.get_file_content_from_app_interface_master(
+                    file_path=app.path
+                )
+            except GitlabGetError as e:
+                if e.response_code == 404:
+                    logging.error(
+                        "The file %s does not exist anylonger. Most likely qontract-server data not in synch. This should resolve soon on its own.",
+                        app.path,
+                    )
+                    return None
+                raise e
+            content = self._renderer.render_merge_request_content(
+                current_content=content,
+                endpoints_to_add=[item.data for item in app.endpoints_to_add],
+                endpoints_to_change={
+                    item.name: item.data for item in app.endpoints_to_change
+                },
+                endpoints_to_delete=[item.name for item in app.endpoints_to_delete],
+            )
+            endpoints_discovery_mr.add_commit(
+                path=f"data/{app.path.lstrip('/')}",
+                content=content,
+                msg=f"endpoints-discovery: update application endpoints for {app.name}",
+            )
+
+        logging.info("Open MR for %d app(s)", len(apps))
+        self._vcs.open_app_interface_merge_request(mr=endpoints_discovery_mr)

reconcile/external_resources/aws.py

@@ -0,0 +1,204 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from reconcile.external_resources.model import (
+    ExternalResource,
+    ExternalResourceKey,
+    ExternalResourcesInventory,
+)
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+from reconcile.utils.external_resources import ResourceValueResolver
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class AWSResourceFactory(ABC):
+    def __init__(
+        self, er_inventory: ExternalResourcesInventory, secret_reader: SecretReaderBase
+    ):
+        self.er_inventory = er_inventory
+        self.secret_reader = secret_reader
+
+    @abstractmethod
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]: ...
+
+    @abstractmethod
+    def validate(self, resource: ExternalResource) -> None: ...
+
+    def find_linked_resources(
+        self, spec: ExternalResourceSpec
+    ) -> set[ExternalResourceKey]:
+        """Method to find dependant resources. Resources in this list
+        will be reconciled every time the parent resource finishes its reconciliation."""
+        return set()
+
+
+class AWSDefaultResourceFactory(AWSResourceFactory):
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        return ResourceValueResolver(spec=spec, identifier_as_value=True).resolve()
+
+    def validate(self, resource: ExternalResource) -> None: ...
+
+
+class AWSElasticacheFactory(AWSDefaultResourceFactory):
+    def _get_source_db_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "elasticache", identifier
+        )
+
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        """Resolve the elasticache resource specification and translate some attributes to AWS >= 5.60.0 provider format."""
+        rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
+        data = rvr.resolve()
+        data["output_prefix"] = spec.output_prefix
+
+        if "replication_group_id" not in data:
+            data["replication_group_id"] = spec.identifier
+
+        if cluster_mode := data.pop("cluster_mode", {}):
+            for k, v in cluster_mode.items():
+                data[k] = v
+
+        if "parameter_group" in data:
+            pg_data = rvr._get_values(data["parameter_group"])
+            data["parameter_group"] = pg_data
+
+        return data
+
+    def validate(self, resource: ExternalResource) -> None:
+        """Validate the elasticache resource specification."""
+        data = resource.data
+        if data.get("parameter_group"):
+            if not data["parameter_group"].get("name"):
+                data["parameter_group"]["name"] = f"{data['replication_group_id']}-pg"
+            else:
+                # prefix the parameter_group name with the replication_group_id
+                data["parameter_group"]["name"] = (
+                    f"{data['replication_group_id']}-{data['parameter_group']['name']}"
+                )
+
+            if (
+                data.get("parameter_group_name")
+                and data["parameter_group"]["name"] != data["parameter_group_name"]
+            ):
+                raise ValueError(
+                    "Custom parameter_group set and parameter_group_name given. Either remove parameter_group_name or set it to the same value as parameter_group.name."
+                )
+
+            if not data.get("parameter_group_name"):
+                # automatically set /aws/elasticache-defaults-1.yml/parameter_group_name to /aws/parameter-group-1.yml/name
+                data["parameter_group_name"] = data["parameter_group"]["name"]
+
+
+class AWSRdsFactory(AWSDefaultResourceFactory):
+    def _get_source_db_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "rds", identifier
+        )
+
+    def _get_kms_key_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "kms", identifier
+        )
+
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
+        data = rvr.resolve()
+
+        data["output_prefix"] = spec.output_prefix
+
+        if "parameter_group" in data:
+            pg_data = rvr._get_values(data["parameter_group"])
+            data["parameter_group"] = pg_data
+        if "old_parameter_group" in data:
+            old_pg_data = rvr._get_values(data["old_parameter_group"])
+            data["old_parameter_group"] = old_pg_data
+        if "replica_source" in data:
+            sourcedb_spec = self._get_source_db_spec(
+                spec.provisioner_name, data["replica_source"]
+            )
+            sourcedb = self.resolve(sourcedb_spec)
+            sourcedb_region = (
+                sourcedb.get("region", None)
+                or sourcedb_spec.provisioner["resources_default_region"]
+            )
+            data["replica_source"] = {
+                "identifier": sourcedb["identifier"],
+                "region": sourcedb_region,
+            }
+
+        kms_key_id: str = data.get("kms_key_id", None)
+        if kms_key_id and not kms_key_id.startswith("arn:"):
+            data["kms_key_id"] = self._get_kms_key_spec(
+                spec.provisioner_name, kms_key_id
+            ).identifier
+
+        return data
+
+    def validate(self, resource: ExternalResource) -> None: ...
+
+    def find_linked_resources(
+        self, spec: ExternalResourceSpec
+    ) -> set[ExternalResourceKey]:
+        return {
+            k
+            for k, s in self.er_inventory.items()
+            if s.provision_provider == "aws"
+            and s.provider == "rds"
+            and s.resource["replica_source"] == spec.identifier
+        }
+
+
+class AWSMskFactory(AWSDefaultResourceFactory):
+    def _get_source_db_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "msk", identifier
+        )
+
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
+        data = rvr.resolve()
+        data["output_prefix"] = spec.output_prefix
+
+        scram_enabled = (
+            data.get("client_authentication", {}).get("sasl", {}).get("scram", False)
+        )
+        data["scram_users"] = {}
+        if scram_enabled:
+            if not data.get("users", []):
+                raise ValueError(
+                    "users attribute must be given when client_authentication.sasl.scram is enabled."
+                )
+            data["scram_users"] = {
+                user["name"]: self.secret_reader.read_all(user["secret"])
+                for user in data["users"]
+            }
+            # the users attribute is not needed in the final data
+            del data["users"]
+        return data
+
+    def validate(self, resource: ExternalResource) -> None:
+        data = resource.data
+        if (
+            data["number_of_broker_nodes"]
+            % len(data["broker_node_group_info"]["client_subnets"])
+            != 0
+        ):
+            raise ValueError(
+                "number_of_broker_nodes must be a multiple of the number of specified client subnets."
+            )
+        # validate user objects
+        for user, secret in data["scram_users"].items():
+            if secret.keys() != {"password", "username"}:
+                raise ValueError(
+                    f"MSK user '{user}' secret must contain only 'username' and 'password' keys!"
+                )

reconcile/external_resources/factories.py

@@ -0,0 +1,163 @@
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from typing import Generic, TypeVar
+
+from reconcile.external_resources.aws import (
+    AWSDefaultResourceFactory,
+    AWSElasticacheFactory,
+    AWSMskFactory,
+    AWSRdsFactory,
+    AWSResourceFactory,
+)
+from reconcile.external_resources.meta import QONTRACT_INTEGRATION
+from reconcile.external_resources.model import (
+    ExternalResource,
+    ExternalResourceKey,
+    ExternalResourceProvision,
+    ExternalResourcesInventory,
+    ModuleInventory,
+    ModuleProvisionData,
+    TerraformModuleProvisionData,
+)
+from reconcile.gql_definitions.external_resources.external_resources_settings import (
+    ExternalResourcesSettingsV1,
+)
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+T = TypeVar("T")
+
+AWS_DEFAULT_TAGS = [
+    {
+        "tags": {
+            "app": "app-sre-infra",
+        }
+    }
+]
+
+
+class ObjectFactory(Generic[T]):
+    def __init__(self) -> None:
+        self._factories: dict[str, T] = {}
+
+    def register_factory(self, id: str, t: T) -> None:
+        self._factories[id] = t
+
+    def get_factory(self, id: str) -> T:
+        return self._factories[id]
+
+
+class ExternalResourceFactory(ABC):
+    @abstractmethod
+    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+        pass
+
+    @abstractmethod
+    def validate_external_resource(self, resource: ExternalResource) -> None:
+        pass
+
+    def find_linked_resources(
+        self, spec: ExternalResourceSpec
+    ) -> set[ExternalResourceKey]:
+        """Method to find dependant resources. Resources in this list
+        will be reconciled every time the parent resource finishes its reconciliation."""
+        return set()
+
+
+class ModuleProvisionDataFactory(ABC):
+    @abstractmethod
+    def create_provision_data(self, ers: ExternalResourceSpec) -> ModuleProvisionData:
+        pass
+
+
+class TerraformModuleProvisionDataFactory(ModuleProvisionDataFactory):
+    def __init__(self, settings: ExternalResourcesSettingsV1):
+        self.settings = settings
+
+    def create_provision_data(
+        self, spec: ExternalResourceSpec
+    ) -> TerraformModuleProvisionData:
+        key = ExternalResourceKey.from_spec(spec)
+
+        return TerraformModuleProvisionData(
+            tf_state_bucket=self.settings.tf_state_bucket,
+            tf_state_region=self.settings.tf_state_region,
+            tf_state_dynamodb_table=self.settings.tf_state_dynamodb_table,
+            tf_state_key=key.state_path + "/terraform.tfstate",
+        )
+
+
+def setup_aws_resource_factories(
+    er_inventory: ExternalResourcesInventory, secret_reader: SecretReaderBase
+) -> ObjectFactory[AWSResourceFactory]:
+    f = ObjectFactory[AWSResourceFactory]()
+    f.register_factory(
+        "elasticache", AWSElasticacheFactory(er_inventory, secret_reader)
+    )
+    f.register_factory("rds", AWSRdsFactory(er_inventory, secret_reader))
+    f.register_factory("msk", AWSMskFactory(er_inventory, secret_reader))
+    f.register_factory(
+        "default", AWSDefaultResourceFactory(er_inventory, secret_reader)
+    )
+    return f
+
+
+class AWSExternalResourceFactory(ExternalResourceFactory):
+    def __init__(
+        self,
+        module_inventory: ModuleInventory,
+        er_inventory: ExternalResourcesInventory,
+        secret_reader: SecretReaderBase,
+        provision_factories: ObjectFactory[ModuleProvisionDataFactory],
+        resource_factories: ObjectFactory[AWSResourceFactory],
+    ):
+        self.provision_factories = provision_factories
+        self.resource_factories = resource_factories
+        self.module_inventory = module_inventory
+        self.er_inventory = er_inventory
+        self.secret_reader = secret_reader
+
+    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+        f = self.resource_factories.get_factory(spec.provider)
+        data = f.resolve(spec)
+        data["tags"] = spec.tags(integration=QONTRACT_INTEGRATION)
+        data["default_tags"] = AWS_DEFAULT_TAGS
+
+        region = data.get("region")
+        if region:
+            if region not in spec.provisioner["supported_deployment_regions"]:
+                raise ValueError(region)
+        else:
+            region = spec.provisioner["resources_default_region"]
+        data["region"] = region
+
+        module_type = self.module_inventory.get_from_spec(spec).module_type
+        provision_factory = self.provision_factories.get_factory(module_type)
+        module_provision_data = provision_factory.create_provision_data(spec)
+
+        provision = ExternalResourceProvision(
+            provision_provider=spec.provision_provider,
+            provisioner=spec.provisioner_name,
+            provider=spec.provider,
+            identifier=spec.identifier,
+            target_cluster=spec.cluster_name,
+            target_namespace=spec.namespace_name,
+            target_secret_name=spec.output_resource_name,
+            module_provision_data=module_provision_data,
+        )
+
+        return ExternalResource(data=data, provision=provision)
+
+    def validate_external_resource(self, resource: ExternalResource) -> None:
+        f = self.resource_factories.get_factory(resource.provision.provider)
+        f.validate(resource)
+
+    def find_linked_resources(
+        self, spec: ExternalResourceSpec
+    ) -> set[ExternalResourceKey]:
+        f = self.resource_factories.get_factory(spec.provider)
+        return f.find_linked_resources(spec)