qontract-reconcile 0.10.1rc763__py3-none-any.whl → 0.10.1rc764__py3-none-any.whl

reconcile/external_resources/manager.py

@@ -0,0 +1,350 @@
+import json
+import logging
+from collections.abc import Iterable
+from datetime import datetime, timezone
+from enum import Enum
+
+from sretoolbox.utils import threaded
+
+from reconcile.external_resources.factories import (
+    AWSExternalResourceFactory,
+    ExternalResourceFactory,
+    ModuleProvisionDataFactory,
+    ObjectFactory,
+    TerraformModuleProvisionDataFactory,
+    setup_aws_resource_factories,
+)
+from reconcile.external_resources.metrics import ExternalResourcesReconcileErrorsGauge
+from reconcile.external_resources.model import (
+    Action,
+    ExternalResource,
+    ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
+    ExternalResourcesInventory,
+    ExternalResourceValidationError,
+    ModuleInventory,
+    Reconciliation,
+)
+from reconcile.external_resources.reconciler import ExternalResourcesReconciler
+from reconcile.external_resources.secrets_sync import InClusterSecretsReconciler
+from reconcile.external_resources.state import (
+    ExternalResourcesStateDynamoDB,
+    ExternalResourceState,
+    ReconcileStatus,
+    ResourceStatus,
+)
+from reconcile.gql_definitions.external_resources.external_resources_settings import (
+    ExternalResourcesSettingsV1,
+)
+from reconcile.utils import metrics
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
+
+
+def setup_factories(
+    settings: ExternalResourcesSettingsV1,
+    module_inventory: ModuleInventory,
+    er_inventory: ExternalResourcesInventory,
+    secret_reader: SecretReaderBase,
+) -> ObjectFactory[ExternalResourceFactory]:
+    tf_factory = TerraformModuleProvisionDataFactory(settings=settings)
+
+    aws_provision_factories = ObjectFactory[ModuleProvisionDataFactory]()
+    aws_provision_factories.register_factory("terraform", tf_factory)
+    aws_provision_factories.register_factory("cdktf", tf_factory)
+
+    of = ObjectFactory[ExternalResourceFactory]()
+    of.register_factory(
+        "aws",
+        AWSExternalResourceFactory(
+            module_inventory=module_inventory,
+            er_inventory=er_inventory,
+            secret_reader=secret_reader,
+            provision_factories=aws_provision_factories,
+            resource_factories=setup_aws_resource_factories(
+                er_inventory, secret_reader
+            ),
+        ),
+    )
+    return of
+
+
+class ReconcileAction(str, Enum):
+    NOOP = "NOOP"
+    APPLY_NOT_EXISTS = "Resource does not exist"
+    APPLY_ERROR = "Resource status in ERROR state"
+    APPLY_SPEC_CHANGED = "Resource spec has changed"
+    APPLY_DRIFT_DETECTION = "Resource drift detection run"
+    DESTROY_CREATED = "Resource no longer exists in the configuration"
+    DESTROY_ERROR = "Resource status in ERROR state"
+
+
+class ExternalResourcesManager:
+    def __init__(
+        self,
+        state_manager: ExternalResourcesStateDynamoDB,
+        settings: ExternalResourcesSettingsV1,
+        module_inventory: ModuleInventory,
+        reconciler: ExternalResourcesReconciler,
+        secret_reader: SecretReaderBase,
+        er_inventory: ExternalResourcesInventory,
+        factories: ObjectFactory[ExternalResourceFactory],
+        secrets_reconciler: InClusterSecretsReconciler,
+        thread_pool_size: int,
+    ) -> None:
+        self.state_mgr = state_manager
+        self.settings = settings
+        self.module_inventory = module_inventory
+        self.reconciler = reconciler
+        self.er_inventory = er_inventory
+        self.factories = factories
+        self.secret_reader = secret_reader
+        self.secrets_reconciler = secrets_reconciler
+        self.errors: dict[ExternalResourceKey, ExternalResourceValidationError] = {}
+        self.thread_pool_size = thread_pool_size
+
+    def _get_reconcile_action(
+        self, reconciliation: Reconciliation, state: ExternalResourceState
+    ) -> ReconcileAction:
+        if reconciliation.action == Action.APPLY:
+            match state.resource_status:
+                case ResourceStatus.NOT_EXISTS:
+                    return ReconcileAction.APPLY_NOT_EXISTS
+                case ResourceStatus.ERROR:
+                    return ReconcileAction.APPLY_ERROR
+                case ResourceStatus.CREATED:
+                    if (
+                        reconciliation.resource_hash
+                        != state.reconciliation.resource_hash
+                    ):
+                        return ReconcileAction.APPLY_SPEC_CHANGED
+                    elif (
+                        (datetime.now(state.ts.tzinfo) - state.ts).total_seconds()
+                        > reconciliation.module_configuration.reconcile_drift_interval_minutes
+                        * 60
+                    ):
+                        return ReconcileAction.APPLY_DRIFT_DETECTION
+        elif reconciliation.action == Action.DESTROY:
+            match state.resource_status:
+                case ResourceStatus.CREATED:
+                    return ReconcileAction.DESTROY_CREATED
+                case ResourceStatus.ERROR:
+                    return ReconcileAction.DESTROY_ERROR
+
+        return ReconcileAction.NOOP
+
+    def _resource_needs_reconciliation(
+        self,
+        reconciliation: Reconciliation,
+        state: ExternalResourceState,
+    ) -> bool:
+        reconcile_action = self._get_reconcile_action(reconciliation, state)
+        reconcile = reconcile_action != ReconcileAction.NOOP
+        if reconcile:
+            logging.info(
+                "Reconciling: Status: [%s], Action: [%s], reason: [%s], key:[%s]",
+                state.resource_status.value,
+                reconciliation.action.value,
+                reconcile_action.value,
+                reconciliation.key,
+            )
+        return reconcile
+
+    def _get_desired_objects_reconciliations(self) -> set[Reconciliation]:
+        r: set[Reconciliation] = set()
+        for key, spec in self.er_inventory.items():
+            module = self.module_inventory.get_from_spec(spec)
+
+            try:
+                resource = self._build_external_resource(spec, self.er_inventory)
+            except ExternalResourceValidationError as e:
+                k = ExternalResourceKey.from_spec(spec)
+                self.errors[k] = e
+                continue
+
+            reconciliation = Reconciliation(
+                key=key,
+                resource_hash=resource.hash(),
+                input=self._serialize_resource_input(resource),
+                action=Action.APPLY,
+                module_configuration=ExternalResourceModuleConfiguration.resolve_configuration(
+                    module, spec
+                ),
+            )
+            r.add(reconciliation)
+        return r
+
+    def _get_deleted_objects_reconciliations(self) -> set[Reconciliation]:
+        desired_keys = set(self.er_inventory.keys())
+        state_resource_keys = self.state_mgr.get_all_resource_keys()
+        deleted_keys = state_resource_keys - desired_keys
+        r: set[Reconciliation] = set()
+        for key in deleted_keys:
+            state = self.state_mgr.get_external_resource_state(key)
+            reconciliation = Reconciliation(
+                key=key,
+                resource_hash=state.reconciliation.resource_hash,
+                module_configuration=state.reconciliation.module_configuration,
+                input=state.reconciliation.input,
+                action=Action.DESTROY,
+            )
+            r.add(reconciliation)
+        return r
+
+    def _update_in_progress_state(
+        self, r: Reconciliation, state: ExternalResourceState
+    ) -> bool:
+        """Gets the resource reconciliation state from the Job and updates the
+        Resource state accordingly. It also returns if the target outputs secret needs
+        to be reconciled.
+
+        :param r: Reconciliation object
+        :param state: State object
+        :return: True/False if target Secret needs to be reconciled.
+        """
+        need_secret_sync = False
+
+        if state.resource_status not in set([
+            ResourceStatus.DELETE_IN_PROGRESS,
+            ResourceStatus.IN_PROGRESS,
+        ]):
+            return need_secret_sync
+
+        logging.info(
+            "Reconciliation In progress. Action: %s, Key:%s",
+            state.reconciliation.action,
+            state.reconciliation.key,
+        )
+
+        # Need to check the reconciliation set in the state, not the desired one
+        # as the reconciliation object might be from a previous desired state
+        error = False
+        match self.reconciler.get_resource_reconcile_status(state.reconciliation):
+            case ReconcileStatus.SUCCESS:
+                logging.info(
+                    "Reconciliation ended SUCCESSFULLY. Action: %s, key:%s",
+                    r.action.value,
+                    r.key,
+                )
+                if r.action == Action.APPLY:
+                    state.resource_status = ResourceStatus.CREATED
+                    state.reconciliation_errors = 0
+                    self.state_mgr.set_external_resource_state(state)
+                    need_secret_sync = True
+                elif r.action == Action.DESTROY:
+                    state.resource_status = ResourceStatus.DELETED
+                    self.state_mgr.del_external_resource_state(r.key)
+            case ReconcileStatus.ERROR:
+                logging.info(
+                    "Reconciliation ended with ERROR: Action:%s, Key:%s",
+                    r.action.value,
+                    r.key,
+                )
+                error = True
+            case ReconcileStatus.NOT_EXISTS:
+                logging.info(
+                    "Reconciliation should exist but it doesn't. Marking as ERROR to retrigger: Action:%s, Key:%s",
+                    r.action.value,
+                    r.key,
+                )
+                error = True
+        if error:
+            state.resource_status = ResourceStatus.ERROR
+            state.reconciliation_errors += 1
+            self.state_mgr.set_external_resource_state(state)
+
+        return need_secret_sync
+
+    def _update_state(self, r: Reconciliation, state: ExternalResourceState) -> None:
+        state.ts = datetime.now(timezone.utc)
+        if r.action == Action.APPLY:
+            state.resource_status = ResourceStatus.IN_PROGRESS
+        elif r.action == Action.DESTROY:
+            state.resource_status = ResourceStatus.DELETE_IN_PROGRESS
+        state.reconciliation = r
+        self.state_mgr.set_external_resource_state(state)
+
+    def _need_secret_sync(
+        self, r: Reconciliation, state: ExternalResourceState
+    ) -> bool:
+        return (
+            r.action == Action.APPLY and state.resource_status == ResourceStatus.CREATED
+        )
+
+    def _sync_secrets(self, keys: Iterable[ExternalResourceKey]) -> None:
+        specs = [spec for key in keys if (spec := self.er_inventory.get(key))]
+        self.secrets_reconciler.sync_secrets(specs=specs)
+
+    def _build_external_resource(
+        self, spec: ExternalResourceSpec, er_inventory: ExternalResourcesInventory
+    ) -> ExternalResource:
+        f = self.factories.get_factory(spec.provision_provider)
+        resource = f.create_external_resource(spec)
+        f.validate_external_resource(resource)
+        return resource
+
+    def _serialize_resource_input(self, resource: ExternalResource) -> str:
+        return json.dumps(
+            resource.dict(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
+        )
+
+    def handle_resources(self) -> None:
+        desired_r = self._get_desired_objects_reconciliations()
+        deleted_r = self._get_deleted_objects_reconciliations()
+        to_sync: set[ExternalResourceKey] = set()
+        for r in desired_r.union(deleted_r):
+            state = self.state_mgr.get_external_resource_state(r.key)
+            need_sync = self._update_in_progress_state(r, state)
+            if need_sync:
+                to_sync.add(r.key)
+
+            metrics.set_gauge(
+                ExternalResourcesReconcileErrorsGauge(
+                    provision_provider=r.key.provision_provider,
+                    provisioner_name=r.key.provisioner_name,
+                    provider=r.key.provider,
+                    identifier=r.key.identifier,
+                ),
+                float(state.reconciliation_errors),
+            )
+
+            if self._resource_needs_reconciliation(reconciliation=r, state=state):
+                self.reconciler.reconcile_resource(reconciliation=r)
+                self._update_state(r, state)
+
+        if to_sync:
+            self._sync_secrets(keys=to_sync)
+
+    def handle_dry_run_resources(self) -> None:
+        desired_r = self._get_desired_objects_reconciliations()
+        deleted_r = self._get_deleted_objects_reconciliations()
+        reconciliations = desired_r.union(deleted_r)
+        triggered = set[Reconciliation]()
+
+        for r in reconciliations:
+            state = self.state_mgr.get_external_resource_state(key=r.key)
+            if (
+                r.action == Action.APPLY
+                and state.reconciliation.resource_hash != r.resource_hash
+            ) or r.action == Action.DESTROY:
+                triggered.add(r)
+
+        threaded.run(
+            self.reconciler.reconcile_resource,
+            triggered,
+            thread_pool_size=self.thread_pool_size,
+        )
+
+        results = self.reconciler.wait_for_reconcile_list_completion(
+            triggered, check_interval_seconds=10, timeout_seconds=-1
+        )
+
+        for r in triggered:
+            self.reconciler.get_resource_reconcile_logs(reconciliation=r)
+
+        if ReconcileStatus.ERROR in [rs for rs in results.values()]:
+            raise Exception("Some Resources have reconciliation errors.")

reconcile/external_resources/meta.py

@@ -0,0 +1,4 @@
+from reconcile.utils.semver_helper import make_semver
+
+QONTRACT_INTEGRATION = "external_resources"
+QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)

reconcile/external_resources/metrics.py

@@ -0,0 +1,20 @@
+from pydantic import BaseModel
+
+from reconcile.utils.metrics import (
+    GaugeMetric,
+)
+
+
+class ExternalResourcesBaseMetric(BaseModel):
+    integration = "external_resources"
+
+
+class ExternalResourcesReconcileErrorsGauge(ExternalResourcesBaseMetric, GaugeMetric):
+    provision_provider: str
+    provisioner_name: str
+    provider: str
+    identifier: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "external_resources_reconcile_status"

reconcile/external_resources/model.py

@@ -0,0 +1,244 @@
+import base64
+import hashlib
+import json
+from abc import (
+    ABC,
+)
+from collections.abc import (
+    Iterable,
+    Iterator,
+    MutableMapping,
+)
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel
+
+from reconcile.gql_definitions.external_resources.external_resources_modules import (
+    ExternalResourcesModuleV1,
+)
+from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
+    NamespaceTerraformProviderResourceAWSV1,
+    NamespaceTerraformResourceRDSV1,
+    NamespaceTerraformResourceRoleV1,
+    NamespaceV1,
+)
+from reconcile.utils.exceptions import FetchResourceError
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+
+
+class ExternalResourceValidationError(Exception):
+    errors: list[str] = []
+
+    def add_validation_error(self, msg: str) -> None:
+        self.errors.append(msg)
+
+
+class ExternalResourceKey(BaseModel, frozen=True):
+    provision_provider: str
+    provisioner_name: str
+    provider: str
+    identifier: str
+
+    @staticmethod
+    def from_spec(spec: ExternalResourceSpec) -> "ExternalResourceKey":
+        return ExternalResourceKey(
+            provision_provider=spec.provision_provider,
+            provisioner_name=spec.provisioner_name,
+            identifier=spec.identifier,
+            provider=spec.provider,
+        )
+
+    def hash(self) -> str:
+        return hashlib.md5(
+            json.dumps(self.dict(), sort_keys=True).encode("utf-8")
+        ).hexdigest()
+
+    @property
+    def state_path(self) -> str:
+        return f"{self.provision_provider}/{self.provisioner_name}/{self.provider}/{self.identifier}"
+
+
+class ExternalResourcesInventory(MutableMapping):
+    _inventory: dict[ExternalResourceKey, ExternalResourceSpec] = {}
+
+    def __init__(self, namespaces: Iterable[NamespaceV1]) -> None:
+        desired_providers = [
+            (p, ns)
+            for ns in namespaces
+            for p in ns.external_resources or []
+            if isinstance(p, NamespaceTerraformProviderResourceAWSV1) and p.resources
+        ]
+
+        desired_specs = [
+            ExternalResourceSpec(
+                provision_provider=p.provider,
+                provisioner=p.provisioner.dict(),
+                resource=r.dict(),
+                namespace=ns.dict(),
+            )
+            for (p, ns) in desired_providers
+            for r in p.resources
+            if isinstance(
+                r, (NamespaceTerraformResourceRDSV1, NamespaceTerraformResourceRoleV1)
+            )
+            and r.managed_by_erv2
+        ]
+
+        for spec in desired_specs:
+            # self.set(ExternalResourceKey.from_spec(spec), spec)
+            self._inventory[ExternalResourceKey.from_spec(spec)] = spec
+
+    def __getitem__(self, key: ExternalResourceKey) -> ExternalResourceSpec | None:
+        return self._inventory[key]
+
+    def __setitem__(self, key: ExternalResourceKey, spec: ExternalResourceSpec) -> None:
+        self._inventory[key] = spec
+
+    def __delitem__(self, key: ExternalResourceKey) -> None:
+        del self._inventory[key]
+
+    def __iter__(self) -> Iterator[ExternalResourceKey]:
+        return iter(self._inventory)
+
+    def __len__(self) -> int:
+        return len(self._inventory)
+
+    def get_inventory_spec(
+        self, provision_provider: str, provisioner: str, provider: str, identifier: str
+    ) -> ExternalResourceSpec:
+        """Convinience method to find referenced specs in the inventory. For example, finding a sourcedb reference for an RDS instance."""
+        key = ExternalResourceKey(
+            provision_provider=provision_provider,
+            provisioner_name=provisioner,
+            provider=provider,
+            identifier=identifier,
+        )
+        try:
+            return self._inventory[key]
+        except KeyError:
+            msg = f"Resource spec not found: Provider {provider}, Id: {identifier}"
+            raise FetchResourceError(msg)
+
+
+class Action(str, Enum):
+    DESTROY: str = "Destroy"
+    APPLY: str = "Apply"
+
+
+class ExternalResourceModuleKey(BaseModel, frozen=True):
+    provision_provider: str
+    provider: str
+
+
+class ModuleInventory:
+    inventory: dict[ExternalResourceModuleKey, ExternalResourcesModuleV1]
+
+    def __init__(
+        self, inventory: dict[ExternalResourceModuleKey, ExternalResourcesModuleV1]
+    ):
+        self.inventory = inventory
+
+    def get_from_external_resource_key(
+        self, key: ExternalResourceKey
+    ) -> ExternalResourcesModuleV1:
+        return self.inventory[
+            ExternalResourceModuleKey(
+                provision_provider=key.provision_provider, provider=key.provider
+            )
+        ]
+
+    def get_from_spec(self, spec: ExternalResourceSpec) -> ExternalResourcesModuleV1:
+        return self.inventory[
+            ExternalResourceModuleKey(
+                provision_provider=spec.provision_provider, provider=spec.provider
+            )
+        ]
+
+
+def load_module_inventory(
+    modules: Iterable[ExternalResourcesModuleV1],
+) -> ModuleInventory:
+    return ModuleInventory({
+        ExternalResourceModuleKey(
+            provision_provider=m.provision_provider, provider=m.provider
+        ): m
+        for m in modules
+    })
+
+
+class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
+    image: str = ""
+    version: str = ""
+    reconcile_drift_interval_minutes: int = -1000
+    reconcile_timeout_minutes: int = -1000
+
+    @property
+    def image_version(self) -> str:
+        return f"{self.image}:{self.version}"
+
+    @staticmethod
+    def resolve_configuration(
+        module: ExternalResourcesModuleV1, spec: ExternalResourceSpec
+    ) -> "ExternalResourceModuleConfiguration":
+        # TODO: Modify resource schemas to include this attributes
+        data = {
+            "image": module.image,
+            "version": module.default_version,
+            "reconcile_drift_interval_minutes": module.reconcile_drift_interval_minutes,
+            "reconcile_timeout_minutes": module.reconcile_timeout_minutes,
+        }
+        return ExternalResourceModuleConfiguration.parse_obj(data)
+
+
+class Reconciliation(BaseModel, frozen=True):
+    key: ExternalResourceKey
+    resource_hash: str = ""
+    input: str = ""
+    action: Action = Action.APPLY
+    module_configuration: ExternalResourceModuleConfiguration = (
+        ExternalResourceModuleConfiguration()
+    )
+
+
+class ModuleProvisionData(ABC, BaseModel):
+    pass
+
+
+class TerraformModuleProvisionData(ModuleProvisionData):
+    """Specific Provision Options for modules based on Terraform or CDKTF"""
+
+    tf_state_bucket: str
+    tf_state_region: str
+    tf_state_dynamodb_table: str
+    tf_state_key: str
+
+
+class ExternalResourceProvision(BaseModel):
+    """External resource app-interface attributes. They are not part of the resource but are needed
+    for annotating secrets or other stuff
+    """
+
+    provision_provider: str
+    provisioner: str
+    provider: str
+    identifier: str
+    target_cluster: str
+    target_namespace: str
+    target_secret_name: str
+    module_provision_data: ModuleProvisionData
+
+
+class ExternalResource(BaseModel):
+    data: dict[str, Any]
+    provision: ExternalResourceProvision
+
+    def hash(self) -> str:
+        return hashlib.md5(
+            json.dumps(self.data, sort_keys=True).encode("utf-8")
+        ).hexdigest()
+
+    def serialize_input(self) -> str:
+        return base64.b64encode(json.dumps(self.dict()).encode()).decode()