qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/external_resources/metrics.py

@@ -0,0 +1,95 @@
+from pydantic import BaseModel
+
+from reconcile.external_resources.meta import QONTRACT_INTEGRATION
+from reconcile.external_resources.model import Reconciliation, ReconciliationStatus
+from reconcile.external_resources.reconciler import ReconciliationK8sJob
+from reconcile.utils import metrics
+from reconcile.utils.external_resources import ExternalResourceSpec
+from reconcile.utils.metrics import (
+    CounterMetric,
+    GaugeMetric,
+    normalize_integration_name,
+)
+
+
+class ExternalResourcesBaseMetric(BaseModel):
+    integration = normalize_integration_name(QONTRACT_INTEGRATION)
+    app: str
+    environment: str
+    provision_provider: str
+    provisioner_name: str
+    provider: str
+    identifier: str
+    job_name: str
+
+
+class ExternalResourcesReconcileErrorsCounter(
+    ExternalResourcesBaseMetric, CounterMetric
+):
+    @classmethod
+    def name(cls) -> str:
+        return "external_resources_reconcile_errors"
+
+
+class ExternalResourcesReconcileTimeGauge(ExternalResourcesBaseMetric, GaugeMetric):
+    @classmethod
+    def name(cls) -> str:
+        return "external_resources_reconcile_time"
+
+
+class ExternalResourcesResourceStatus(ExternalResourcesBaseMetric, GaugeMetric):
+    status: str
+
+    @classmethod
+    def name(cls) -> str:
+        return "external_resources_resource_status"
+
+
+def publish_metrics(
+    r: Reconciliation,
+    spec: ExternalResourceSpec,
+    reconciliation_status: ReconciliationStatus,
+) -> None:
+    job_name = ReconciliationK8sJob(reconciliation=r).name()
+
+    # Use transactional_metrics to remove old status, we just want to expose the latest status
+    with metrics.transactional_metrics(scope=job_name) as metrics_container:
+        metrics_container.set_gauge(
+            ExternalResourcesResourceStatus(
+                app=spec.namespace["app"]["name"],
+                environment=spec.namespace["environment"]["name"],
+                provision_provider=r.key.provision_provider,
+                provisioner_name=r.key.provisioner_name,
+                provider=r.key.provider,
+                identifier=r.key.identifier,
+                job_name=job_name,
+                status=reconciliation_status.resource_status,
+            ),
+            1,
+        )
+
+    metrics.set_gauge(
+        ExternalResourcesReconcileTimeGauge(
+            app=spec.namespace["app"]["name"],
+            environment=spec.namespace["environment"]["name"],
+            provision_provider=r.key.provision_provider,
+            provisioner_name=r.key.provisioner_name,
+            provider=r.key.provider,
+            identifier=r.key.identifier,
+            job_name=job_name,
+        ),
+        reconciliation_status.reconcile_time,
+    )
+
+    if reconciliation_status.resource_status.has_errors:
+        metrics.inc_counter(
+            ExternalResourcesReconcileErrorsCounter(
+                app=spec.namespace["app"]["name"],
+                environment=spec.namespace["environment"]["name"],
+                provision_provider=r.key.provision_provider,
+                provisioner_name=r.key.provisioner_name,
+                provider=r.key.provider,
+                identifier=r.key.identifier,
+                job_name=job_name,
+            )
+        )

reconcile/external_resources/model.py

@@ -0,0 +1,350 @@
+import hashlib
+import json
+from abc import (
+    ABC,
+)
+from collections.abc import ItemsView, Iterable, Iterator, MutableMapping
+from enum import StrEnum
+from typing import Any
+
+from pydantic import BaseModel
+
+from reconcile.external_resources.meta import (
+    FLAG_DELETE_RESOURCE,
+    FLAG_RESOURCE_MANAGED_BY_ERV2,
+    MODULE_OVERRIDES,
+)
+from reconcile.gql_definitions.external_resources.external_resources_modules import (
+    ExternalResourcesModuleV1,
+)
+from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
+    ExternalResourcesModuleOverridesV1,
+    NamespaceTerraformProviderResourceAWSV1,
+    NamespaceTerraformResourceElastiCacheV1,
+    NamespaceTerraformResourceMskV1,
+    NamespaceTerraformResourceRDSV1,
+    NamespaceV1,
+)
+from reconcile.gql_definitions.external_resources.external_resources_settings import (
+    ExternalResourcesSettingsV1,
+)
+from reconcile.utils.exceptions import FetchResourceError
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+
+
+class ExternalResourceOrphanedResourcesError(Exception):
+    def __init__(self, orphans: Iterable["ExternalResourceKey"]) -> None:
+        msg = [
+            "There are orphaned resources in the configuration. ",
+            "To delete ERv2 managed external resources, set the 'delete: true' attribute.\n",
+            "Orphans:\n",
+            "\n".join(map(str, orphans)),
+        ]
+        super().__init__("".join(msg))
+
+
+class ExternalResourceValidationError(Exception):
+    errors: list[str] = []
+
+    def add_validation_error(self, msg: str) -> None:
+        self.errors.append(msg)
+
+
+class ExternalResourceKey(BaseModel, frozen=True):
+    provision_provider: str
+    provisioner_name: str
+    provider: str
+    identifier: str
+
+    @staticmethod
+    def from_spec(spec: ExternalResourceSpec) -> "ExternalResourceKey":
+        return ExternalResourceKey(
+            provision_provider=spec.provision_provider,
+            provisioner_name=spec.provisioner_name,
+            identifier=spec.identifier,
+            provider=spec.provider,
+        )
+
+    def hash(self) -> str:
+        return hashlib.md5(
+            json.dumps(self.dict(), sort_keys=True).encode("utf-8")
+        ).hexdigest()
+
+    @property
+    def state_path(self) -> str:
+        return f"{self.provision_provider}/{self.provisioner_name}/{self.provider}/{self.identifier}"
+
+
+SUPPORTED_RESOURCE_PROVIDERS = NamespaceTerraformProviderResourceAWSV1
+SUPPORTED_RESOURCE_TYPES = (
+    NamespaceTerraformResourceRDSV1
+    | NamespaceTerraformResourceMskV1
+    | NamespaceTerraformResourceElastiCacheV1
+)
+
+
+class ExternalResourcesInventory(MutableMapping):
+    def __init__(self, namespaces: Iterable[NamespaceV1]) -> None:
+        self._inventory: dict[ExternalResourceKey, ExternalResourceSpec] = {}
+
+        resource_providers = [
+            (rp, ns)
+            for ns in namespaces
+            for rp in ns.external_resources or []
+            if isinstance(rp, SUPPORTED_RESOURCE_PROVIDERS) and rp.resources
+        ]
+
+        desired_specs = [
+            self._build_external_resource_spec(ns, rp, r)
+            for (rp, ns) in resource_providers
+            for r in rp.resources
+            if isinstance(r, SUPPORTED_RESOURCE_TYPES) and r.managed_by_erv2
+        ]
+
+        for spec in desired_specs:
+            self._inventory[ExternalResourceKey.from_spec(spec)] = spec
+
+    def _build_external_resource_spec(
+        self,
+        namespace: NamespaceV1,
+        provider: SUPPORTED_RESOURCE_PROVIDERS,
+        resource: SUPPORTED_RESOURCE_TYPES,
+    ) -> ExternalResourceSpec:
+        spec = ExternalResourceSpec(
+            provision_provider=provider.provider,
+            provisioner=provider.provisioner.dict(),
+            resource=resource.dict(
+                exclude={
+                    FLAG_RESOURCE_MANAGED_BY_ERV2,
+                    FLAG_DELETE_RESOURCE,
+                    MODULE_OVERRIDES,
+                }
+            ),
+            namespace=namespace.dict(),
+        )
+        spec.metadata[FLAG_DELETE_RESOURCE] = resource.delete
+        spec.metadata[MODULE_OVERRIDES] = resource.module_overrides
+        return spec
+
+    def __getitem__(self, key: ExternalResourceKey) -> ExternalResourceSpec | None:
+        return self._inventory[key]
+
+    def __setitem__(self, key: ExternalResourceKey, spec: ExternalResourceSpec) -> None:
+        self._inventory[key] = spec
+
+    def __delitem__(self, key: ExternalResourceKey) -> None:
+        del self._inventory[key]
+
+    def __iter__(self) -> Iterator[ExternalResourceKey]:
+        return iter(self._inventory)
+
+    def __len__(self) -> int:
+        return len(self._inventory)
+
+    def items(self) -> ItemsView[ExternalResourceKey, ExternalResourceSpec]:
+        return self._inventory.items()
+
+    def get_inventory_spec(
+        self, provision_provider: str, provisioner: str, provider: str, identifier: str
+    ) -> ExternalResourceSpec:
+        """Convinience method to find referenced specs in the inventory. For example, finding a sourcedb reference for an RDS instance."""
+        key = ExternalResourceKey(
+            provision_provider=provision_provider,
+            provisioner_name=provisioner,
+            provider=provider,
+            identifier=identifier,
+        )
+        try:
+            return self._inventory[key]
+        except KeyError:
+            msg = f"Resource spec not found: Provider {provider}, Id: {identifier}"
+            raise FetchResourceError(msg) from None
+
+
+class Action(StrEnum):
+    DESTROY: str = "Destroy"
+    APPLY: str = "Apply"
+
+
+class ExternalResourceModuleKey(BaseModel, frozen=True):
+    provision_provider: str
+    provider: str
+
+
+class ModuleInventory:
+    def __init__(
+        self, inventory: dict[ExternalResourceModuleKey, ExternalResourcesModuleV1]
+    ):
+        self.inventory = inventory
+
+    def get_from_external_resource_key(
+        self, key: ExternalResourceKey
+    ) -> ExternalResourcesModuleV1:
+        return self.inventory[
+            ExternalResourceModuleKey(
+                provision_provider=key.provision_provider, provider=key.provider
+            )
+        ]
+
+    def get_from_spec(self, spec: ExternalResourceSpec) -> ExternalResourcesModuleV1:
+        return self.inventory[
+            ExternalResourceModuleKey(
+                provision_provider=spec.provision_provider, provider=spec.provider
+            )
+        ]
+
+
+def load_module_inventory(
+    modules: Iterable[ExternalResourcesModuleV1],
+) -> ModuleInventory:
+    return ModuleInventory({
+        ExternalResourceModuleKey(
+            provision_provider=m.provision_provider, provider=m.provider
+        ): m
+        for m in modules
+    })
+
+
+class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
+    image: str = ""
+    version: str = ""
+    reconcile_drift_interval_minutes: int = -1000
+    reconcile_timeout_minutes: int = -1000
+    outputs_secret_image: str = ""
+    outputs_secret_version: str = ""
+
+    @property
+    def image_version(self) -> str:
+        return f"{self.image}:{self.version}"
+
+    @property
+    def outputs_secret_image_version(self) -> str:
+        return f"{self.outputs_secret_image}:{self.outputs_secret_version}"
+
+    @staticmethod
+    def resolve_configuration(
+        module: ExternalResourcesModuleV1,
+        spec: ExternalResourceSpec,
+        settings: ExternalResourcesSettingsV1,
+    ) -> "ExternalResourceModuleConfiguration":
+        module_overrides = spec.metadata.get(
+            "module_overrides"
+        ) or ExternalResourcesModuleOverridesV1(
+            module_type=None,
+            image=None,
+            version=None,
+            reconcile_timeout_minutes=None,
+            outputs_secret_image=None,
+            outputs_secret_version=None,
+        )
+
+        return ExternalResourceModuleConfiguration(
+            image=module_overrides.image or module.image,
+            version=module_overrides.version or module.version,
+            reconcile_drift_interval_minutes=module.reconcile_drift_interval_minutes,
+            reconcile_timeout_minutes=module_overrides.reconcile_timeout_minutes
+            or module.reconcile_timeout_minutes,
+            outputs_secret_image=module_overrides.outputs_secret_image
+            or module.outputs_secret_image
+            or settings.outputs_secret_image,
+            outputs_secret_version=module_overrides.outputs_secret_version
+            or module.outputs_secret_version
+            or settings.outputs_secret_version,
+        )
+
+
+class ResourceStatus(StrEnum):
+    CREATED: str = "CREATED"
+    DELETED: str = "DELETED"
+    ABANDONED: str = "ABANDONED"
+    NOT_EXISTS: str = "NOT_EXISTS"
+    IN_PROGRESS: str = "IN_PROGRESS"
+    DELETE_IN_PROGRESS: str = "DELETE_IN_PROGRESS"
+    ERROR: str = "ERROR"
+    PENDING_SECRET_SYNC: str = "PENDING_SECRET_SYNC"
+    RECONCILIATION_REQUESTED: str = "RECONCILIATION_REQUESTED"
+
+    @property
+    def does_not_exist(self) -> bool:
+        return self == ResourceStatus.NOT_EXISTS
+
+    @property
+    def is_in_progress(self) -> bool:
+        return self in {ResourceStatus.IN_PROGRESS, ResourceStatus.DELETE_IN_PROGRESS}
+
+    @property
+    def needs_secret_sync(self) -> bool:
+        return self == ResourceStatus.PENDING_SECRET_SYNC
+
+    @property
+    def has_errors(self) -> bool:
+        return self == ResourceStatus.ERROR
+
+
+class Reconciliation(BaseModel, frozen=True):
+    key: ExternalResourceKey
+    resource_hash: str = ""
+    input: str = ""
+    action: Action = Action.APPLY
+    module_configuration: ExternalResourceModuleConfiguration = (
+        ExternalResourceModuleConfiguration()
+    )
+    # linked_resources store dependants resources. They will get reconciled
+    # every time the parent resource reconciliation finishes.
+    linked_resources: frozenset[ExternalResourceKey] | None
+
+
+class ReconcileAction(StrEnum):
+    NOOP = "NOOP"
+    APPLY_NOT_EXISTS = "Resource does not exist"
+    APPLY_ERROR = "Resource status in ERROR state"
+    APPLY_SPEC_CHANGED = "Resource spec has changed"
+    APPLY_DRIFT_DETECTION = "Resource drift detection run"
+    APPLY_USER_REQUESTED = "Resource reconciliation requested"
+    DESTROY_CREATED = "Resource no longer exists in the configuration"
+    DESTROY_ERROR = "Resource status in ERROR state"
+
+
+class ReconciliationStatus(BaseModel):
+    reconcile_time: int = 0
+    resource_status: ResourceStatus
+
+
+class ModuleProvisionData(ABC, BaseModel):
+    pass
+
+
+class TerraformModuleProvisionData(ModuleProvisionData):
+    """Specific Provision Options for modules based on Terraform or CDKTF"""
+
+    tf_state_bucket: str
+    tf_state_region: str
+    tf_state_dynamodb_table: str
+    tf_state_key: str
+
+
+class ExternalResourceProvision(BaseModel):
+    """External resource app-interface attributes. They are not part of the resource but are needed
+    for annotating secrets or other stuff
+    """
+
+    provision_provider: str
+    provisioner: str
+    provider: str
+    identifier: str
+    target_cluster: str
+    target_namespace: str
+    target_secret_name: str
+    module_provision_data: ModuleProvisionData
+
+
+class ExternalResource(BaseModel):
+    data: dict[str, Any]
+    provision: ExternalResourceProvision
+
+    def hash(self) -> str:
+        return hashlib.md5(
+            json.dumps(self.data, sort_keys=True).encode("utf-8")
+        ).hexdigest()

reconcile/external_resources/reconciler.py

@@ -0,0 +1,265 @@
+import sys
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+from typing import Any
+
+from kubernetes.client import (
+    V1Container,
+    V1EmptyDirVolumeSource,
+    V1EnvVar,
+    V1EnvVarSource,
+    V1JobSpec,
+    V1LocalObjectReference,
+    V1ObjectFieldSelector,
+    V1ObjectMeta,
+    V1PodSpec,
+    V1PodTemplateSpec,
+    V1SecretVolumeSource,
+    V1Volume,
+    V1VolumeMount,
+)
+from pydantic import BaseModel
+
+from reconcile.external_resources.model import (
+    Action,
+    Reconciliation,
+)
+from reconcile.external_resources.state import ReconcileStatus
+from reconcile.utils.jobcontroller.controller import (
+    JobConcurrencyPolicy,
+    K8sJobController,
+)
+from reconcile.utils.jobcontroller.models import K8sJob
+
+
+class ExternalResourcesReconciler(ABC):
+    @abstractmethod
+    def get_resource_reconcile_status(
+        self,
+        reconciliation: Reconciliation,
+    ) -> ReconcileStatus: ...
+
+    @abstractmethod
+    def get_resource_reconcile_duration(
+        self, reconciliation: Reconciliation
+    ) -> int | None: ...
+
+    @abstractmethod
+    def reconcile_resource(self, reconciliation: Reconciliation) -> None: ...
+
+    @abstractmethod
+    def get_resource_reconcile_logs(self, reconciliation: Reconciliation) -> None: ...
+
+    @abstractmethod
+    def wait_for_reconcile_list_completion(
+        self,
+        reconcile_list: Iterable[Reconciliation],
+        check_interval_seconds: int,
+        timeout_seconds: int,
+    ) -> dict[str, ReconcileStatus]: ...
+
+
+class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
+    """
+    Wraps a reconciliation request into a Kubernetes Job
+    """
+
+    reconciliation: Reconciliation
+    is_dry_run: bool = False
+    dry_run_suffix: str = ""
+
+    def name_prefix(self) -> str:
+        if self.is_dry_run:
+            return f"er-dry-run-mr-{self.dry_run_suffix}"
+        else:
+            return "er"
+
+    def unit_of_work_identity(self) -> Any:
+        return self.reconciliation.key
+
+    def description(self) -> str:
+        return f"Action: {self.reconciliation.action}, Key: {self.reconciliation.key} "
+
+    def annotations(self) -> dict[str, Any]:
+        return {
+            "provision_provider": self.reconciliation.key.provision_provider,
+            "provisioner": self.reconciliation.key.provisioner_name,
+            "provider": self.reconciliation.key.provision_provider,
+            "identifier": self.reconciliation.key.identifier,
+        }
+
+    def job_spec(self) -> V1JobSpec:
+        job_container = V1Container(
+            name="job",
+            image=self.reconciliation.module_configuration.image_version,
+            image_pull_policy="Always",
+            env=[
+                V1EnvVar(
+                    name="DRY_RUN",
+                    value=str(self.is_dry_run),
+                ),
+                V1EnvVar(
+                    name="ACTION",
+                    value=self.reconciliation.action.value,
+                ),
+            ],
+            volume_mounts=[
+                V1VolumeMount(
+                    name="credentials",
+                    mount_path="/credentials",
+                    sub_path="credentials",
+                ),
+                V1VolumeMount(
+                    name="workdir",
+                    mount_path="/work",
+                ),
+                self.scripts_volume_mount("/inputs"),
+            ],
+        )
+        outputs_secret_container = V1Container(
+            name="outputs",
+            image=self.reconciliation.module_configuration.outputs_secret_image_version,
+            image_pull_policy="Always",
+            env=[
+                V1EnvVar(
+                    name="NAMESPACE",
+                    value_from=V1EnvVarSource(
+                        field_ref=V1ObjectFieldSelector(field_path="metadata.namespace")
+                    ),
+                ),
+                V1EnvVar(
+                    name="ACTION",
+                    value=self.reconciliation.action,
+                ),
+                V1EnvVar(
+                    name="DRY_RUN",
+                    value=str(self.is_dry_run),
+                ),
+            ],
+            volume_mounts=[
+                V1VolumeMount(
+                    name="credentials",
+                    mount_path="/.aws/credentials",
+                    sub_path="credentials",
+                ),
+                V1VolumeMount(
+                    name="workdir",
+                    mount_path="/work",
+                ),
+                self.scripts_volume_mount("/inputs"),
+            ],
+        )
+
+        # For delete actions, we don't need to run the outputs_secrets container
+        if self.reconciliation.action == Action.APPLY:
+            init_containers = [job_container]
+            containers = [outputs_secret_container]
+        else:
+            init_containers = []
+            containers = [job_container]
+
+        return V1JobSpec(
+            backoff_limit=0,
+            active_deadline_seconds=self.reconciliation.module_configuration.reconcile_timeout_minutes
+            * 60,
+            ttl_seconds_after_finished=3600,
+            template=V1PodTemplateSpec(
+                metadata=V1ObjectMeta(
+                    annotations=self.annotations(), labels=self.labels()
+                ),
+                spec=V1PodSpec(
+                    init_containers=init_containers,
+                    containers=containers,
+                    image_pull_secrets=[V1LocalObjectReference(name="quay.io")],
+                    volumes=[
+                        V1Volume(
+                            name="credentials",
+                            secret=V1SecretVolumeSource(
+                                secret_name=f"credentials-{self.reconciliation.key.provisioner_name}",
+                            ),
+                        ),
+                        V1Volume(
+                            name="workdir",
+                            empty_dir=V1EmptyDirVolumeSource(size_limit="10Mi"),
+                        ),
+                        self.scripts_volume(),
+                    ],
+                    restart_policy="Never",
+                    service_account_name="external-resources-sa",
+                ),
+            ),
+        )
+
+    def scripts(self) -> dict[str, str]:
+        return {"input.json": self.reconciliation.input}
+
+
+class K8sExternalResourcesReconciler(ExternalResourcesReconciler):
+    def __init__(
+        self, controller: K8sJobController, dry_run: bool, dry_run_job_suffix: str = ""
+    ) -> None:
+        self.controller = controller
+        self.dry_run = dry_run
+        self.dry_run_job_suffix = dry_run_job_suffix
+
+    def get_resource_reconcile_status(
+        self,
+        reconciliation: Reconciliation,
+    ) -> ReconcileStatus:
+        job_name = ReconciliationK8sJob(reconciliation=reconciliation).name()
+        return ReconcileStatus(self.controller.get_job_status(job_name))
+
+    def get_resource_reconcile_duration(
+        self, reconciliation: Reconciliation
+    ) -> int | None:
+        job_name = ReconciliationK8sJob(reconciliation=reconciliation).name()
+        return self.controller.get_success_job_duration(job_name)
+
+    def reconcile_resource(self, reconciliation: Reconciliation) -> None:
+        concurrency_policy = (
+            JobConcurrencyPolicy.REPLACE_FAILED | JobConcurrencyPolicy.REPLACE_FINISHED
+        )
+        if self.dry_run:
+            concurrency_policy = (
+                JobConcurrencyPolicy.REPLACE_FAILED
+                | JobConcurrencyPolicy.REPLACE_FINISHED
+                | JobConcurrencyPolicy.REPLACE_IN_PROGRESS
+            )
+
+        self.controller.enqueue_job(
+            ReconciliationK8sJob(
+                reconciliation=reconciliation,
+                is_dry_run=self.dry_run,
+                dry_run_suffix=self.dry_run_job_suffix,
+            ),
+            concurrency_policy=concurrency_policy,
+        )
+
+    def wait_for_reconcile_list_completion(
+        self,
+        reconcile_list: Iterable[Reconciliation],
+        check_interval_seconds: int,
+        timeout_seconds: int,
+    ) -> dict[str, ReconcileStatus]:
+        job_names = {
+            ReconciliationK8sJob(
+                reconciliation=r,
+                is_dry_run=self.dry_run,
+                dry_run_suffix=self.dry_run_job_suffix,
+            ).name()
+            for r in reconcile_list
+        }
+        job_status = self.controller.wait_for_job_list_completion(
+            job_names=job_names,
+            check_interval_seconds=check_interval_seconds,
+            timeout_seconds=timeout_seconds,
+        )
+        return {job: ReconcileStatus(status) for job, status in job_status.items()}
+
+    def get_resource_reconcile_logs(self, reconciliation: Reconciliation) -> None:
+        job = ReconciliationK8sJob(
+            reconciliation=reconciliation,
+            is_dry_run=True,
+            dry_run_suffix=self.dry_run_job_suffix,
+        )
+        self.controller.get_job_logs(job_name=job.name(), output=sys.stdout)