qontract-reconcile 0.10.1rc762__py3-none-any.whl → 0.10.1rc764__py3-none-any.whl

{qontract_reconcile-0.10.1rc762.dist-info → qontract_reconcile-0.10.1rc764.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc762
+Version: 0.10.1rc764
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc762.dist-info → qontract_reconcile-0.10.1rc764.dist-info}/RECORD

@@ -176,6 +176,17 @@ reconcile/cna/assets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 reconcile/cna/assets/asset.py,sha256=1v51uYSaD1NOc9cI_YxG7h0NOcR1ng-mkmD2UzQ8PXE,866
 reconcile/cna/assets/asset_factory.py,sha256=7T7X_J6xIsoGETqBRI45_EyIKEdQcnRPt_GAuVuLQcc,785
 reconcile/cna/assets/null.py,sha256=Fby1Fbn7oNRIGNasdyhRDvXJ0ktpxv-pUAPN0lZWSzk,1684
+reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG-zlHOM,2956
+reconcile/external_resources/factories.py,sha256=bLboXX5Dq0xN60mtDGNjCOLC6HlKofXMWQxVbRwMMwo,4485
+reconcile/external_resources/integration.py,sha256=QQZdKTeHoUD7afUTNKh878u-KWovYptAFZwjUBTIbCg,3317
+reconcile/external_resources/manager.py,sha256=APszaw9PRIiHnFyCffHZjIFseIwhlYROIPLt18pHUTQ,13685
+reconcile/external_resources/meta.py,sha256=SA4Km1r7ePdcNqHn2GA4ByQp4ZnIeo_n8qOOd-11IEg,151
+reconcile/external_resources/metrics.py,sha256=m2TIOao2N7pD6k45driFbBGVCC_N7ai44m-lLPfa5qk,454
+reconcile/external_resources/model.py,sha256=FJUb7rHU2l7YSAv-t4QaacL9pqheFBxhPydWSPqu3vY,7413
+reconcile/external_resources/reconciler.py,sha256=E50X_lnOD0OWYXMzyZld1P6dCFJFYjHGyICWff9bxlc,9323
+reconcile/external_resources/secrets_sync.py,sha256=g-ksvzmTlCTwo3PM3FgYXm0LUBcnwfAxcvisuR1jAMY,7982
+reconcile/external_resources/state.py,sha256=rRePQdA3A6U9oFDDs9aaL5Ja7nSIXpOC1wHwY8R47_8,9197
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=Y7ofQg_xCt3dOln3pjeXp7rAnwohCgD2zcUAb-Hciis,8375
 reconcile/glitchtip/reconciler.py,sha256=nUvDv7qG1ly0cA16MmlL6NV71yl1mJYLT2mui7lmi0Y,12402
@@ -194,7 +205,7 @@ reconcile/gql_definitions/advanced_upgrade_service/aus_organization.py,sha256=uF
 reconcile/gql_definitions/app_interface_metrics_exporter/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/app_interface_metrics_exporter/onboarding_status.py,sha256=uVEEqU6YYmKsNTo6EWlFnoVmqha2rvBDx-wiD64VmG0,1679
 reconcile/gql_definitions/aws_account_manager/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/aws_account_manager/aws_accounts.py,sha256=c3RmQwbHa9UlfGwruufkA8PUfeiRJZ-lXEDInAREZqE,4405
+reconcile/gql_definitions/aws_account_manager/aws_accounts.py,sha256=jJfIzTtDiW6rasv3PFEbyVHA0d8bfRYSVYP5HxslYnY,4649
 reconcile/gql_definitions/aws_ami_cleanup/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/aws_ami_cleanup/asg_namespaces.py,sha256=OJmeTu7uirLGAysZ3IQTtRXqMyL8noi_QZxPuWYxxmI,3678
 reconcile/gql_definitions/aws_saml_idp/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -786,8 +797,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=w2l4BHB09k1d-BGJ1jBUNCqDv7zkqYrMHojQXg-21kQ,4155
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc762.dist-info/METADATA,sha256=f9qo3Mn92aMIw-oLh-t9rMXsdX6CkqcfJOP14BamQ3M,2382
-qontract_reconcile-0.10.1rc762.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc762.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
-qontract_reconcile-0.10.1rc762.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc762.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc764.dist-info/METADATA,sha256=Eov5cvApy_dlSvDlyNdI1Uk8pAO7tJZfv4cOMjUfeEA,2382
+qontract_reconcile-0.10.1rc764.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc764.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
+qontract_reconcile-0.10.1rc764.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc764.dist-info/RECORD,,

reconcile/external_resources/aws.py

@@ -0,0 +1,85 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+from reconcile.external_resources.model import (
+    ExternalResource,
+    ExternalResourcesInventory,
+)
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+from reconcile.utils.external_resources import ResourceValueResolver
+from reconcile.utils.secret_reader import SecretReaderBase
+
+
+class AWSResourceFactory(ABC):
+    def __init__(
+        self, er_inventory: ExternalResourcesInventory, secrets_reader: SecretReaderBase
+    ):
+        self.er_inventory = er_inventory
+        self.secrets_reader = secrets_reader
+
+    @abstractmethod
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]: ...
+
+    @abstractmethod
+    def validate(self, resource: ExternalResource) -> None: ...
+
+
+class AWSDefaultResourceFactory(AWSResourceFactory):
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        return ResourceValueResolver(spec=spec, identifier_as_value=True).resolve()
+
+    def validate(self, resource: ExternalResource) -> None: ...
+
+
+class AWSRdsFactory(AWSDefaultResourceFactory):
+    def _get_source_db_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "rds", identifier
+        )
+
+    def _get_kms_key_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "kms", identifier
+        )
+
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
+        data = rvr.resolve()
+
+        data["output_prefix"] = spec.output_prefix
+
+        if "parameter_group" in data:
+            pg_data = rvr._get_values(data["parameter_group"])
+            data["parameter_group"] = pg_data
+        if "old_parameter_group" in data:
+            old_pg_data = rvr._get_values(data["old_parameter_group"])
+            data["old_parameter_group"] = old_pg_data
+        if "replica_source" in data:
+            sourcedb_spec = self._get_source_db_spec(
+                spec.provisioner_name, data["replica_source"]
+            )
+            sourcedb = self.resolve(sourcedb_spec)
+            sourcedb_region = (
+                sourcedb.get("region", None)
+                or sourcedb_spec.provisioner["resources_default_region"]
+            )
+            data["replica_source"] = {
+                "identifier": sourcedb["identifier"],
+                "region": sourcedb_region,
+            }
+
+        kms_key_id: str = data.get("kms_key_id", None)
+        if kms_key_id and not kms_key_id.startswith("arn:"):
+            data["kms_key_id"] = self._get_kms_key_spec(
+                spec.provisioner_name, kms_key_id
+            ).identifier
+
+        return data
+
+    def validate(self, resource: ExternalResource) -> None: ...

reconcile/external_resources/factories.py

@@ -0,0 +1,133 @@
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from typing import Generic, TypeVar
+
+from reconcile.external_resources.aws import (
+    AWSDefaultResourceFactory,
+    AWSRdsFactory,
+    AWSResourceFactory,
+)
+from reconcile.external_resources.model import (
+    ExternalResource,
+    ExternalResourceKey,
+    ExternalResourceProvision,
+    ExternalResourcesInventory,
+    ModuleInventory,
+    ModuleProvisionData,
+    TerraformModuleProvisionData,
+)
+from reconcile.gql_definitions.external_resources.external_resources_settings import (
+    ExternalResourcesSettingsV1,
+)
+from reconcile.utils.external_resource_spec import (
+    ExternalResourceSpec,
+)
+from reconcile.utils.secret_reader import SecretReaderBase
+
+T = TypeVar("T")
+
+
+class ObjectFactory(Generic[T]):
+    def __init__(self) -> None:
+        self._factories: dict[str, T] = {}
+
+    def register_factory(self, id: str, t: T) -> None:
+        self._factories[id] = t
+
+    def get_factory(self, id: str) -> T:
+        return self._factories[id]
+
+
+class ExternalResourceFactory(ABC):
+    @abstractmethod
+    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+        pass
+
+    @abstractmethod
+    def validate_external_resource(self, resource: ExternalResource) -> None:
+        pass
+
+
+class ModuleProvisionDataFactory(ABC):
+    @abstractmethod
+    def create_provision_data(self, ers: ExternalResourceSpec) -> ModuleProvisionData:
+        pass
+
+
+class TerraformModuleProvisionDataFactory(ModuleProvisionDataFactory):
+    def __init__(self, settings: ExternalResourcesSettingsV1):
+        self.settings = settings
+
+    def create_provision_data(
+        self, spec: ExternalResourceSpec
+    ) -> TerraformModuleProvisionData:
+        key = ExternalResourceKey.from_spec(spec)
+
+        return TerraformModuleProvisionData(
+            tf_state_bucket=self.settings.tf_state_bucket,
+            tf_state_region=self.settings.tf_state_region,
+            tf_state_dynamodb_table=self.settings.tf_state_dynamodb_table,
+            tf_state_key=key.state_path + "/terraform.tfstate",
+        )
+
+
+def setup_aws_resource_factories(
+    er_inventory: ExternalResourcesInventory, secrets_reader: SecretReaderBase
+) -> ObjectFactory[AWSResourceFactory]:
+    f = ObjectFactory[AWSResourceFactory]()
+    f.register_factory("rds", AWSRdsFactory(er_inventory, secrets_reader))
+    f.register_factory(
+        "default", AWSDefaultResourceFactory(er_inventory, secrets_reader)
+    )
+    return f
+
+
+class AWSExternalResourceFactory(ExternalResourceFactory):
+    def __init__(
+        self,
+        module_inventory: ModuleInventory,
+        er_inventory: ExternalResourcesInventory,
+        secret_reader: SecretReaderBase,
+        provision_factories: ObjectFactory[ModuleProvisionDataFactory],
+        resource_factories: ObjectFactory[AWSResourceFactory],
+    ):
+        self.provision_factories = provision_factories
+        self.resource_factories = resource_factories
+        self.module_inventory = module_inventory
+        self.er_inventory = er_inventory
+        self.secret_reader = secret_reader
+
+    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+        f = self.resource_factories.get_factory(spec.provider)
+        data = f.resolve(spec)
+
+        region = data.get("region")
+        if region:
+            if region not in spec.provisioner["supported_deployment_regions"]:
+                raise ValueError(region)
+        else:
+            region = spec.provisioner["resources_default_region"]
+        data["region"] = region
+
+        module_type = self.module_inventory.get_from_spec(spec).module_type
+        provision_factory = self.provision_factories.get_factory(module_type)
+        module_provision_data = provision_factory.create_provision_data(spec)
+
+        provision = ExternalResourceProvision(
+            provision_provider=spec.provision_provider,
+            provisioner=spec.provisioner_name,
+            provider=spec.provider,
+            identifier=spec.identifier,
+            target_cluster=spec.cluster_name,
+            target_namespace=spec.namespace_name,
+            target_secret_name=spec.output_resource_name,
+            module_provision_data=module_provision_data,
+        )
+
+        return ExternalResource(data=data, provision=provision)
+
+    def validate_external_resource(self, resource: ExternalResource) -> None:
+        f = self.resource_factories.get_factory(resource.provision.provider)
+        f.validate(resource)

reconcile/external_resources/integration.py

@@ -0,0 +1,95 @@
+import logging
+
+from reconcile.external_resources.manager import (
+    ExternalResourcesInventory,
+    ExternalResourcesManager,
+    setup_factories,
+)
+from reconcile.external_resources.meta import (
+    QONTRACT_INTEGRATION,
+    QONTRACT_INTEGRATION_VERSION,
+)
+from reconcile.external_resources.model import load_module_inventory
+from reconcile.external_resources.reconciler import K8sExternalResourcesReconciler
+from reconcile.external_resources.secrets_sync import (
+    build_incluster_secrets_reconciler,
+)
+from reconcile.external_resources.state import ExternalResourcesStateDynamoDB
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.external_resources import (
+    get_modules,
+    get_namespaces,
+    get_settings,
+)
+from reconcile.utils.jobcontroller.controller import (
+    build_job_controller,
+)
+from reconcile.utils.oc import (
+    OCCli,
+)
+from reconcile.utils.openshift_resource import OpenshiftResource, ResourceInventory
+from reconcile.utils.secret_reader import create_secret_reader
+
+
+def fetch_current_state(
+    ri: ResourceInventory, oc: OCCli, cluster: str, namespace: str
+) -> None:
+    for item in oc.get_items("Job", namespace=namespace):
+        r = OpenshiftResource(item, QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION)
+        ri.add_current(cluster, namespace, "Job", r.name, r)
+
+
+def run(
+    dry_run: bool,
+    cluster: str,
+    namespace: str,
+    dry_run_job_suffix: str,
+    thread_pool_size: int,
+) -> None:
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    er_settings = get_settings()[0]
+    m_inventory = load_module_inventory(get_modules())
+    namespaces = [ns for ns in get_namespaces() if ns.external_resources]
+    er_inventory = ExternalResourcesInventory(namespaces)
+
+    er_mgr = ExternalResourcesManager(
+        thread_pool_size=thread_pool_size,
+        settings=er_settings,
+        secret_reader=secret_reader,
+        factories=setup_factories(
+            er_settings, m_inventory, er_inventory, secret_reader
+        ),
+        er_inventory=er_inventory,
+        module_inventory=m_inventory,
+        state_manager=ExternalResourcesStateDynamoDB(
+            table_name=er_settings.state_dynamodb_table,
+            region_name=er_settings.state_dynamodb_region,
+        ),
+        reconciler=K8sExternalResourcesReconciler(
+            controller=build_job_controller(
+                integration=QONTRACT_INTEGRATION,
+                integration_version=QONTRACT_INTEGRATION_VERSION,
+                cluster=cluster,
+                namespace=namespace,
+                secret_reader=secret_reader,
+                dry_run=dry_run,
+            ),
+            dry_run=dry_run,
+            dry_run_job_suffix=dry_run_job_suffix,
+        ),
+        secrets_reconciler=build_incluster_secrets_reconciler(
+            cluster, namespace, secret_reader, vault_path="app-sre"
+        ),
+    )
+
+    if dry_run:
+        er_mgr.handle_dry_run_resources()
+        if er_mgr.errors:
+            logging.error("Validation Errors:")
+            for k, e in er_mgr.errors.items():
+                logging.error("ExternalResourceKey: %s, Error: %s" % (k, e))
+    else:
+        er_mgr.handle_resources()