qontract-reconcile 0.10.1rc1152__py3-none-any.whl → 0.10.1rc1154__py3-none-any.whl

{qontract_reconcile-0.10.1rc1152.dist-info → qontract_reconcile-0.10.1rc1154.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1152
+Version: 0.10.1rc1154
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1152.dist-info → qontract_reconcile-0.10.1rc1154.dist-info}/RECORD

@@ -192,14 +192,14 @@ reconcile/endpoints_discovery/integration.py,sha256=znfnlm8bZesfcNbQnaR2aaVM-DTB
 reconcile/endpoints_discovery/merge_request.py,sha256=_yLb4tnvoZMCko8rta2C_CvOInJa9pa3HzSmHNtjgGU,2978
 reconcile/endpoints_discovery/merge_request_manager.py,sha256=wUMsumxv8RnWaRattax4HfoRlhtVzmgro3GiJJ1C4Vc,6392
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/external_resources/aws.py,sha256=7W-6d-lXO6JGwaxtO1Uc3Lw0p8csJ1EVgz__O1YWUOc,4793
-reconcile/external_resources/factories.py,sha256=nhdTqf1WEfRfgd5-70KAUJVz0ZvZ19C3Pz7wmotSdrs,4857
+reconcile/external_resources/aws.py,sha256=309Zui7rE8XFJA1ZBLupl55Vp8Y5KKgXdsKQWqKbg8I,7069
+reconcile/external_resources/factories.py,sha256=KrJDh52_8PeCEVjwfeVr1jwAJDdhMXRQ_XcBETfnKY4,4988
 reconcile/external_resources/integration.py,sha256=gBVO5dE8JyZ3xYcYik-MTIp_18oU7_hpYc_oztyfElQ,6753
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
 reconcile/external_resources/manager.py,sha256=wcqTawNS4qoBHFVfyCfHtWXh4L3AlgcNYx_Ov_vEjNg,17914
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=nMbyonGZEJDD1lYzpQY2eR9TNwvxYC4ZCcpi6wrExcM,1037
-reconcile/external_resources/model.py,sha256=UuQgrnv-SSkvSEQQGeCE2IZkhXjLTCVkP_mw8zBZsIQ,8349
+reconcile/external_resources/model.py,sha256=Ta8eOJ97RxCgBtWqsF4MWzQhgsOC2PoFfBKM5y3rM6U,8378
 reconcile/external_resources/reconciler.py,sha256=3KFmkHsN7YAwJUSBpN1Xd_D2zM9Ea5_c2uMGWsfruZo,9707
 reconcile/external_resources/secrets_sync.py,sha256=6n0oDPLjd9Ql0lf6zsr1AZw8A6EEe3yCzl20XodtgkE,16229
 reconcile/external_resources/state.py,sha256=UupSa6tl4-73_J6Fhisn-qHal3v3uAUS5s5sk85LGDs,9343
@@ -292,7 +292,7 @@ reconcile/gql_definitions/endpoints_discovery/namespaces.py,sha256=FqJ0H7NdsIm5B
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
 reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=HFOQjmNbNxk0j5nChxppQeCnJjeDsqibJkPgA7R1zRw,2417
-reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=-ehru0zFyTGOmhh3YB1nHVBd480Iq3rlOnPM3PEA20A,42585
+reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=KSASnKycr-e5kCUSOkqnS8w7Gvo_nXA_5NQcS8vGeLY,43466
 reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=Hw9n_90BPG6Lnt2PT3mHc6p0KEm2CxKxvSGRFc_Dhus,2982
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
@@ -683,7 +683,7 @@ reconcile/utils/gpg.py,sha256=EKG7_fdMv8BMlV5yUdPiqoTx-KrzmVSEAl2sLkaKwWI,1123
 reconcile/utils/gql.py,sha256=C0thIm_k9MBldfqwHzyqtYZk9sIvMdm9IbbnXLGwjD8,14158
 reconcile/utils/grouping.py,sha256=vr9SFHZ7bqmHYrvYcEZt-Er3-yQYfAAdq5sHLZVmXPY,456
 reconcile/utils/helm.py,sha256=hr4J_9mBZwbc1FDNfFh4QKAj0h3eLxyTN2Y3UxIRp8U,3893
-reconcile/utils/helpers.py,sha256=k9svgFFZG7H5FvHYY0g5jJyvgvh2UDZxf0Ib221teag,1179
+reconcile/utils/helpers.py,sha256=1PK_6gTzg2kJ8Ac8zZHB2yEubewDWtVspTl5bVbvbEY,1462
 reconcile/utils/imap_client.py,sha256=h8YDiCSCvroErhpH_-KGYI7Y2WU2Q2oSpuxDFbOkSbY,1989
 reconcile/utils/instrumented_wrappers.py,sha256=eVwMoa6FCrYxLv3RML3WpZF9qKVfCTjMxphgVXG03OM,1073
 reconcile/utils/jenkins_api.py,sha256=RaKuZmO7_lbI-hE6c_Pq2a6CQdmBVj7BcP2jR68cIbI,7081
@@ -880,8 +880,8 @@ tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6d
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1152.dist-info/METADATA,sha256=DfunuqAeeqnFMYWAd1YufogTDGsI_ff0inMYI-ZU5dY,2213
-qontract_reconcile-0.10.1rc1152.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1152.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1152.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1152.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1154.dist-info/METADATA,sha256=S0xB9gH65bXGiy_EaEsoUw8L6KbifPatRapCwmnBAS0,2213
+qontract_reconcile-0.10.1rc1154.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1154.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1154.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1154.dist-info/RECORD,,

reconcile/external_resources/aws.py

@@ -9,6 +9,7 @@ from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
 )
 from reconcile.utils.external_resources import ResourceValueResolver
+from reconcile.utils.helpers import generate_random_password
 from reconcile.utils.secret_reader import SecretReaderBase
 
 
@@ -33,6 +34,57 @@ class AWSDefaultResourceFactory(AWSResourceFactory):
     def validate(self, resource: ExternalResource) -> None: ...
 
 
+class AWSElasticacheFactory(AWSDefaultResourceFactory):
+    def _get_source_db_spec(
+        self, provisioner: str, identifier: str
+    ) -> ExternalResourceSpec:
+        return self.er_inventory.get_inventory_spec(
+            "aws", provisioner, "elasticache", identifier
+        )
+
+    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+        """Resolve the elasticache resource specification and translate some attributes to AWS >= 5.60.0 provider format."""
+        rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
+        data = rvr.resolve()
+        data["output_prefix"] = spec.output_prefix
+
+        if "replication_group_id" not in data:
+            data["replication_group_id"] = spec.identifier
+
+        if cluster_mode := data.pop("cluster_mode", {}):
+            for k, v in cluster_mode.items():
+                data[k] = v
+
+        if "parameter_group" in data:
+            pg_data = rvr._get_values(data["parameter_group"])
+            data["parameter_group"] = pg_data
+
+        if data.get("transit_encryption_enabled", False):
+            data["auth_token"] = (
+                spec.get_secret_field("db.auth_token") or generate_random_password()
+            )
+        return data
+
+    def validate(self, resource: ExternalResource) -> None:
+        """Validate the elasticache resource specification."""
+        data = resource.data
+        if data.get("parameter_group"):
+            if not data["parameter_group"].get("name"):
+                data["parameter_group"]["name"] = f"{data['replication_group_id']}-pg"
+
+            if (
+                data.get("parameter_group_name")
+                and data["parameter_group"]["name"] != data["parameter_group_name"]
+            ):
+                raise ValueError(
+                    "Custom parameter_group set and parameter_group_name given. Either remove parameter_group_name or set it to the same value as parameter_group.name."
+                )
+
+            if not data.get("parameter_group_name"):
+                # automatically set /aws/elasticache-defaults-1.yml/parameter_group_name to /aws/parameter-group-1.yml/name
+                data["parameter_group_name"] = data["parameter_group"]["name"]
+
+
 class AWSRdsFactory(AWSDefaultResourceFactory):
     def _get_source_db_spec(
         self, provisioner: str, identifier: str

reconcile/external_resources/factories.py

@@ -6,6 +6,7 @@ from typing import Generic, TypeVar
 
 from reconcile.external_resources.aws import (
     AWSDefaultResourceFactory,
+    AWSElasticacheFactory,
     AWSMskFactory,
     AWSRdsFactory,
     AWSResourceFactory,
@@ -87,6 +88,9 @@ def setup_aws_resource_factories(
     er_inventory: ExternalResourcesInventory, secret_reader: SecretReaderBase
 ) -> ObjectFactory[AWSResourceFactory]:
     f = ObjectFactory[AWSResourceFactory]()
+    f.register_factory(
+        "elasticache", AWSElasticacheFactory(er_inventory, secret_reader)
+    )
     f.register_factory("rds", AWSRdsFactory(er_inventory, secret_reader))
     f.register_factory("msk", AWSMskFactory(er_inventory, secret_reader))
     f.register_factory(

reconcile/external_resources/model.py

@@ -24,6 +24,7 @@ from reconcile.gql_definitions.external_resources.external_resources_modules imp
 from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
     ExternalResourcesModuleOverridesV1,
     NamespaceTerraformProviderResourceAWSV1,
+    NamespaceTerraformResourceElastiCacheV1,
     NamespaceTerraformResourceMskV1,
     NamespaceTerraformResourceRDSV1,
     NamespaceV1,
@@ -68,12 +69,32 @@ class ExternalResourceKey(BaseModel, frozen=True):
 
 SUPPORTED_RESOURCE_PROVIDERS = NamespaceTerraformProviderResourceAWSV1
 SUPPORTED_RESOURCE_TYPES = (
-    NamespaceTerraformResourceRDSV1 | NamespaceTerraformResourceMskV1
+    NamespaceTerraformResourceRDSV1
+    | NamespaceTerraformResourceMskV1
+    | NamespaceTerraformResourceElastiCacheV1
 )
 
 
 class ExternalResourcesInventory(MutableMapping):
-    _inventory: dict[ExternalResourceKey, ExternalResourceSpec] = {}
+    def __init__(self, namespaces: Iterable[NamespaceV1]) -> None:
+        self._inventory: dict[ExternalResourceKey, ExternalResourceSpec] = {}
+
+        desired_providers = [
+            (p, ns)
+            for ns in namespaces
+            for p in ns.external_resources or []
+            if isinstance(p, SUPPORTED_RESOURCE_PROVIDERS) and p.resources
+        ]
+
+        desired_specs = [
+            self._build_external_resource_spec(ns, p, r)
+            for (p, ns) in desired_providers
+            for r in p.resources
+            if isinstance(r, SUPPORTED_RESOURCE_TYPES) and r.managed_by_erv2
+        ]
+
+        for spec in desired_specs:
+            self._inventory[ExternalResourceKey.from_spec(spec)] = spec
 
     def _build_external_resource_spec(
         self,
@@ -97,24 +118,6 @@ class ExternalResourcesInventory(MutableMapping):
         spec.metadata[MODULE_OVERRIDES] = resource.module_overrides
         return spec
 
-    def __init__(self, namespaces: Iterable[NamespaceV1]) -> None:
-        desired_providers = [
-            (p, ns)
-            for ns in namespaces
-            for p in ns.external_resources or []
-            if isinstance(p, SUPPORTED_RESOURCE_PROVIDERS) and p.resources
-        ]
-
-        desired_specs = [
-            self._build_external_resource_spec(ns, p, r)
-            for (p, ns) in desired_providers
-            for r in p.resources
-            if isinstance(r, SUPPORTED_RESOURCE_TYPES) and r.managed_by_erv2
-        ]
-
-        for spec in desired_specs:
-            self._inventory[ExternalResourceKey.from_spec(spec)] = spec
-
     def __getitem__(self, key: ExternalResourceKey) -> ExternalResourceSpec | None:
         return self._inventory[key]
 
@@ -158,8 +161,6 @@ class ExternalResourceModuleKey(BaseModel, frozen=True):
 
 
 class ModuleInventory:
-    inventory: dict[ExternalResourceModuleKey, ExternalResourcesModuleV1]
-
     def __init__(
         self, inventory: dict[ExternalResourceModuleKey, ExternalResourcesModuleV1]
     ):

reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -144,6 +144,14 @@ query ExternalResourcesNamespaces {
                 overrides
                 output_resource_name
                 annotations
+                managed_by_erv2
+                delete
+                module_overrides {
+                  module_type
+                  image
+                  version
+                  reconcile_timeout_minutes
+                }
             }
             ... on NamespaceTerraformResourceServiceAccount_v1 {
                 identifier
@@ -602,6 +610,13 @@ class NamespaceTerraformResourceS3V1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
 
 
+class NamespaceTerraformResourceElastiCacheV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+
+
 class NamespaceTerraformResourceElastiCacheV1(NamespaceTerraformResourceAWSV1):
     identifier: str = Field(..., alias="identifier")
     defaults: str = Field(..., alias="defaults")
@@ -610,6 +625,9 @@ class NamespaceTerraformResourceElastiCacheV1(NamespaceTerraformResourceAWSV1):
     overrides: Optional[str] = Field(..., alias="overrides")
     output_resource_name: Optional[str] = Field(..., alias="output_resource_name")
     annotations: Optional[str] = Field(..., alias="annotations")
+    managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
+    delete: Optional[bool] = Field(..., alias="delete")
+    module_overrides: Optional[NamespaceTerraformResourceElastiCacheV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
 
 
 class ClusterV1(ConfiguredBaseModel):
@@ -1014,7 +1032,7 @@ class NamespaceTerraformResourceMskV1(NamespaceTerraformResourceAWSV1):
 
 class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):
     provisioner: AWSAccountV1 = Field(..., alias="provisioner")
-    resources: list[Union[NamespaceTerraformResourceRDSV1, NamespaceTerraformResourceRosaAuthenticatorV1, NamespaceTerraformResourceALBV1, NamespaceTerraformResourceS3V1, NamespaceTerraformResourceASGV1, NamespaceTerraformResourceMskV1, NamespaceTerraformResourceRoleV1, NamespaceTerraformResourceSNSTopicV1, NamespaceTerraformResourceElastiCacheV1, NamespaceTerraformResourceServiceAccountV1, NamespaceTerraformResourceS3SQSV1, NamespaceTerraformResourceCloudWatchV1, NamespaceTerraformResourceRosaAuthenticatorVPCEV1, NamespaceTerraformResourceS3CloudFrontV1, NamespaceTerraformResourceKMSV1, NamespaceTerraformResourceElasticSearchV1, NamespaceTerraformResourceACMV1, NamespaceTerraformResourceKinesisV1, NamespaceTerraformResourceRoute53ZoneV1, NamespaceTerraformResourceSQSV1, NamespaceTerraformResourceDynamoDBV1, NamespaceTerraformResourceECRV1, NamespaceTerraformResourceS3CloudFrontPublicKeyV1, NamespaceTerraformResourceSecretsManagerV1, NamespaceTerraformResourceSecretsManagerServiceAccountV1, NamespaceTerraformResourceAWSV1]] = Field(..., alias="resources")
+    resources: list[Union[NamespaceTerraformResourceRDSV1, NamespaceTerraformResourceRosaAuthenticatorV1, NamespaceTerraformResourceALBV1, NamespaceTerraformResourceS3V1, NamespaceTerraformResourceElastiCacheV1, NamespaceTerraformResourceASGV1, NamespaceTerraformResourceMskV1, NamespaceTerraformResourceRoleV1, NamespaceTerraformResourceSNSTopicV1, NamespaceTerraformResourceServiceAccountV1, NamespaceTerraformResourceS3SQSV1, NamespaceTerraformResourceCloudWatchV1, NamespaceTerraformResourceRosaAuthenticatorVPCEV1, NamespaceTerraformResourceS3CloudFrontV1, NamespaceTerraformResourceKMSV1, NamespaceTerraformResourceElasticSearchV1, NamespaceTerraformResourceACMV1, NamespaceTerraformResourceKinesisV1, NamespaceTerraformResourceRoute53ZoneV1, NamespaceTerraformResourceSQSV1, NamespaceTerraformResourceDynamoDBV1, NamespaceTerraformResourceECRV1, NamespaceTerraformResourceS3CloudFrontPublicKeyV1, NamespaceTerraformResourceSecretsManagerV1, NamespaceTerraformResourceSecretsManagerServiceAccountV1, NamespaceTerraformResourceAWSV1]] = Field(..., alias="resources")
 
 
 class EnvironmentV1(ConfiguredBaseModel):

reconcile/utils/helpers.py

@@ -1,4 +1,6 @@
 import logging
+import random
+import string
 from collections import Counter
 from collections.abc import (
     Iterable,
@@ -44,3 +46,9 @@ Item = TypeVar("Item")
 
 def find_duplicates(items: Iterable[Item]) -> list[Item]:
     return [item for item, count in Counter(items).items() if count > 1]
+
+
+def generate_random_password(string_length: int = 20) -> str:
+    """Generate a random string of letters and digits"""
+    letters_and_digits = string.ascii_letters + string.digits
+    return "".join(random.choices(letters_and_digits, k=string_length))