qontract-reconcile 0.10.2.dev40__py3-none-any.whl → 0.10.2.dev42__py3-none-any.whl

{qontract_reconcile-0.10.2.dev40.dist-info → qontract_reconcile-0.10.2.dev42.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev40
+Version: 0.10.2.dev42
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev40.dist-info → qontract_reconcile-0.10.2.dev42.dist-info}/RECORD

@@ -202,10 +202,10 @@ reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYY
 reconcile/external_resources/manager.py,sha256=DtxjWx34WdPjPR5TzqV4mZpN_Gn20LcNTZHBbPxqzuQ,16953
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=KiBjMUaN_z0cSkF_7Ar_a8RiuiwVqjyMcVdISlxhzXE,3898
-reconcile/external_resources/model.py,sha256=dxwiyI3J9xyLeue8_W9NJoap-CkKLMAoY0S0ml5-NbU,13450
-reconcile/external_resources/reconciler.py,sha256=_bUzDGieSikDOLcLxCim2BsbOCtFZusJu4tz6baYk9o,9676
+reconcile/external_resources/model.py,sha256=EpgIgVRPUsyfHhgjHv_TLUKjzFiIQt0wUd30K0NJJpI,11826
+reconcile/external_resources/reconciler.py,sha256=K9QvbQCIOCuOHnPIxQE_P_jFtrkF3dGo8d_cCCh08Ys,8973
 reconcile/external_resources/secrets_sync.py,sha256=50fK4fzgSz-K8uy5_DQQWA_ju_rTDYAC2HRymgfY7TA,16344
-reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
+reconcile/external_resources/state.py,sha256=ye8yjMoCtTHSRhDH7skFLDIHIuYTjisWYCTJrwnmbEw,9565
 reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=vCyg8W4ZUGxjU8tB1Gkre_auSpzo83n05mmO8_-7al0,8263
@@ -215,7 +215,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
 reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
 reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/introspection.json,sha256=g-GShl6Gr0Q-XLLEp1BvSBJ7c513YzmjEZvynMhztBk,2219565
+reconcile/gql_definitions/introspection.json,sha256=ErIjLKDbNw4SLyGXhp9iGmL1RSfT0U7gU1A3tsn1674,2217790
 reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
 reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -299,11 +299,9 @@ reconcile/gql_definitions/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_T
 reconcile/gql_definitions/endpoints_discovery/apps.py,sha256=aBWRAwDUJQ32ghJS4cPQcR9SNl20Fcwd3pxHDB3YJQY,3172
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
-reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=JViHtDWEBwjStBUo_bUdm_sxdpjCHcoATeFvwFRLQpU,3009
-reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=-0Q48lfH-fxAvwqYfXbLQ1aP7bFFh1BRQIYPnF7W9ZI,43067
-reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=WBkJqnoyYCe1Vimwbp_Pa0RdyTdmWNf6oEWyA749QzA,3589
-reconcile/gql_definitions/external_resources/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py,sha256=T_qWCRtzU8F9frebBXG9TkeQdrKGt3R9YinSngPoFqM,1262
+reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=cbbvGq1Te9DP8XiFg3bp4Y0q6LxpGYov8ugcROPyPLI,2647
+reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=dW7RrIA9gAJS7AHQwHggsJXhlSVnX0jV1c4W9e7a7Jc,45393
+reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=K8m9EKlfIVGP2KyqTduo7MMSKFjVC3yk5ZfO9hgdA7A,3192
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
 reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -635,7 +633,7 @@ reconcile/utils/state.py,sha256=az4tBmZ0EdbFcAGiBVUxs3cr2-BVWsuDQiNTvjjQq8s,1637
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
 reconcile/utils/terraform_client.py,sha256=H8frsS370y8xfivKLNBD1dwlBLHvfuR6JSN_syBL5Qc,36033
-reconcile/utils/terrascript_aws_client.py,sha256=UdEM3JeTMiE0VRqtz7gcBWR-c0fouORtPFrniRJ3pao,283505
+reconcile/utils/terrascript_aws_client.py,sha256=SNGtsG1n-IDZaI0blKLm3t3AfVNmxW-O8Y8NtX08OOc,270318
 reconcile/utils/three_way_diff_strategy.py,sha256=oQcHXd9LVhirJfoaOBoHUYuZVGfyL2voKr6KVI34zZE,4833
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/vault.py,sha256=aSA8l9cJlPUHpChFGl27nSY-Mpq9FMjBo7Dcgb1BVfM,15036
@@ -775,7 +773,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev40.dist-info/METADATA,sha256=HqfCumriHpBqayb84v68akimmifp5FDN5X7yAz7hsnY,24665
-qontract_reconcile-0.10.2.dev40.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev40.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev40.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev42.dist-info/METADATA,sha256=77Cr8xiwD37_vvev53K72DZy0oVPeSlFtUBb2HOyTzI,24665
+qontract_reconcile-0.10.2.dev42.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev42.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev42.dist-info/RECORD,,

reconcile/external_resources/model.py

@@ -18,6 +18,7 @@ from reconcile.gql_definitions.external_resources.external_resources_modules imp
     ExternalResourcesModuleV1,
 )
 from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
+    ExternalResourcesModuleOverridesV1,
     NamespaceTerraformProviderResourceAWSV1,
     NamespaceTerraformResourceElastiCacheV1,
     NamespaceTerraformResourceKMSV1,
@@ -28,10 +29,6 @@ from reconcile.gql_definitions.external_resources.external_resources_namespaces
 from reconcile.gql_definitions.external_resources.external_resources_settings import (
     ExternalResourcesSettingsV1,
 )
-from reconcile.gql_definitions.external_resources.fragments.external_resources_module_overrides import (
-    ExternalResourcesModuleOverrides,
-)
-from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
 from reconcile.utils.exceptions import FetchResourceError
 from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
@@ -223,38 +220,6 @@ def load_module_inventory(
     })
 
 
-class ResourcesSpec(BaseModel, frozen=True):
-    cpu: str | None = None
-    memory: str | None = None
-
-
-class Resources(BaseModel, frozen=True):
-    """Hashable class to store module resources in reconciliations.
-    Default values are used as a fallback for existent objects that were
-    created without container resources, hence they don't have mem/cpu resources
-    in the ERv2 State. Eventually, all resources will have resources assignments
-    from the module spec, module_overrides, or app-interface settings.
-    """
-
-    requests: ResourcesSpec = ResourcesSpec()
-    limits: ResourcesSpec = ResourcesSpec()
-
-    @staticmethod
-    def from_deploy_resources_fields(fields: DeployResourcesFields) -> "Resources":
-        """Create Resource obect from GQL DeployResourcesFields.
-
-        DeployResourceFields can not be used directly as it not hashable."""
-        return Resources(
-            requests=ResourcesSpec(
-                cpu=fields.requests.cpu, memory=fields.requests.memory
-            ),
-            limits=ResourcesSpec(
-                cpu=fields.limits.cpu,
-                memory=fields.limits.memory,
-            ),
-        )
-
-
 class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     image: str = ""
     version: str = ""
@@ -262,7 +227,6 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     reconcile_timeout_minutes: int = -1000
     outputs_secret_image: str = ""
     outputs_secret_version: str = ""
-    resources: Resources = Resources()
 
     @property
     def image_version(self) -> str:
@@ -280,14 +244,13 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     ) -> "ExternalResourceModuleConfiguration":
         module_overrides = spec.metadata.get(
             "module_overrides"
-        ) or ExternalResourcesModuleOverrides(
+        ) or ExternalResourcesModuleOverridesV1(
             module_type=None,
             image=None,
             version=None,
             reconcile_timeout_minutes=None,
             outputs_secret_image=None,
             outputs_secret_version=None,
-            resources=None,
         )
 
         return ExternalResourceModuleConfiguration(
@@ -302,11 +265,6 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
             outputs_secret_version=module_overrides.outputs_secret_version
             or module.outputs_secret_version
             or settings.outputs_secret_version,
-            resources=Resources.from_deploy_resources_fields(
-                module_overrides.resources
-                or module.resources
-                or settings.module_default_resources
-            ),
         )
 
 

reconcile/external_resources/reconciler.py

@@ -14,7 +14,6 @@ from kubernetes.client import (
     V1ObjectMeta,
     V1PodSpec,
     V1PodTemplateSpec,
-    V1ResourceRequirements,
     V1SecretVolumeSource,
     V1Volume,
     V1VolumeMount,
@@ -90,19 +89,10 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
         }
 
     def job_spec(self) -> V1JobSpec:
-        assert self.reconciliation.module_configuration.resources is not None
         job_container = V1Container(
             name="job",
             image=self.reconciliation.module_configuration.image_version,
             image_pull_policy="Always",
-            resources=V1ResourceRequirements(
-                requests=self.reconciliation.module_configuration.resources.requests.dict(
-                    exclude_none=True
-                ),
-                limits=self.reconciliation.module_configuration.resources.limits.dict(
-                    exclude_none=True
-                ),
-            ),
             env=[
                 V1EnvVar(
                     name="DRY_RUN",
@@ -112,12 +102,6 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
                     name="ACTION",
                     value=self.reconciliation.action.value,
                 ),
-                V1EnvVar(
-                    name="RECONCILE_TIMEOUT_MINUTES",
-                    value=str(
-                        self.reconciliation.module_configuration.reconcile_timeout_minutes
-                    ),
-                ),
             ],
             volume_mounts=[
                 V1VolumeMount(

reconcile/external_resources/state.py

@@ -11,8 +11,6 @@ from reconcile.external_resources.model import (
     ExternalResourceModuleConfiguration,
     Reconciliation,
     ReconciliationStatus,
-    Resources,
-    ResourcesSpec,
     ResourceStatus,
 )
 from reconcile.utils.aws_api_typed.api import AWSApi
@@ -75,48 +73,10 @@ class DynamoDBStateAdapter:
     MODCONF_VERSION = "version"
     MODCONF_DRIFT_MINS = "drift_detection_minutes"
     MODCONF_TIMEOUT_MINS = "timeout_minutes"
-    MODCONF_RESOURCES = "resources"
-    MODCONF_RESOURCES_REQUESTS = "requests"
-    MODCONF_RESOURCES_REQUESTS_CPU = "cpu"
-    MODCONF_RESOURCES_REQUESTS_MEMORY = "memory"
-    MODCONF_RESOURCES_LIMITS = "limits"
-    MODCONF_RESOURCES_LIMITS_CPU = "cpu"
-    MODCONF_RESOURCES_LIMITS_MEMORY = "memory"
 
     def _get_value(self, item: Mapping[str, Any], key: str, type: str = "S") -> Any:
-        if item[key][type] == "None":
-            return None
         return item[key][type]
 
-    def _build_resources(self, modconf: Mapping[str, Any]) -> Resources | None:
-        if self.MODCONF_RESOURCES not in modconf:
-            return Resources()
-        mc_resources = self._get_value(modconf, self.MODCONF_RESOURCES, type="M")
-        mc_resources_requests = self._get_value(
-            mc_resources, self.MODCONF_RESOURCES_REQUESTS, type="M"
-        )
-        mc_resources_limits = self._get_value(
-            mc_resources, self.MODCONF_RESOURCES_LIMITS, type="M"
-        )
-        return Resources(
-            requests=ResourcesSpec(
-                cpu=self._get_value(
-                    mc_resources_requests, self.MODCONF_RESOURCES_REQUESTS_CPU
-                ),
-                memory=self._get_value(
-                    mc_resources_requests, self.MODCONF_RESOURCES_REQUESTS_MEMORY
-                ),
-            ),
-            limits=ResourcesSpec(
-                cpu=self._get_value(
-                    mc_resources_limits, self.MODCONF_RESOURCES_LIMITS_CPU
-                ),
-                memory=self._get_value(
-                    mc_resources_limits, self.MODCONF_RESOURCES_LIMITS_MEMORY
-                ),
-            ),
-        )
-
     def deserialize(
         self,
         item: Mapping[str, Any],
@@ -156,7 +116,6 @@ class DynamoDBStateAdapter:
                     reconcile_timeout_minutes=self._get_value(
                         modconf, self.MODCONF_TIMEOUT_MINS, type="N"
                     ),
-                    resources=self._build_resources(modconf),
                 ),
             )
 
@@ -205,39 +164,7 @@ class DynamoDBStateAdapter:
                                     state.reconciliation.module_configuration.reconcile_timeout_minutes
                                 )
                             },
-                            self.MODCONF_RESOURCES: {
-                                "M": {
-                                    self.MODCONF_RESOURCES_REQUESTS: {
-                                        "M": {
-                                            self.MODCONF_RESOURCES_REQUESTS_CPU: {
-                                                "S": str(
-                                                    state.reconciliation.module_configuration.resources.requests.cpu
-                                                )
-                                            },
-                                            self.MODCONF_RESOURCES_REQUESTS_MEMORY: {
-                                                "S": str(
-                                                    state.reconciliation.module_configuration.resources.requests.memory
-                                                )
-                                            },
-                                        }
-                                    },
-                                    self.MODCONF_RESOURCES_LIMITS: {
-                                        "M": {
-                                            self.MODCONF_RESOURCES_LIMITS_CPU: {
-                                                "S": str(
-                                                    state.reconciliation.module_configuration.resources.limits.cpu
-                                                )
-                                            },
-                                            self.MODCONF_RESOURCES_LIMITS_MEMORY: {
-                                                "S": str(
-                                                    state.reconciliation.module_configuration.resources.limits.memory
-                                                )
-                                            },
-                                        }
-                                    },
-                                }
-                            },
-                        },
+                        }
                     },
                 }
             },

reconcile/gql_definitions/external_resources/external_resources_modules.py

@@ -17,21 +17,8 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
     Json,
 )
 
-from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
-
 
 DEFINITION = """
-fragment DeployResourcesFields on DeployResources_v1 {
-  requests {
-    cpu
-    memory
-  }
-  limits {
-    cpu
-    memory
-  }
-}
-
 query ExternalResourcesModules {
     modules: external_resources_modules_v1 {
         provision_provider
@@ -44,9 +31,6 @@ query ExternalResourcesModules {
         outputs_secret_sync
         outputs_secret_image
         outputs_secret_version
-        resources {
-          ... DeployResourcesFields
-        }
     }
 }
 """
@@ -69,7 +53,6 @@ class ExternalResourcesModuleV1(ConfiguredBaseModel):
     outputs_secret_sync: bool = Field(..., alias="outputs_secret_sync")
     outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
     outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
-    resources: Optional[DeployResourcesFields] = Field(..., alias="resources")
 
 
 class ExternalResourcesModulesQueryData(ConfiguredBaseModel):

reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -19,7 +19,6 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
 
 from reconcile.gql_definitions.fragments.aws_vpc import AWSVPC
 from reconcile.gql_definitions.fragments.jumphost_common_fields import CommonJumphostFields
-from reconcile.gql_definitions.external_resources.fragments.external_resources_module_overrides import ExternalResourcesModuleOverrides
 from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
 
 
@@ -54,29 +53,6 @@ fragment CommonJumphostFields on ClusterJumpHost_v1 {
   }
 }
 
-fragment DeployResourcesFields on DeployResources_v1 {
-  requests {
-    cpu
-    memory
-  }
-  limits {
-    cpu
-    memory
-  }
-}
-
-fragment ExternalResourcesModuleOverrides on ExternalResourcesModuleOverrides_v1 {
-  module_type
-  image
-  version
-  reconcile_timeout_minutes
-  outputs_secret_image
-  outputs_secret_version
-  resources {
-    ... DeployResourcesFields
-  }
-}
-
 fragment VaultSecret on VaultSecret_v1 {
     path
     field
@@ -135,7 +111,12 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                   ... ExternalResourcesModuleOverrides
+                    module_type
+                    image
+                    version
+                    reconcile_timeout_minutes
+                    outputs_secret_image
+                    outputs_secret_version
                 }
             }
             ... on NamespaceTerraformResourceS3_v1 {
@@ -168,7 +149,12 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                   ... ExternalResourcesModuleOverrides
+                  module_type
+                  image
+                  version
+                  reconcile_timeout_minutes
+                  outputs_secret_image
+                  outputs_secret_version
                 }
             }
             ... on NamespaceTerraformResourceServiceAccount_v1 {
@@ -289,7 +275,12 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                   ... ExternalResourcesModuleOverrides
+                  module_type
+                  image
+                  version
+                  reconcile_timeout_minutes
+                  outputs_secret_image
+                  outputs_secret_version
                 }
             }
             ... on NamespaceTerraformResourceElasticSearch_v1 {
@@ -501,7 +492,12 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                   ... ExternalResourcesModuleOverrides
+                  module_type
+                  image
+                  version
+                  reconcile_timeout_minutes
+                  outputs_secret_image
+                  outputs_secret_version
                 }
             }
         }
@@ -580,6 +576,15 @@ class AWSRDSDataClassificationV1(ConfiguredBaseModel):
     loss_impact: Optional[str] = Field(..., alias="loss_impact")
 
 
+class ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
+    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
+
+
 class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -599,7 +604,7 @@ class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     data_classification: Optional[AWSRDSDataClassificationV1] = Field(..., alias="data_classification")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
+    module_overrides: Optional[ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
 
 
 class AWSS3EventNotificationV1(ConfiguredBaseModel):
@@ -624,6 +629,15 @@ class NamespaceTerraformResourceS3V1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
 
 
+class NamespaceTerraformResourceElastiCacheV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
+    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
+
+
 class NamespaceTerraformResourceElastiCacheV1(NamespaceTerraformResourceAWSV1):
     identifier: str = Field(..., alias="identifier")
     defaults: str = Field(..., alias="defaults")
@@ -634,7 +648,7 @@ class NamespaceTerraformResourceElastiCacheV1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
+    module_overrides: Optional[NamespaceTerraformResourceElastiCacheV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
 
 
 class ClusterV1(ConfiguredBaseModel):
@@ -770,6 +784,15 @@ class NamespaceTerraformResourceCloudWatchV1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
 
 
+class NamespaceTerraformResourceKMSV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
+    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
+
+
 class NamespaceTerraformResourceKMSV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -779,7 +802,7 @@ class NamespaceTerraformResourceKMSV1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
+    module_overrides: Optional[NamespaceTerraformResourceKMSV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
 
 
 class NamespaceTerraformResourceElasticSearchV1(NamespaceTerraformResourceAWSV1):
@@ -1021,6 +1044,15 @@ class MskSecretParametersV1(ConfiguredBaseModel):
     secret: VaultSecret = Field(..., alias="secret")
 
 
+class NamespaceTerraformResourceMskV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
+    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
+
+
 class NamespaceTerraformResourceMskV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -1030,7 +1062,7 @@ class NamespaceTerraformResourceMskV1(NamespaceTerraformResourceAWSV1):
     users: Optional[list[MskSecretParametersV1]] = Field(..., alias="users")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
+    module_overrides: Optional[NamespaceTerraformResourceMskV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
 
 
 class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):

reconcile/gql_definitions/external_resources/external_resources_settings.py

@@ -17,21 +17,8 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
     Json,
 )
 
-from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
-
 
 DEFINITION = """
-fragment DeployResourcesFields on DeployResources_v1 {
-  requests {
-    cpu
-    memory
-  }
-  limits {
-    cpu
-    memory
-  }
-}
-
 query ExternalResourcesSettings {
     settings: external_resources_settings_v1 {
         state_dynamodb_account {
@@ -51,9 +38,6 @@ query ExternalResourcesSettings {
         vault_secrets_path
         outputs_secret_image
         outputs_secret_version
-        module_default_resources {
-          ... DeployResourcesFields
-        }
     }
 }
 """
@@ -89,7 +73,6 @@ class ExternalResourcesSettingsV1(ConfiguredBaseModel):
     vault_secrets_path: str = Field(..., alias="vault_secrets_path")
     outputs_secret_image: str = Field(..., alias="outputs_secret_image")
     outputs_secret_version: str = Field(..., alias="outputs_secret_version")
-    module_default_resources: DeployResourcesFields = Field(..., alias="module_default_resources")
 
 
 class ExternalResourcesSettingsQueryData(ConfiguredBaseModel):

reconcile/gql_definitions/introspection.json

@@ -33803,22 +33803,6 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
-                        },
-                        {
-                            "name": "module_default_resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "NON_NULL",
-                                "name": null,
-                                "ofType": {
-                                    "kind": "OBJECT",
-                                    "name": "DeployResources_v1",
-                                    "ofType": null
-                                }
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -34020,18 +34004,6 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
-                        },
-                        {
-                            "name": "resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "OBJECT",
-                                "name": "DeployResources_v1",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -42955,18 +42927,6 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
-                        },
-                        {
-                            "name": "resources",
-                            "description": null,
-                            "args": [],
-                            "type": {
-                                "kind": "OBJECT",
-                                "name": "DeployResources_v1",
-                                "ofType": null
-                            },
-                            "isDeprecated": false,
-                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,

reconcile/utils/terrascript_aws_client.py

@@ -77,8 +77,6 @@ from terrascript.resource import (
     aws_ec2_transit_gateway_vpc_attachment,
     aws_ec2_transit_gateway_vpc_attachment_accepter,
     aws_ecr_repository,
-    aws_elasticache_parameter_group,
-    aws_elasticache_replication_group,
     aws_elasticsearch_domain,
     aws_iam_access_key,
     aws_iam_group,
@@ -106,8 +104,6 @@ from terrascript.resource import (
     aws_lb_listener_rule,
     aws_lb_target_group,
     aws_lb_target_group_attachment,
-    aws_msk_cluster,
-    aws_msk_configuration,
     aws_ram_principal_association,
     aws_ram_resource_association,
     aws_ram_resource_share,
@@ -1645,8 +1641,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             self.populate_tf_resource_rds(spec)
         elif provider == "s3":
             self.populate_tf_resource_s3(spec)
-        elif provider == "elasticache":
-            self.populate_tf_resource_elasticache(spec)
         elif provider == "aws-iam-service-account":
             self.populate_tf_resource_service_account(spec, ocm_map=ocm_map)
         elif provider == "secrets-manager-service-account":
@@ -1689,8 +1683,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             self.populate_tf_resource_rosa_authenticator(spec)
         elif provider == "rosa-authenticator-vpce":
             self.populate_tf_resource_rosa_authenticator_vpce(spec)
-        elif provider == "msk":
-            self.populate_tf_resource_msk(spec)
         else:
             raise UnknownProviderError(provider)
 
@@ -2491,94 +2483,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         return bucket_tf_resource
 
-    def populate_tf_resource_elasticache(self, spec):
-        account = spec.provisioner_name
-        identifier = spec.identifier
-        values = self.init_values(spec)
-        output_prefix = spec.output_prefix
-        values.setdefault("replication_group_id", values["identifier"])
-        values.pop("identifier", None)
-
-        tf_resources = []
-        self.init_common_outputs(tf_resources, spec)
-
-        default_region = self.default_regions.get(account)
-        desired_region = values.pop("region", default_region)
-
-        provider = ""
-        if desired_region is not None and self._multiregion_account(account):
-            provider = "aws." + desired_region
-            values["provider"] = provider
-
-        if not values.get("apply_immediately"):
-            values["apply_immediately"] = False
-
-        parameter_group = values.get("parameter_group")
-        # Assume that cluster enabled is false if parameter group unset
-        pg_cluster_enabled = False
-
-        if parameter_group:
-            pg_values = self.get_values(parameter_group)
-            pg_name = pg_values["name"]
-            pg_identifier = pg_name
-
-            # If the desired region is not the same as the default region
-            # we append the region to the identifier to make it unique
-            # in the terraform config
-            if desired_region is not None and desired_region != default_region:
-                pg_identifier = f"{pg_name}-{desired_region}"
-
-            pg_values["parameter"] = pg_values.pop("parameters")
-            for param in pg_values["parameter"]:
-                if param["name"] == "cluster-enabled" and param["value"] == "yes":
-                    pg_cluster_enabled = True
-
-            if self._multiregion_account(account) and len(provider) > 0:
-                pg_values["provider"] = provider
-            pg_tf_resource = aws_elasticache_parameter_group(pg_identifier, **pg_values)
-            tf_resources.append(pg_tf_resource)
-            values["depends_on"] = [
-                f"aws_elasticache_parameter_group.{pg_identifier}",
-            ]
-            values["parameter_group_name"] = pg_name
-            values.pop("parameter_group", None)
-
-        auth_token = spec.get_secret_field("db.auth_token")
-        if not auth_token:
-            auth_token = self.generate_random_password()
-
-        if values.get("transit_encryption_enabled", False):
-            values["auth_token"] = auth_token
-
-        # elasticache replication group
-        # Ref: https://www.terraform.io/docs/providers/aws/r/
-        # elasticache_replication_group.html
-        tf_resource = aws_elasticache_replication_group(identifier, **values)
-        tf_resources.append(tf_resource)
-        # elasticache outputs
-        # we want the outputs to be formed into an OpenShift Secret
-        # with the following fields
-        # db.endpoint
-        output_name = output_prefix + "__db_endpoint"
-        # https://docs.aws.amazon.com/AmazonElastiCache/
-        # latest/red-ug/Endpoints.html
-        if pg_cluster_enabled:
-            output_value = "${" + tf_resource.configuration_endpoint_address + "}"
-        else:
-            output_value = "${" + tf_resource.primary_endpoint_address + "}"
-        tf_resources.append(Output(output_name, value=output_value))
-        # db.port
-        output_name = output_prefix + "__db_port"
-        output_value = "${" + str(tf_resource.port) + "}"
-        tf_resources.append(Output(output_name, value=output_value))
-        # db.auth_token
-        if values.get("transit_encryption_enabled", False):
-            output_name = output_prefix + "__db_auth_token"
-            output_value = values["auth_token"]
-            tf_resources.append(Output(output_name, value=output_value, sensitive=True))
-
-        self.add_resources(account, tf_resources)
-
     def populate_tf_resource_service_account(self, spec, ocm_map=None):
         account = spec.provisioner_name
         identifier = spec.identifier
@@ -4287,24 +4191,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
                     values["db_name"] = db_name
                 if values.get("replica_source"):
                     values.pop("db_name", None)
-            elif spec.provider == "elasticache":
-                if description := values.pop("replication_group_description", None):
-                    values["description"] = description
-                if num_cache_clusters := values.pop("number_cache_clusters", None):
-                    values["num_cache_clusters"] = num_cache_clusters
-                if cluster_mode := values.pop("cluster_mode", {}):
-                    for k, v in cluster_mode.items():
-                        values[k] = v
-                values.pop("availability_zones", None)
-            elif spec.provider == "msk":
-                if ebs_volume_size := values.get("broker_node_group_info", {}).pop(
-                    "ebs_volume_size", None
-                ):
-                    values["broker_node_group_info"].setdefault(
-                        "storage_info", {}
-                    ).setdefault("ebs_storage_info", {})[
-                        "volume_size"
-                    ] = ebs_volume_size
 
         return values
 
@@ -6656,205 +6542,6 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
 
         self.add_resources(account, tf_resources)
 
-    def populate_tf_resource_msk(self, spec):
-        account = spec.provisioner_name
-        values = self.init_values(spec)
-        output_prefix = spec.output_prefix
-        tf_resources = []
-        resource_id = spec.identifier
-
-        del values["identifier"]
-        values.setdefault("cluster_name", spec.identifier)
-
-        # common
-        self.init_common_outputs(tf_resources, spec)
-
-        # validations
-        if (
-            values["number_of_broker_nodes"]
-            % len(values["broker_node_group_info"]["client_subnets"])
-            != 0
-        ):
-            raise ValueError(
-                "number_of_broker_nodes must be a multiple of the number of specified client subnets."
-            )
-
-        scram_enabled = (
-            values.get("client_authentication", {}).get("sasl", {}).get("scram", False)
-        )
-        scram_users = {}
-        if scram_enabled:
-            if not spec.resource.get("users", []):
-                raise ValueError(
-                    "users attribute must be given when client_authentication.sasl.scram is enabled."
-                )
-            scram_users = {
-                user["name"]: self.secret_reader.read_all(user["secret"])
-                for user in spec.resource["users"]
-            }
-            # validate user objects
-            for user, secret in scram_users.items():
-                if secret.keys() != {"password", "username"}:
-                    raise ValueError(
-                        f"MSK user '{user}' secret must contain only 'username' and 'password' keys!"
-                    )
-
-        # resource - msk config
-        # unique msk config resource name enables "create_before_destroy" lifecycle
-        # which is required when changing version which requires a resource replacement
-        msk_version_str = values["kafka_version"].replace(".", "-")
-        msk_config_name = f"{resource_id}-{msk_version_str}"
-        msk_config = aws_msk_configuration(
-            msk_config_name,
-            name=msk_config_name,
-            kafka_versions=[values["kafka_version"]],
-            server_properties=values["server_properties"],
-            # lifecycle create_before_destroy is required to ensure that the config is created
-            # before it is assigned to the cluster
-            lifecycle={
-                "create_before_destroy": True,
-            },
-        )
-        tf_resources.append(msk_config)
-        values.pop("server_properties", None)
-
-        # resource - cluster
-        values["configuration_info"] = {
-            "arn": "${" + msk_config.arn + "}",
-            "revision": "${" + msk_config.latest_revision + "}",
-        }
-        msk_cluster = aws_msk_cluster(resource_id, **values)
-        tf_resources.append(msk_cluster)
-
-        # resource - cloudwatch
-        if (
-            values.get("logging_info", {})
-            .get("broker_logs", {})
-            .get("cloudwatch_logs", {})
-            .get("enabled", False)
-        ):
-            log_group_values = {
-                "name": f"{resource_id}-msk-broker-logs",
-                "tags": values["tags"],
-                "retention_in_days": values["logging_info"]["broker_logs"][
-                    "cloudwatch_logs"
-                ]["retention_in_days"],
-            }
-            log_group_tf_resource = aws_cloudwatch_log_group(
-                resource_id, **log_group_values
-            )
-            tf_resources.append(log_group_tf_resource)
-            del values["logging_info"]["broker_logs"]["cloudwatch_logs"][
-                "retention_in_days"
-            ]
-            values["logging_info"]["broker_logs"]["cloudwatch_logs"]["log_group"] = (
-                log_group_tf_resource.name
-            )
-
-        # resource - secret manager for SCRAM client credentials
-        if scram_enabled and scram_users:
-            scram_secrets: list[
-                tuple[aws_secretsmanager_secret, aws_secretsmanager_secret_version]
-            ] = []
-
-            # kms
-            kms_values = {
-                "description": "KMS key for MSK SCRAM credentials",
-                "tags": values["tags"],
-            }
-            kms_key = aws_kms_key(resource_id, **kms_values)
-            tf_resources.append(kms_key)
-
-            kms_key_alias = aws_kms_alias(
-                resource_id,
-                name=f"alias/{resource_id}-msk-scram",
-                target_key_id="${" + kms_key.arn + "}",
-            )
-            tf_resources.append(kms_key_alias)
-
-            for user, secret in scram_users.items():
-                secret_identifier = f"AmazonMSK_{resource_id}-{user}"
-
-                secret_values = {
-                    "name": secret_identifier,
-                    "tags": values["tags"],
-                    "kms_key_id": "${" + kms_key.arn + "}",
-                }
-                secret_resource = aws_secretsmanager_secret(
-                    secret_identifier, **secret_values
-                )
-                tf_resources.append(secret_resource)
-
-                version_values = {
-                    "secret_id": "${" + secret_resource.arn + "}",
-                    "secret_string": json.dumps(secret, sort_keys=True),
-                }
-                version_resource = aws_secretsmanager_secret_version(
-                    secret_identifier, **version_values
-                )
-                tf_resources.append(version_resource)
-
-                secret_policy_values = {
-                    "secret_arn": "${" + secret_resource.arn + "}",
-                    "policy": json.dumps({
-                        "Version": "2012-10-17",
-                        "Statement": [
-                            {
-                                "Sid": "AWSKafkaResourcePolicy",
-                                "Effect": "Allow",
-                                "Principal": {"Service": "kafka.amazonaws.com"},
-                                "Action": "secretsmanager:getSecretValue",
-                                "Resource": "${" + secret_resource.arn + "}",
-                            }
-                        ],
-                    }),
-                }
-                secret_policy = aws_secretsmanager_secret_policy(
-                    secret_identifier, **secret_policy_values
-                )
-                tf_resources.append(secret_policy)
-                scram_secrets.append((secret_resource, version_resource))
-
-            # create ONE scram secret association for each secret created above
-            scram_secret_association_values = {
-                "cluster_arn": "${" + msk_cluster.arn + "}",
-                "secret_arn_list": ["${" + s.arn + "}" for s, _ in scram_secrets],
-                "depends_on": self.get_dependencies([v for _, v in scram_secrets]),
-            }
-            scram_secret_association = aws_msk_scram_secret_association(
-                resource_id, **scram_secret_association_values
-            )
-            tf_resources.append(scram_secret_association)
-
-        # outputs
-        tf_resources += [
-            Output(
-                output_prefix + "__zookeeper_connect_string",
-                value="${" + msk_cluster.zookeeper_connect_string + "}",
-            ),
-            Output(
-                output_prefix + "__zookeeper_connect_string_tls",
-                value="${" + msk_cluster.zookeeper_connect_string_tls + "}",
-            ),
-            Output(
-                output_prefix + "__bootstrap_brokers",
-                value="${" + msk_cluster.bootstrap_brokers + "}",
-            ),
-            Output(
-                output_prefix + "__bootstrap_brokers_tls",
-                value="${" + msk_cluster.bootstrap_brokers_tls + "}",
-            ),
-            Output(
-                output_prefix + "__bootstrap_brokers_sasl_iam",
-                value="${" + msk_cluster.bootstrap_brokers_sasl_iam + "}",
-            ),
-            Output(
-                output_prefix + "__bootstrap_brokers_sasl_scram",
-                value="${" + msk_cluster.bootstrap_brokers_sasl_scram + "}",
-            ),
-        ]
-        self.add_resources(account, tf_resources)
-
     def populate_saml_idp(self, account_name: str, name: str, metadata: str) -> None:
         saml_idp = aws_iam_saml_provider(
             f"{account_name}-{name}", name=name, saml_metadata_document=metadata

reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py

@@ -1,36 +0,0 @@
-"""
-Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
-"""
-from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
-from datetime import datetime  # noqa: F401 # pylint: disable=W0611
-from enum import Enum  # noqa: F401 # pylint: disable=W0611
-from typing import (  # noqa: F401 # pylint: disable=W0611
-    Any,
-    Optional,
-    Union,
-)
-
-from pydantic import (  # noqa: F401 # pylint: disable=W0611
-    BaseModel,
-    Extra,
-    Field,
-    Json,
-)
-
-from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
-
-
-class ConfiguredBaseModel(BaseModel):
-    class Config:
-        smart_union=True
-        extra=Extra.forbid
-
-
-class ExternalResourcesModuleOverrides(ConfiguredBaseModel):
-    module_type: Optional[str] = Field(..., alias="module_type")
-    image: Optional[str] = Field(..., alias="image")
-    version: Optional[str] = Field(..., alias="version")
-    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
-    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
-    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
-    resources: Optional[DeployResourcesFields] = Field(..., alias="resources")