qontract-reconcile 0.10.2.dev44__py3-none-any.whl → 0.10.2.dev46__py3-none-any.whl

{qontract_reconcile-0.10.2.dev44.dist-info → qontract_reconcile-0.10.2.dev46.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev44
+Version: 0.10.2.dev46
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev44.dist-info → qontract_reconcile-0.10.2.dev46.dist-info}/RECORD

@@ -195,17 +195,17 @@ reconcile/endpoints_discovery/integration.py,sha256=fzy5tv4c_6WoAZGGBNJ276NVNB1O
 reconcile/endpoints_discovery/merge_request.py,sha256=_yLb4tnvoZMCko8rta2C_CvOInJa9pa3HzSmHNtjgGU,2978
 reconcile/endpoints_discovery/merge_request_manager.py,sha256=wUMsumxv8RnWaRattax4HfoRlhtVzmgro3GiJJ1C4Vc,6392
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/external_resources/aws.py,sha256=2vimGEtcwcIyVaFvt5Ab2PAAWu2hawJyCu4fuZWSobI,7783
-reconcile/external_resources/factories.py,sha256=ldvm3n13IdE899Ei850NSf0RQ35rzsKJBCUkROrbE5Y,5579
+reconcile/external_resources/aws.py,sha256=KAs656zj4oZYpVbgWsDLXnJWJlLOZdR1JnRhikbF4x0,10712
+reconcile/external_resources/factories.py,sha256=C0QHT0soEv6z99-ELAAE19S5MaMHhV0t1fSiQn0Coc4,5970
 reconcile/external_resources/integration.py,sha256=JF38M7R0Z4ADUTx57TZqSZH9k_xpPlbAxQAcGyIISuM,6925
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
-reconcile/external_resources/manager.py,sha256=DtxjWx34WdPjPR5TzqV4mZpN_Gn20LcNTZHBbPxqzuQ,16953
+reconcile/external_resources/manager.py,sha256=ZagwLn6YQ1XmgmMN3qpuDzQsQxa4VOYl-IQPZBwCDqM,17103
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=KiBjMUaN_z0cSkF_7Ar_a8RiuiwVqjyMcVdISlxhzXE,3898
-reconcile/external_resources/model.py,sha256=EpgIgVRPUsyfHhgjHv_TLUKjzFiIQt0wUd30K0NJJpI,11826
-reconcile/external_resources/reconciler.py,sha256=K9QvbQCIOCuOHnPIxQE_P_jFtrkF3dGo8d_cCCh08Ys,8973
+reconcile/external_resources/model.py,sha256=dxwiyI3J9xyLeue8_W9NJoap-CkKLMAoY0S0ml5-NbU,13450
+reconcile/external_resources/reconciler.py,sha256=PK00cyMfBZLHH0W7w2bNomgAy85SotIVXbCBkGPSJdM,9432
 reconcile/external_resources/secrets_sync.py,sha256=50fK4fzgSz-K8uy5_DQQWA_ju_rTDYAC2HRymgfY7TA,16344
-reconcile/external_resources/state.py,sha256=ye8yjMoCtTHSRhDH7skFLDIHIuYTjisWYCTJrwnmbEw,9565
+reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
 reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=vCyg8W4ZUGxjU8tB1Gkre_auSpzo83n05mmO8_-7al0,8263
@@ -215,7 +215,7 @@ reconcile/glitchtip_project_alerts/integration.py,sha256=BgMx-NyV9mTuv7Sotb2OioC
 reconcile/glitchtip_project_dsn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip_project_dsn/integration.py,sha256=2iugub-kHYkHNK33n0v9_TeWonuxCPah_VkoTPvaajE,8077
 reconcile/gql_definitions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/gql_definitions/introspection.json,sha256=6e7VKZ_Ds1-fNKLF8Jf3yOHZ0SiIeYhAY277cpSoPdE,2235110
+reconcile/gql_definitions/introspection.json,sha256=jAWK0rPdJY39eevr2NWgD8lqbJew_m5BIhyEMbb_3lo,2236885
 reconcile/gql_definitions/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/acs/acs_instances.py,sha256=L91WW9LbhJbBSrECqShQpFtjoBOsmNIYLRpMbx1io5o,2181
 reconcile/gql_definitions/acs/acs_policies.py,sha256=bN5i4mks10Z23KJSj7jqp966Osq2dps4d-sPH9gjxEA,7008
@@ -299,9 +299,11 @@ reconcile/gql_definitions/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_T
 reconcile/gql_definitions/endpoints_discovery/apps.py,sha256=aBWRAwDUJQ32ghJS4cPQcR9SNl20Fcwd3pxHDB3YJQY,3172
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
-reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=cbbvGq1Te9DP8XiFg3bp4Y0q6LxpGYov8ugcROPyPLI,2647
-reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=dW7RrIA9gAJS7AHQwHggsJXhlSVnX0jV1c4W9e7a7Jc,45393
-reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=K8m9EKlfIVGP2KyqTduo7MMSKFjVC3yk5ZfO9hgdA7A,3192
+reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=JViHtDWEBwjStBUo_bUdm_sxdpjCHcoATeFvwFRLQpU,3009
+reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=-0Q48lfH-fxAvwqYfXbLQ1aP7bFFh1BRQIYPnF7W9ZI,43067
+reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=WBkJqnoyYCe1Vimwbp_Pa0RdyTdmWNf6oEWyA749QzA,3589
+reconcile/gql_definitions/external_resources/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py,sha256=T_qWCRtzU8F9frebBXG9TkeQdrKGt3R9YinSngPoFqM,1262
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
 reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -751,7 +753,7 @@ tools/sd_app_sre_alert_report.py,sha256=jQpJdXVID68bSNtJNOGDh0-ei1CfEUS4Itr4MAaB
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/cli_commands/container_images_report.py,sha256=8fG9XU-eEhJ7hKCdQzBcdPpvIJR-8WGkHOgFEulpfYQ,5213
-tools/cli_commands/erv2.py,sha256=iqihq908iJYIQoaNq3Iofp5SQP8S-DhTurzGdnEragU,23218
+tools/cli_commands/erv2.py,sha256=VxUlNXllo947UwmtvS-42IeI9x_t_X3MHrrSI3K_GRo,23274
 tools/cli_commands/gpg_encrypt.py,sha256=NhzwN49UN7P5_FJgTUN5A4BIwNbFokIE4lwDax2iP5k,4891
 tools/cli_commands/systems_and_tools.py,sha256=EMHOF1AtUDaoSk0bbjl6oUKYAz4rTZjIBaF-6E6GspM,16816
 tools/cli_commands/cost_report/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -773,7 +775,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev44.dist-info/METADATA,sha256=VYLbNmLeQFocQBk-JLSHKadTfEMFX8Oq3W0GgOBUpkk,24665
-qontract_reconcile-0.10.2.dev44.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev44.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev44.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev46.dist-info/METADATA,sha256=C5aPbZDbkFrj5j4y25HOkeCX-7xvvhvxDFYG1ymRabE,24665
+qontract_reconcile-0.10.2.dev46.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev46.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev46.dist-info/RECORD,,

reconcile/external_resources/aws.py

@@ -1,9 +1,11 @@
+import re
 from abc import ABC, abstractmethod
 from typing import Any
 
 from reconcile.external_resources.model import (
     ExternalResource,
     ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
     ExternalResourcesInventory,
 )
 from reconcile.utils.external_resource_spec import (
@@ -21,10 +23,18 @@ class AWSResourceFactory(ABC):
         self.secret_reader = secret_reader
 
     @abstractmethod
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]: ...
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]: ...
 
     @abstractmethod
-    def validate(self, resource: ExternalResource) -> None: ...
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None: ...
 
     def find_linked_resources(
         self, spec: ExternalResourceSpec
@@ -35,10 +45,18 @@ class AWSResourceFactory(ABC):
 
 
 class AWSDefaultResourceFactory(AWSResourceFactory):
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         return ResourceValueResolver(spec=spec, identifier_as_value=True).resolve()
 
-    def validate(self, resource: ExternalResource) -> None: ...
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None: ...
 
 
 class AWSElasticacheFactory(AWSDefaultResourceFactory):
@@ -49,7 +67,11 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
             "aws", provisioner, "elasticache", identifier
         )
 
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         """Resolve the elasticache resource specification and translate some attributes to AWS >= 5.60.0 provider format."""
         rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
         data = rvr.resolve()
@@ -70,7 +92,11 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
 
         return data
 
-    def validate(self, resource: ExternalResource) -> None:
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         """Validate the elasticache resource specification."""
         data = resource.data
         if data.get("parameter_group"):
@@ -96,6 +122,8 @@ class AWSElasticacheFactory(AWSDefaultResourceFactory):
 
 
 class AWSRdsFactory(AWSDefaultResourceFactory):
+    TIMEOUT_RE = re.compile(r"^\d+m$")
+
     def _get_source_db_spec(
         self, provisioner: str, identifier: str
     ) -> ExternalResourceSpec:
@@ -110,7 +138,11 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
             "aws", provisioner, "kms", identifier
         )
 
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
         data = rvr.resolve()
 
@@ -126,7 +158,7 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
             sourcedb_spec = self._get_source_db_spec(
                 spec.provisioner_name, data["replica_source"]
             )
-            sourcedb = self.resolve(sourcedb_spec)
+            sourcedb = self.resolve(sourcedb_spec, module_conf)
             sourcedb_region = (
                 sourcedb.get("region", None)
                 or sourcedb_spec.provisioner["resources_default_region"]
@@ -142,9 +174,59 @@ class AWSRdsFactory(AWSDefaultResourceFactory):
                 spec.provisioner_name, kms_key_id
             ).identifier
 
+        # If not timeouts are set, set default timeouts according to the module reconcile timeout configuration
+        # 5 minutes are substracted to let terraform finish gracefully before the Job is killed.
+        if "timeouts" not in data:
+            data["timeouts"] = {
+                "create": f"{module_conf.reconcile_timeout_minutes - 5}m",
+                "update": f"{module_conf.reconcile_timeout_minutes - 5}m",
+                "delete": f"{module_conf.reconcile_timeout_minutes - 5}m",
+            }
         return data
 
-    def validate(self, resource: ExternalResource) -> None: ...
+    def _get_timeout_minutes(
+        self,
+        timeout: str,
+    ) -> int:
+        if not re.match(AWSRdsFactory.TIMEOUT_RE, timeout):
+            raise ValueError(
+                f"Invalid RDS instance timeout format: {timeout}. Specify timeout in minutes(m)."
+            )
+        return int(timeout[:-1])
+
+    def _validate_timeouts(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
+        timeouts = resource.data.get("timeouts")
+        if not timeouts:
+            return
+
+        if not isinstance(timeouts, dict):
+            raise ValueError(
+                "Timeouts must be a dictionary with 'create', 'update' and/or 'delete' keys."
+            )
+
+        allowed_keys = {"create", "update", "delete"}
+        if unknown_keys := timeouts.keys() - allowed_keys:
+            raise ValueError(
+                f"Timeouts must be a dictionary with 'create', 'update' and/or 'delete' keys. Offending keys: {unknown_keys}."
+            )
+
+        for option, timeout in timeouts.items():
+            timeout_minutes = self._get_timeout_minutes(timeout)
+            if timeout_minutes >= module_conf.reconcile_timeout_minutes:
+                raise ValueError(
+                    f"RDS instance {option} timeout value {timeout_minutes} must be lower than the module reconcile_timeout_minutes value {module_conf.reconcile_timeout_minutes}."
+                )
+
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
+        self._validate_timeouts(resource, module_conf)
 
     def find_linked_resources(
         self, spec: ExternalResourceSpec
@@ -166,7 +248,11 @@ class AWSMskFactory(AWSDefaultResourceFactory):
             "aws", provisioner, "msk", identifier
         )
 
-    def resolve(self, spec: ExternalResourceSpec) -> dict[str, Any]:
+    def resolve(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> dict[str, Any]:
         rvr = ResourceValueResolver(spec=spec, identifier_as_value=True)
         data = rvr.resolve()
         data["output_prefix"] = spec.output_prefix
@@ -188,7 +274,11 @@ class AWSMskFactory(AWSDefaultResourceFactory):
             del data["users"]
         return data
 
-    def validate(self, resource: ExternalResource) -> None:
+    def validate(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         data = resource.data
         if (
             data["number_of_broker_nodes"]

reconcile/external_resources/factories.py

@@ -15,6 +15,7 @@ from reconcile.external_resources.meta import QONTRACT_INTEGRATION
 from reconcile.external_resources.model import (
     ExternalResource,
     ExternalResourceKey,
+    ExternalResourceModuleConfiguration,
     ExternalResourceProvision,
     ExternalResourcesInventory,
     ModuleInventory,
@@ -55,11 +56,19 @@ class ObjectFactory(Generic[T]):
 
 class ExternalResourceFactory(ABC):
     @abstractmethod
-    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+    def create_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
         pass
 
     @abstractmethod
-    def validate_external_resource(self, resource: ExternalResource) -> None:
+    def validate_external_resource(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         pass
 
     def find_linked_resources(
@@ -121,9 +130,13 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
         self.er_inventory = er_inventory
         self.secret_reader = secret_reader
 
-    def create_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+    def create_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
         f = self.resource_factories.get_factory(spec.provider)
-        data = f.resolve(spec)
+        data = f.resolve(spec, module_conf)
         data["tags"] = spec.tags(integration=QONTRACT_INTEGRATION)
         data["default_tags"] = AWS_DEFAULT_TAGS
 
@@ -152,9 +165,13 @@ class AWSExternalResourceFactory(ExternalResourceFactory):
 
         return ExternalResource(data=data, provision=provision)
 
-    def validate_external_resource(self, resource: ExternalResource) -> None:
+    def validate_external_resource(
+        self,
+        resource: ExternalResource,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> None:
         f = self.resource_factories.get_factory(resource.provision.provider)
-        f.validate(resource)
+        f.validate(resource, module_conf)
 
     def find_linked_resources(
         self, spec: ExternalResourceSpec

reconcile/external_resources/manager.py

@@ -197,8 +197,11 @@ class ExternalResourcesManager:
             if spec.marked_to_delete:
                 continue
             module = self.module_inventory.get_from_spec(spec)
+            module_conf = ExternalResourceModuleConfiguration.resolve_configuration(
+                module, spec, self.settings
+            )
             try:
-                resource = self._build_external_resource(spec)
+                resource = self._build_external_resource(spec, module_conf)
             except ExternalResourceValidationError as e:
                 self.errors[key] = e
                 continue
@@ -208,9 +211,7 @@ class ExternalResourcesManager:
                 resource_hash=resource.hash(),
                 input=resource.json(),
                 action=Action.APPLY,
-                module_configuration=ExternalResourceModuleConfiguration.resolve_configuration(
-                    module, spec, self.settings
-                ),
+                module_configuration=module_conf,
                 linked_resources=self._find_linked_resources(spec),
             )
             r.add(reconciliation)
@@ -362,10 +363,14 @@ class ExternalResourcesManager:
                 )
                 self.state_mgr.update_resource_status(key, ResourceStatus.CREATED)
 
-    def _build_external_resource(self, spec: ExternalResourceSpec) -> ExternalResource:
+    def _build_external_resource(
+        self,
+        spec: ExternalResourceSpec,
+        module_conf: ExternalResourceModuleConfiguration,
+    ) -> ExternalResource:
         f = self.factories.get_factory(spec.provision_provider)
-        resource = f.create_external_resource(spec)
-        f.validate_external_resource(resource)
+        resource = f.create_external_resource(spec, module_conf)
+        f.validate_external_resource(resource, module_conf)
         return resource
 
     def _find_linked_resources(

reconcile/external_resources/model.py

@@ -18,7 +18,6 @@ from reconcile.gql_definitions.external_resources.external_resources_modules imp
     ExternalResourcesModuleV1,
 )
 from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
-    ExternalResourcesModuleOverridesV1,
     NamespaceTerraformProviderResourceAWSV1,
     NamespaceTerraformResourceElastiCacheV1,
     NamespaceTerraformResourceKMSV1,
@@ -29,6 +28,10 @@ from reconcile.gql_definitions.external_resources.external_resources_namespaces
 from reconcile.gql_definitions.external_resources.external_resources_settings import (
     ExternalResourcesSettingsV1,
 )
+from reconcile.gql_definitions.external_resources.fragments.external_resources_module_overrides import (
+    ExternalResourcesModuleOverrides,
+)
+from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
 from reconcile.utils.exceptions import FetchResourceError
 from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
@@ -220,6 +223,38 @@ def load_module_inventory(
     })
 
 
+class ResourcesSpec(BaseModel, frozen=True):
+    cpu: str | None = None
+    memory: str | None = None
+
+
+class Resources(BaseModel, frozen=True):
+    """Hashable class to store module resources in reconciliations.
+    Default values are used as a fallback for existent objects that were
+    created without container resources, hence they don't have mem/cpu resources
+    in the ERv2 State. Eventually, all resources will have resources assignments
+    from the module spec, module_overrides, or app-interface settings.
+    """
+
+    requests: ResourcesSpec = ResourcesSpec()
+    limits: ResourcesSpec = ResourcesSpec()
+
+    @staticmethod
+    def from_deploy_resources_fields(fields: DeployResourcesFields) -> "Resources":
+        """Create Resource obect from GQL DeployResourcesFields.
+
+        DeployResourceFields can not be used directly as it not hashable."""
+        return Resources(
+            requests=ResourcesSpec(
+                cpu=fields.requests.cpu, memory=fields.requests.memory
+            ),
+            limits=ResourcesSpec(
+                cpu=fields.limits.cpu,
+                memory=fields.limits.memory,
+            ),
+        )
+
+
 class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     image: str = ""
     version: str = ""
@@ -227,6 +262,7 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     reconcile_timeout_minutes: int = -1000
     outputs_secret_image: str = ""
     outputs_secret_version: str = ""
+    resources: Resources = Resources()
 
     @property
     def image_version(self) -> str:
@@ -244,13 +280,14 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     ) -> "ExternalResourceModuleConfiguration":
         module_overrides = spec.metadata.get(
             "module_overrides"
-        ) or ExternalResourcesModuleOverridesV1(
+        ) or ExternalResourcesModuleOverrides(
             module_type=None,
             image=None,
             version=None,
             reconcile_timeout_minutes=None,
             outputs_secret_image=None,
             outputs_secret_version=None,
+            resources=None,
         )
 
         return ExternalResourceModuleConfiguration(
@@ -265,6 +302,11 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
             outputs_secret_version=module_overrides.outputs_secret_version
             or module.outputs_secret_version
             or settings.outputs_secret_version,
+            resources=Resources.from_deploy_resources_fields(
+                module_overrides.resources
+                or module.resources
+                or settings.module_default_resources
+            ),
         )
 
 

reconcile/external_resources/reconciler.py

@@ -14,6 +14,7 @@ from kubernetes.client import (
     V1ObjectMeta,
     V1PodSpec,
     V1PodTemplateSpec,
+    V1ResourceRequirements,
     V1SecretVolumeSource,
     V1Volume,
     V1VolumeMount,
@@ -89,10 +90,19 @@ class ReconciliationK8sJob(K8sJob, BaseModel, frozen=True):
         }
 
     def job_spec(self) -> V1JobSpec:
+        assert self.reconciliation.module_configuration.resources is not None
         job_container = V1Container(
             name="job",
             image=self.reconciliation.module_configuration.image_version,
             image_pull_policy="Always",
+            resources=V1ResourceRequirements(
+                requests=self.reconciliation.module_configuration.resources.requests.dict(
+                    exclude_none=True
+                ),
+                limits=self.reconciliation.module_configuration.resources.limits.dict(
+                    exclude_none=True
+                ),
+            ),
             env=[
                 V1EnvVar(
                     name="DRY_RUN",

reconcile/external_resources/state.py

@@ -11,6 +11,8 @@ from reconcile.external_resources.model import (
     ExternalResourceModuleConfiguration,
     Reconciliation,
     ReconciliationStatus,
+    Resources,
+    ResourcesSpec,
     ResourceStatus,
 )
 from reconcile.utils.aws_api_typed.api import AWSApi
@@ -73,10 +75,48 @@ class DynamoDBStateAdapter:
     MODCONF_VERSION = "version"
     MODCONF_DRIFT_MINS = "drift_detection_minutes"
     MODCONF_TIMEOUT_MINS = "timeout_minutes"
+    MODCONF_RESOURCES = "resources"
+    MODCONF_RESOURCES_REQUESTS = "requests"
+    MODCONF_RESOURCES_REQUESTS_CPU = "cpu"
+    MODCONF_RESOURCES_REQUESTS_MEMORY = "memory"
+    MODCONF_RESOURCES_LIMITS = "limits"
+    MODCONF_RESOURCES_LIMITS_CPU = "cpu"
+    MODCONF_RESOURCES_LIMITS_MEMORY = "memory"
 
     def _get_value(self, item: Mapping[str, Any], key: str, type: str = "S") -> Any:
+        if item[key][type] == "None":
+            return None
         return item[key][type]
 
+    def _build_resources(self, modconf: Mapping[str, Any]) -> Resources | None:
+        if self.MODCONF_RESOURCES not in modconf:
+            return Resources()
+        mc_resources = self._get_value(modconf, self.MODCONF_RESOURCES, type="M")
+        mc_resources_requests = self._get_value(
+            mc_resources, self.MODCONF_RESOURCES_REQUESTS, type="M"
+        )
+        mc_resources_limits = self._get_value(
+            mc_resources, self.MODCONF_RESOURCES_LIMITS, type="M"
+        )
+        return Resources(
+            requests=ResourcesSpec(
+                cpu=self._get_value(
+                    mc_resources_requests, self.MODCONF_RESOURCES_REQUESTS_CPU
+                ),
+                memory=self._get_value(
+                    mc_resources_requests, self.MODCONF_RESOURCES_REQUESTS_MEMORY
+                ),
+            ),
+            limits=ResourcesSpec(
+                cpu=self._get_value(
+                    mc_resources_limits, self.MODCONF_RESOURCES_LIMITS_CPU
+                ),
+                memory=self._get_value(
+                    mc_resources_limits, self.MODCONF_RESOURCES_LIMITS_MEMORY
+                ),
+            ),
+        )
+
     def deserialize(
         self,
         item: Mapping[str, Any],
@@ -116,6 +156,7 @@ class DynamoDBStateAdapter:
                     reconcile_timeout_minutes=self._get_value(
                         modconf, self.MODCONF_TIMEOUT_MINS, type="N"
                     ),
+                    resources=self._build_resources(modconf),
                 ),
             )
 
@@ -164,7 +205,39 @@ class DynamoDBStateAdapter:
                                     state.reconciliation.module_configuration.reconcile_timeout_minutes
                                 )
                             },
-                        }
+                            self.MODCONF_RESOURCES: {
+                                "M": {
+                                    self.MODCONF_RESOURCES_REQUESTS: {
+                                        "M": {
+                                            self.MODCONF_RESOURCES_REQUESTS_CPU: {
+                                                "S": str(
+                                                    state.reconciliation.module_configuration.resources.requests.cpu
+                                                )
+                                            },
+                                            self.MODCONF_RESOURCES_REQUESTS_MEMORY: {
+                                                "S": str(
+                                                    state.reconciliation.module_configuration.resources.requests.memory
+                                                )
+                                            },
+                                        }
+                                    },
+                                    self.MODCONF_RESOURCES_LIMITS: {
+                                        "M": {
+                                            self.MODCONF_RESOURCES_LIMITS_CPU: {
+                                                "S": str(
+                                                    state.reconciliation.module_configuration.resources.limits.cpu
+                                                )
+                                            },
+                                            self.MODCONF_RESOURCES_LIMITS_MEMORY: {
+                                                "S": str(
+                                                    state.reconciliation.module_configuration.resources.limits.memory
+                                                )
+                                            },
+                                        }
+                                    },
+                                }
+                            },
+                        },
                     },
                 }
             },

reconcile/gql_definitions/external_resources/external_resources_modules.py

@@ -17,8 +17,21 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
     Json,
 )
 
+from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
+
 
 DEFINITION = """
+fragment DeployResourcesFields on DeployResources_v1 {
+  requests {
+    cpu
+    memory
+  }
+  limits {
+    cpu
+    memory
+  }
+}
+
 query ExternalResourcesModules {
     modules: external_resources_modules_v1 {
         provision_provider
@@ -31,6 +44,9 @@ query ExternalResourcesModules {
         outputs_secret_sync
         outputs_secret_image
         outputs_secret_version
+        resources {
+          ... DeployResourcesFields
+        }
     }
 }
 """
@@ -53,6 +69,7 @@ class ExternalResourcesModuleV1(ConfiguredBaseModel):
     outputs_secret_sync: bool = Field(..., alias="outputs_secret_sync")
     outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
     outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
+    resources: Optional[DeployResourcesFields] = Field(..., alias="resources")
 
 
 class ExternalResourcesModulesQueryData(ConfiguredBaseModel):

reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -19,6 +19,7 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
 
 from reconcile.gql_definitions.fragments.aws_vpc import AWSVPC
 from reconcile.gql_definitions.fragments.jumphost_common_fields import CommonJumphostFields
+from reconcile.gql_definitions.external_resources.fragments.external_resources_module_overrides import ExternalResourcesModuleOverrides
 from reconcile.gql_definitions.fragments.vault_secret import VaultSecret
 
 
@@ -53,6 +54,29 @@ fragment CommonJumphostFields on ClusterJumpHost_v1 {
   }
 }
 
+fragment DeployResourcesFields on DeployResources_v1 {
+  requests {
+    cpu
+    memory
+  }
+  limits {
+    cpu
+    memory
+  }
+}
+
+fragment ExternalResourcesModuleOverrides on ExternalResourcesModuleOverrides_v1 {
+  module_type
+  image
+  version
+  reconcile_timeout_minutes
+  outputs_secret_image
+  outputs_secret_version
+  resources {
+    ... DeployResourcesFields
+  }
+}
+
 fragment VaultSecret on VaultSecret_v1 {
     path
     field
@@ -111,12 +135,7 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                    module_type
-                    image
-                    version
-                    reconcile_timeout_minutes
-                    outputs_secret_image
-                    outputs_secret_version
+                   ... ExternalResourcesModuleOverrides
                 }
             }
             ... on NamespaceTerraformResourceS3_v1 {
@@ -149,12 +168,7 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                  module_type
-                  image
-                  version
-                  reconcile_timeout_minutes
-                  outputs_secret_image
-                  outputs_secret_version
+                   ... ExternalResourcesModuleOverrides
                 }
             }
             ... on NamespaceTerraformResourceServiceAccount_v1 {
@@ -275,12 +289,7 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                  module_type
-                  image
-                  version
-                  reconcile_timeout_minutes
-                  outputs_secret_image
-                  outputs_secret_version
+                   ... ExternalResourcesModuleOverrides
                 }
             }
             ... on NamespaceTerraformResourceElasticSearch_v1 {
@@ -492,12 +501,7 @@ query ExternalResourcesNamespaces {
                 managed_by_erv2
                 delete
                 module_overrides {
-                  module_type
-                  image
-                  version
-                  reconcile_timeout_minutes
-                  outputs_secret_image
-                  outputs_secret_version
+                   ... ExternalResourcesModuleOverrides
                 }
             }
         }
@@ -576,15 +580,6 @@ class AWSRDSDataClassificationV1(ConfiguredBaseModel):
     loss_impact: Optional[str] = Field(..., alias="loss_impact")
 
 
-class ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
-    module_type: Optional[str] = Field(..., alias="module_type")
-    image: Optional[str] = Field(..., alias="image")
-    version: Optional[str] = Field(..., alias="version")
-    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
-    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
-    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
-
-
 class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -604,7 +599,7 @@ class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     data_classification: Optional[AWSRDSDataClassificationV1] = Field(..., alias="data_classification")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
+    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
 
 
 class AWSS3EventNotificationV1(ConfiguredBaseModel):
@@ -629,15 +624,6 @@ class NamespaceTerraformResourceS3V1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
 
 
-class NamespaceTerraformResourceElastiCacheV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
-    module_type: Optional[str] = Field(..., alias="module_type")
-    image: Optional[str] = Field(..., alias="image")
-    version: Optional[str] = Field(..., alias="version")
-    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
-    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
-    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
-
-
 class NamespaceTerraformResourceElastiCacheV1(NamespaceTerraformResourceAWSV1):
     identifier: str = Field(..., alias="identifier")
     defaults: str = Field(..., alias="defaults")
@@ -648,7 +634,7 @@ class NamespaceTerraformResourceElastiCacheV1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[NamespaceTerraformResourceElastiCacheV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
+    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
 
 
 class ClusterV1(ConfiguredBaseModel):
@@ -784,15 +770,6 @@ class NamespaceTerraformResourceCloudWatchV1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
 
 
-class NamespaceTerraformResourceKMSV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
-    module_type: Optional[str] = Field(..., alias="module_type")
-    image: Optional[str] = Field(..., alias="image")
-    version: Optional[str] = Field(..., alias="version")
-    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
-    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
-    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
-
-
 class NamespaceTerraformResourceKMSV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -802,7 +779,7 @@ class NamespaceTerraformResourceKMSV1(NamespaceTerraformResourceAWSV1):
     annotations: Optional[str] = Field(..., alias="annotations")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[NamespaceTerraformResourceKMSV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
+    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
 
 
 class NamespaceTerraformResourceElasticSearchV1(NamespaceTerraformResourceAWSV1):
@@ -1044,15 +1021,6 @@ class MskSecretParametersV1(ConfiguredBaseModel):
     secret: VaultSecret = Field(..., alias="secret")
 
 
-class NamespaceTerraformResourceMskV1_ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
-    module_type: Optional[str] = Field(..., alias="module_type")
-    image: Optional[str] = Field(..., alias="image")
-    version: Optional[str] = Field(..., alias="version")
-    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
-    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
-    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
-
-
 class NamespaceTerraformResourceMskV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -1062,7 +1030,7 @@ class NamespaceTerraformResourceMskV1(NamespaceTerraformResourceAWSV1):
     users: Optional[list[MskSecretParametersV1]] = Field(..., alias="users")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
-    module_overrides: Optional[NamespaceTerraformResourceMskV1_ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
+    module_overrides: Optional[ExternalResourcesModuleOverrides] = Field(..., alias="module_overrides")
 
 
 class NamespaceTerraformProviderResourceAWSV1(NamespaceExternalResourceV1):

reconcile/gql_definitions/external_resources/external_resources_settings.py

@@ -17,8 +17,21 @@ from pydantic import (  # noqa: F401 # pylint: disable=W0611
     Json,
 )
 
+from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
+
 
 DEFINITION = """
+fragment DeployResourcesFields on DeployResources_v1 {
+  requests {
+    cpu
+    memory
+  }
+  limits {
+    cpu
+    memory
+  }
+}
+
 query ExternalResourcesSettings {
     settings: external_resources_settings_v1 {
         state_dynamodb_account {
@@ -38,6 +51,9 @@ query ExternalResourcesSettings {
         vault_secrets_path
         outputs_secret_image
         outputs_secret_version
+        module_default_resources {
+          ... DeployResourcesFields
+        }
     }
 }
 """
@@ -73,6 +89,7 @@ class ExternalResourcesSettingsV1(ConfiguredBaseModel):
     vault_secrets_path: str = Field(..., alias="vault_secrets_path")
     outputs_secret_image: str = Field(..., alias="outputs_secret_image")
     outputs_secret_version: str = Field(..., alias="outputs_secret_version")
+    module_default_resources: DeployResourcesFields = Field(..., alias="module_default_resources")
 
 
 class ExternalResourcesSettingsQueryData(ConfiguredBaseModel):

reconcile/gql_definitions/external_resources/fragments/external_resources_module_overrides.py

@@ -0,0 +1,36 @@
+"""
+Generated by qenerate plugin=pydantic_v1. DO NOT MODIFY MANUALLY!
+"""
+from collections.abc import Callable  # noqa: F401 # pylint: disable=W0611
+from datetime import datetime  # noqa: F401 # pylint: disable=W0611
+from enum import Enum  # noqa: F401 # pylint: disable=W0611
+from typing import (  # noqa: F401 # pylint: disable=W0611
+    Any,
+    Optional,
+    Union,
+)
+
+from pydantic import (  # noqa: F401 # pylint: disable=W0611
+    BaseModel,
+    Extra,
+    Field,
+    Json,
+)
+
+from reconcile.gql_definitions.fragments.deplopy_resources import DeployResourcesFields
+
+
+class ConfiguredBaseModel(BaseModel):
+    class Config:
+        smart_union=True
+        extra=Extra.forbid
+
+
+class ExternalResourcesModuleOverrides(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+    outputs_secret_image: Optional[str] = Field(..., alias="outputs_secret_image")
+    outputs_secret_version: Optional[str] = Field(..., alias="outputs_secret_version")
+    resources: Optional[DeployResourcesFields] = Field(..., alias="resources")

reconcile/gql_definitions/introspection.json

@@ -33879,6 +33879,22 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "module_default_resources",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "NON_NULL",
+                                "name": null,
+                                "ofType": {
+                                    "kind": "OBJECT",
+                                    "name": "DeployResources_v1",
+                                    "ofType": null
+                                }
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -34080,6 +34096,18 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "resources",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "OBJECT",
+                                "name": "DeployResources_v1",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,
@@ -43321,6 +43349,18 @@
                             },
                             "isDeprecated": false,
                             "deprecationReason": null
+                        },
+                        {
+                            "name": "resources",
+                            "description": null,
+                            "args": [],
+                            "type": {
+                                "kind": "OBJECT",
+                                "name": "DeployResources_v1",
+                                "ofType": null
+                            },
+                            "isDeprecated": false,
+                            "deprecationReason": null
                         }
                     ],
                     "inputFields": null,

tools/cli_commands/erv2.py

@@ -118,13 +118,13 @@ class Erv2Cli:
             self._er_settings, m_inventory, er_inventory, self._secret_reader
         )
         f = factories.get_factory(spec.provision_provider)
-        self._resource = f.create_external_resource(spec)
-        f.validate_external_resource(self._resource)
         self._module_configuration = (
             ExternalResourceModuleConfiguration.resolve_configuration(
                 m_inventory.get_from_spec(spec), spec, self._er_settings
             )
         )
+        self._resource = f.create_external_resource(spec, self._module_configuration)
+        f.validate_external_resource(self._resource, self._module_configuration)
 
     @property
     def input_data(self) -> str: