qontract-reconcile 0.10.1rc1023__py3-none-any.whl → 0.10.1rc1025__py3-none-any.whl

{qontract_reconcile-0.10.1rc1023.dist-info → qontract_reconcile-0.10.1rc1025.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1023
+Version: 0.10.1rc1025
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1023.dist-info → qontract_reconcile-0.10.1rc1025.dist-info}/RECORD

@@ -186,7 +186,7 @@ reconcile/dynatrace_token_provider/metrics.py,sha256=xiKkl8fTEBQaXJelGCPNTZhHAWd
 reconcile/dynatrace_token_provider/model.py,sha256=gkpqo5rRRueBXnIMjp4EEHqBUBuU65TRI8zpdb8GJ0A,241
 reconcile/dynatrace_token_provider/ocm.py,sha256=iHMsgbsLs-dlrB9UXmWNDF7E4UDe49JOsLa9rnowKfo,4282
 reconcile/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/endpoints_discovery/integration.py,sha256=OFdSno8C5VVOoXALI5BFxHA3hfIhTZ55lJaMa0jLt0s,12210
+reconcile/endpoints_discovery/integration.py,sha256=dnZ42vrrc3q3-EDO7qE5HVjwHrnpb4_2O_QhFhi8Q-U,12333
 reconcile/endpoints_discovery/merge_request.py,sha256=NktpwMUoQ9TvBzOeHSPC09OqxWmgZBZVy-Eqmsm_vrA,2909
 reconcile/endpoints_discovery/merge_request_manager.py,sha256=oY3lovy5DcUgHNEDzWrbOPabzZtaGMns6YLl5ipZJ0c,6346
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -194,10 +194,10 @@ reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG
 reconcile/external_resources/factories.py,sha256=DXgaLxoO87zZ76VOpRpu2GeYGhsbfOnOx5mrzgo4Gf4,4767
 reconcile/external_resources/integration.py,sha256=y1gJ16woMBC3J9qniMmS5y3lCkAs7V_ETZRUwjKqaO0,6628
 reconcile/external_resources/integration_secrets_sync.py,sha256=cMEZhgCvABAMf-DWF051L6CRnJQdfbsISA_b1xuS940,1670
-reconcile/external_resources/manager.py,sha256=xVwcFAPFG0HyMl_uzj8D6XXdH-Zmcw541DX0IHs57Jc,15278
-reconcile/external_resources/meta.py,sha256=cMT9OsKcUY26qwEjlQ02EkorvOBNqWj0JVMwfJa3Mg0,634
+reconcile/external_resources/manager.py,sha256=Q1l53Q0QHS7dvxCU66_BBpWLpNf6he0FfNzxVKJmVN8,15102
+reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=m2TIOao2N7pD6k45driFbBGVCC_N7ai44m-lLPfa5qk,454
-reconcile/external_resources/model.py,sha256=oXxJkjhV53lwwAuxUCBrjJ8aCJmQdgcKWv68ugJPK4k,7229
+reconcile/external_resources/model.py,sha256=RjGMHgt_J1_HQjfBodFGvdWpYKJupKB4xtWuggSnMtA,8270
 reconcile/external_resources/reconciler.py,sha256=E50X_lnOD0OWYXMzyZld1P6dCFJFYjHGyICWff9bxlc,9323
 reconcile/external_resources/secrets_sync.py,sha256=6n0oDPLjd9Ql0lf6zsr1AZw8A6EEe3yCzl20XodtgkE,16229
 reconcile/external_resources/state.py,sha256=fKU6PLYOZ2ZTaIwvt1BNlNOnIqcewLijOyT3Lgcd1NE,9677
@@ -288,8 +288,8 @@ reconcile/gql_definitions/endpoints_discovery/__init__.py,sha256=47DEQpj8HBSa-_T
 reconcile/gql_definitions/endpoints_discovery/namespaces.py,sha256=FqJ0H7NdsIm5BgVnuJV9wLcj7i667VhCN559tWJ-WsA,3054
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
-reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=g2KB2wRnb8zF7xCmDJJFmiRdE4z4aYa9HtY3vCBVwMA,2441
-reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=pXx58C8a2TfClsPqdRmRlQ3ILAX7Irnz-Ank8C8N_gM,40994
+reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=HFOQjmNbNxk0j5nChxppQeCnJjeDsqibJkPgA7R1zRw,2417
+reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=uNk0HOAH96qdLuDBE6VzjcIPB--PHvOLDitJ2IzPM40,41618
 reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=Hw9n_90BPG6Lnt2PT3mHc6p0KEm2CxKxvSGRFc_Dhus,2982
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
@@ -668,7 +668,7 @@ reconcile/utils/environ.py,sha256=psk07d2xyjbUzjOCDdNWgavaNolL_t2sq3sn2gFfY9k,50
 reconcile/utils/exceptions.py,sha256=DwfnWUpVOotpP79RWZ2pycmG6nKCL00RBIeZLYkQPW4,635
 reconcile/utils/expiration.py,sha256=3JaXH4psksR7z262k7FmdyREjCLqm66OpVMEbcfdWRo,1213
 reconcile/utils/extended_early_exit.py,sha256=QSktrmfw37zSRMNk930tDbQsVeKxaPPPD43e79DGwZw,6754
-reconcile/utils/external_resource_spec.py,sha256=3z3Y2JGY8cO7hPc8JHkdczmrP81D7TKQ4uT128WhE_M,7006
+reconcile/utils/external_resource_spec.py,sha256=bhH_xneFwATdFumTPkiQmcVKYI0gcaWuqV6FpFdf_P0,7006
 reconcile/utils/external_resources.py,sha256=ObBOGBRTsDQ2s9rojfeUfFMoR8ls4Kg-AnpsyF6m7u8,7539
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=JkpbUO10oBTtNHZ1IhjyG6dTOUizc7I5H0vm7NvDVNw,1409
@@ -834,7 +834,7 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=1ZP58LYV6ww3XOLVxgy8NKasMb1jQmp4BNqzTEB0VBE,17723
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=bFTYOymVksNKp_ntG2l9GoA40tODuxeRDU3rO4J-CjA,128875
+tools/qontract_cli.py,sha256=8HbIKtepnmoQteb9Q31iJXFG7_y2o_g6JEkIhXnoncM,128916
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/template_validation.py,sha256=qpKYaTgk0GOPGa2Ct5_5sKdwIHtCAKIBGzsMPuJU5fw,3371
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -865,8 +865,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1023.dist-info/METADATA,sha256=95lGKHZNgD5yHqqGO71hefbNr2QuHpG0K7kW7B1VI9c,2213
-qontract_reconcile-0.10.1rc1023.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1023.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1023.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1023.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1025.dist-info/METADATA,sha256=dUsRqkEgrR_aSPJqnY-4retmHRgWmF0hqlNW9n8MXQA,2213
+qontract_reconcile-0.10.1rc1025.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1025.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1025.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1025.dist-info/RECORD,,

reconcile/endpoints_discovery/integration.py

@@ -221,6 +221,9 @@ class EndpointsDiscoveryIntegration(
         """Compile a list of apps with their endpoints to add, change and delete."""
         apps: dict[str, App] = {}
         for namespace in namespaces:
+            logging.debug(
+                f"Processing namespace {namespace.cluster.name}/{namespace.name}"
+            )
             routes = self.get_routes(oc_map, namespace)
             endpoints_to_add, endpoints_to_change, endpoints_to_delete = (
                 self.get_endpoint_changes(

reconcile/external_resources/manager.py

@@ -41,9 +41,6 @@ from reconcile.utils.external_resource_spec import (
 )
 from reconcile.utils.secret_reader import SecretReaderBase
 
-FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
-FLAG_DELETE_RESOURCE = "delete"
-
 
 def setup_factories(
     settings: ExternalResourcesSettingsV1,
@@ -323,9 +320,7 @@ class ExternalResourcesManager:
         return resource
 
     def _serialize_resource_input(self, resource: ExternalResource) -> str:
-        return resource.json(
-            exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2, FLAG_DELETE_RESOURCE}}
-        )
+        return resource.json()
 
     def handle_resources(self) -> None:
         desired_r = self._get_desired_objects_reconciliations()

reconcile/external_resources/meta.py

@@ -14,3 +14,4 @@ SECRET_UPDATED_AT_TIMEFORMAT = "%Y-%m-%dT%H:%M:%SZ"
 
 FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
 FLAG_DELETE_RESOURCE = "delete"
+MODULE_OVERRIDES = "module_overrides"

reconcile/external_resources/model.py

@@ -13,13 +13,18 @@ from typing import Any
 
 from pydantic import BaseModel
 
+from reconcile.external_resources.meta import (
+    FLAG_DELETE_RESOURCE,
+    FLAG_RESOURCE_MANAGED_BY_ERV2,
+    MODULE_OVERRIDES,
+)
 from reconcile.gql_definitions.external_resources.external_resources_modules import (
     ExternalResourcesModuleV1,
 )
 from reconcile.gql_definitions.external_resources.external_resources_namespaces import (
+    ExternalResourcesModuleOverridesV1,
     NamespaceTerraformProviderResourceAWSV1,
     NamespaceTerraformResourceRDSV1,
-    NamespaceTerraformResourceRoleV1,
     NamespaceV1,
 )
 from reconcile.utils.exceptions import FetchResourceError
@@ -60,30 +65,48 @@ class ExternalResourceKey(BaseModel, frozen=True):
         return f"{self.provision_provider}/{self.provisioner_name}/{self.provider}/{self.identifier}"
 
 
+SUPPORTED_RESOURCE_PROVIDERS = NamespaceTerraformProviderResourceAWSV1
+SUPPORTED_RESOURCE_TYPES = NamespaceTerraformResourceRDSV1
+
+
 class ExternalResourcesInventory(MutableMapping):
     _inventory: dict[ExternalResourceKey, ExternalResourceSpec] = {}
 
+    def _build_external_resource_spec(
+        self,
+        namespace: NamespaceV1,
+        provider: SUPPORTED_RESOURCE_PROVIDERS,
+        resource: SUPPORTED_RESOURCE_TYPES,
+    ) -> ExternalResourceSpec:
+        spec = ExternalResourceSpec(
+            provision_provider=provider.provider,
+            provisioner=provider.provisioner.dict(),
+            resource=resource.dict(
+                exclude={
+                    FLAG_RESOURCE_MANAGED_BY_ERV2,
+                    FLAG_DELETE_RESOURCE,
+                    MODULE_OVERRIDES,
+                }
+            ),
+            namespace=namespace.dict(),
+        )
+        spec.metadata[FLAG_DELETE_RESOURCE] = resource.delete
+        spec.metadata[MODULE_OVERRIDES] = resource.module_overrides
+        return spec
+
     def __init__(self, namespaces: Iterable[NamespaceV1]) -> None:
         desired_providers = [
             (p, ns)
             for ns in namespaces
             for p in ns.external_resources or []
-            if isinstance(p, NamespaceTerraformProviderResourceAWSV1) and p.resources
+            if isinstance(p, SUPPORTED_RESOURCE_PROVIDERS) and p.resources
         ]
 
         desired_specs = [
-            ExternalResourceSpec(
-                provision_provider=p.provider,
-                provisioner=p.provisioner.dict(),
-                resource=r.dict(),
-                namespace=ns.dict(),
-            )
+            self._build_external_resource_spec(ns, p, r)
             for (p, ns) in desired_providers
             for r in p.resources
-            if isinstance(
-                r, NamespaceTerraformResourceRDSV1 | NamespaceTerraformResourceRoleV1
-            )
-            and r.managed_by_erv2
+            if isinstance(r, SUPPORTED_RESOURCE_TYPES) and r.managed_by_erv2
         ]
 
         for spec in desired_specs:
@@ -181,14 +204,22 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     def resolve_configuration(
         module: ExternalResourcesModuleV1, spec: ExternalResourceSpec
     ) -> "ExternalResourceModuleConfiguration":
-        # TODO: Modify resource schemas to include this attributes
-        data = {
-            "image": module.image,
-            "version": module.default_version,
-            "reconcile_drift_interval_minutes": module.reconcile_drift_interval_minutes,
-            "reconcile_timeout_minutes": module.reconcile_timeout_minutes,
-        }
-        return ExternalResourceModuleConfiguration.parse_obj(data)
+        module_overrides = spec.metadata.get(
+            "module_overrides"
+        ) or ExternalResourcesModuleOverridesV1(
+            module_type=None,
+            image=None,
+            version=None,
+            reconcile_timeout_minutes=None,
+        )
+
+        return ExternalResourceModuleConfiguration(
+            image=module_overrides.image or module.image,
+            version=module_overrides.version or module.version,
+            reconcile_drift_interval_minutes=module.reconcile_drift_interval_minutes,
+            reconcile_timeout_minutes=module_overrides.reconcile_timeout_minutes
+            or module.reconcile_timeout_minutes,
+        )
 
 
 class Reconciliation(BaseModel, frozen=True):

reconcile/gql_definitions/external_resources/external_resources_modules.py

@@ -25,7 +25,7 @@ query ExternalResourcesModules {
         provider
         module_type
         image
-        default_version
+        version
         reconcile_drift_interval_minutes
         reconcile_timeout_minutes
         outputs_secret_sync
@@ -45,9 +45,9 @@ class ExternalResourcesModuleV1(ConfiguredBaseModel):
     provider: str = Field(..., alias="provider")
     module_type: str = Field(..., alias="module_type")
     image: str = Field(..., alias="image")
-    default_version: str = Field(..., alias="default_version")
-    reconcile_drift_interval_minutes: str = Field(..., alias="reconcile_drift_interval_minutes")
-    reconcile_timeout_minutes: str = Field(..., alias="reconcile_timeout_minutes")
+    version: str = Field(..., alias="version")
+    reconcile_drift_interval_minutes: int = Field(..., alias="reconcile_drift_interval_minutes")
+    reconcile_timeout_minutes: int = Field(..., alias="reconcile_timeout_minutes")
     outputs_secret_sync: bool = Field(..., alias="outputs_secret_sync")
 
 

reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -109,6 +109,12 @@ query ExternalResourcesNamespaces {
                 }
                 managed_by_erv2
                 delete
+                module_overrides {
+                    module_type
+                    image
+                    version
+                    reconcile_timeout_minutes
+                }
             }
             ... on NamespaceTerraformResourceS3_v1 {
                 region
@@ -535,6 +541,13 @@ class AWSRDSDataClassificationV1(ConfiguredBaseModel):
     loss_impact: Optional[str] = Field(..., alias="loss_impact")
 
 
+class ExternalResourcesModuleOverridesV1(ConfiguredBaseModel):
+    module_type: Optional[str] = Field(..., alias="module_type")
+    image: Optional[str] = Field(..., alias="image")
+    version: Optional[str] = Field(..., alias="version")
+    reconcile_timeout_minutes: Optional[int] = Field(..., alias="reconcile_timeout_minutes")
+
+
 class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     region: Optional[str] = Field(..., alias="region")
     identifier: str = Field(..., alias="identifier")
@@ -554,6 +567,7 @@ class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     data_classification: Optional[AWSRDSDataClassificationV1] = Field(..., alias="data_classification")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
     delete: Optional[bool] = Field(..., alias="delete")
+    module_overrides: Optional[ExternalResourcesModuleOverridesV1] = Field(..., alias="module_overrides")
 
 
 class AWSS3EventNotificationV1(ConfiguredBaseModel):

reconcile/utils/external_resource_spec.py

@@ -88,13 +88,13 @@ class ExternalResourceSpec:
     secret: Mapping[str, str] = field(init=False, default_factory=lambda: {})
     # Metadata is used for processing data that shuold not be included in the secret data
     # e.g: ERV2 adds a updated_at attribute that acts as optimistic lock.
-    metadata: MutableMapping[str, str] = field(
+    metadata: MutableMapping[str, Any] = field(
         init=False, compare=False, repr=False, hash=False, default_factory=lambda: {}
     )
 
     @property
     def marked_to_delete(self) -> bool:
-        return self.resource.get("delete") or False
+        return self.metadata.get("delete") or False
 
     @property
     def provider(self) -> str:

tools/qontract_cli.py

@@ -60,9 +60,9 @@ from reconcile.external_resources.integration import (
     get_aws_api,
 )
 from reconcile.external_resources.manager import (
-    FLAG_RESOURCE_MANAGED_BY_ERV2,
     setup_factories,
 )
+from reconcile.external_resources.meta import FLAG_RESOURCE_MANAGED_BY_ERV2
 from reconcile.external_resources.model import (
     ExternalResourceKey,
     ExternalResourcesInventory,