qontract-reconcile 0.10.1rc965__py3-none-any.whl → 0.10.1rc967__py3-none-any.whl

{qontract_reconcile-0.10.1rc965.dist-info → qontract_reconcile-0.10.1rc967.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc965
+Version: 0.10.1rc967
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc965.dist-info → qontract_reconcile-0.10.1rc967.dist-info}/RECORD

@@ -188,10 +188,10 @@ reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG
 reconcile/external_resources/factories.py,sha256=DXgaLxoO87zZ76VOpRpu2GeYGhsbfOnOx5mrzgo4Gf4,4767
 reconcile/external_resources/integration.py,sha256=PJOpz-wUf7NzWvqCDalRWu5OxKjgi5RwiiB6HZfJs0k,5122
 reconcile/external_resources/integration_secrets_sync.py,sha256=cMEZhgCvABAMf-DWF051L6CRnJQdfbsISA_b1xuS940,1670
-reconcile/external_resources/manager.py,sha256=sprARDkMujYreN38sWUDPuZc7hokTL5bM0zEnlPVQKk,14563
-reconcile/external_resources/meta.py,sha256=Z5guBceyaGyAzzA9kVb0-WaNpSli368NVUnWulxtMb4,551
+reconcile/external_resources/manager.py,sha256=5H7anOL7ISIypX2V2Usjviu1vptUQOWoWRXSbyTc69s,14689
+reconcile/external_resources/meta.py,sha256=cMT9OsKcUY26qwEjlQ02EkorvOBNqWj0JVMwfJa3Mg0,634
 reconcile/external_resources/metrics.py,sha256=m2TIOao2N7pD6k45driFbBGVCC_N7ai44m-lLPfa5qk,454
-reconcile/external_resources/model.py,sha256=PZbCLbZ6UGjUzeqcK7RdV4Yh_C7H2FvsfI1GsQGGDxk,7413
+reconcile/external_resources/model.py,sha256=euc7spP3ZRfsn7NTFTv_1Ow6SzvLTskCYTUNWw0XY9Y,7219
 reconcile/external_resources/reconciler.py,sha256=E50X_lnOD0OWYXMzyZld1P6dCFJFYjHGyICWff9bxlc,9323
 reconcile/external_resources/secrets_sync.py,sha256=GsNKhfUAbuER4bgc0p3ZKOrc3vTGNc01HItyuscpMAY,15102
 reconcile/external_resources/state.py,sha256=Yk1YAebiJ7D5J1CFaGI_L-buxdFFx1UrZESVykLJaPM,9629
@@ -278,7 +278,7 @@ reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
 reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=g2KB2wRnb8zF7xCmDJJFmiRdE4z4aYa9HtY3vCBVwMA,2441
-reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=UyOAUY1rROenjTz6y-uSEFjrEwhh-lPsIQPbi6EQLFg,40915
+reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=pXx58C8a2TfClsPqdRmRlQ3ILAX7Irnz-Ank8C8N_gM,40994
 reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=Hw9n_90BPG6Lnt2PT3mHc6p0KEm2CxKxvSGRFc_Dhus,2982
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
@@ -651,7 +651,7 @@ reconcile/utils/environ.py,sha256=psk07d2xyjbUzjOCDdNWgavaNolL_t2sq3sn2gFfY9k,50
 reconcile/utils/exceptions.py,sha256=DwfnWUpVOotpP79RWZ2pycmG6nKCL00RBIeZLYkQPW4,635
 reconcile/utils/expiration.py,sha256=Yv-pBE-UZ6kNkLLgDUdmEUg62uxPEX_RZlA0XAKrs4w,1203
 reconcile/utils/extended_early_exit.py,sha256=QSktrmfw37zSRMNk930tDbQsVeKxaPPPD43e79DGwZw,6754
-reconcile/utils/external_resource_spec.py,sha256=YTUxo6lyF7UrBzsBRd1Zse72jEmWmYQPODTcj1Yxpw0,6899
+reconcile/utils/external_resource_spec.py,sha256=3z3Y2JGY8cO7hPc8JHkdczmrP81D7TKQ4uT128WhE_M,7006
 reconcile/utils/external_resources.py,sha256=T6h172jzz_i9c10tMtoDfp5llRdLlIMkz6OuUZ7tH9Y,7558
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=actOWI2HiNpMIV6nHCzinhRa6b04Y9plWOCcPQa8lNA,1437
@@ -742,8 +742,8 @@ reconcile/utils/internal_groups/client.py,sha256=RL-fNDtMGfpD7pNjCEmrj_0PV3ClCtU
 reconcile/utils/internal_groups/models.py,sha256=y_IqBVqfGqNXiu0VudvBWFrm_-uafVm5KgLG-ca8XAs,2281
 reconcile/utils/jinja2/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/jinja2/extensions.py,sha256=7K-uo6G2eCWa98MHT8fRPYIKCLQB_5D2keqQ_LyAfHM,1293
-reconcile/utils/jinja2/filters.py,sha256=RVVkpf87FllrPUpqk_8KN-r1IsmnS0bpygAVvsvIr5g,4504
-reconcile/utils/jinja2/utils.py,sha256=4KT3r7k5b9rL7l3h3kUzcehjU1eOXYP4KocboVbMqsg,7948
+reconcile/utils/jinja2/filters.py,sha256=tmiaYMhji5fv4B66YtR7zc-mE3wQLyj5I5SeX0WA2l4,4754
+reconcile/utils/jinja2/utils.py,sha256=W_LMB2KiGuwIo04GUNK7jKCpmhKy6wGOH1jgshzXBU0,8004
 reconcile/utils/jobcontroller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/jobcontroller/controller.py,sha256=2V_vm5thFx6adW4bMy9CdHXFesuo6S4lSkEpGxkXSM0,14492
 reconcile/utils/jobcontroller/models.py,sha256=tSRAkUX23iyn4YPsWEicFXwRxw3mXb5B2pDOWmXX8wQ,6350
@@ -847,8 +847,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc965.dist-info/METADATA,sha256=dhDXF0tDhZMesAYNbk_FAaWNPiI-Yigm-PUmjwWxaFE,2262
-qontract_reconcile-0.10.1rc965.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc965.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc965.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc965.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc967.dist-info/METADATA,sha256=U1KHlHKS0EVLd-fnOLrjHk2OZaH3Y5t1BZW8ci9Np9s,2262
+qontract_reconcile-0.10.1rc967.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc967.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc967.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc967.dist-info/RECORD,,

reconcile/external_resources/manager.py

@@ -1,4 +1,3 @@
-import json
 import logging
 from collections.abc import Iterable
 from datetime import UTC, datetime
@@ -43,6 +42,7 @@ from reconcile.utils.external_resource_spec import (
 from reconcile.utils.secret_reader import SecretReaderBase
 
 FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
+FLAG_DELETE_RESOURCE = "delete"
 
 
 def setup_factories(
@@ -157,13 +157,13 @@ class ExternalResourcesManager:
     def _get_desired_objects_reconciliations(self) -> set[Reconciliation]:
         r: set[Reconciliation] = set()
         for key, spec in self.er_inventory.items():
+            if spec.marked_to_delete:
+                continue
             module = self.module_inventory.get_from_spec(spec)
-
             try:
                 resource = self._build_external_resource(spec, self.er_inventory)
             except ExternalResourceValidationError as e:
-                k = ExternalResourceKey.from_spec(spec)
-                self.errors[k] = e
+                self.errors[key] = e
                 continue
 
             reconciliation = Reconciliation(
@@ -179,21 +179,23 @@ class ExternalResourcesManager:
         return r
 
     def _get_deleted_objects_reconciliations(self) -> set[Reconciliation]:
-        desired_keys = set(self.er_inventory.keys())
-        state_resource_keys = self.state_mgr.get_all_resource_keys()
-        deleted_keys = state_resource_keys - desired_keys
-        r: set[Reconciliation] = set()
+        to_reconcile: set[Reconciliation] = set()
+        deleted_keys = (k for k, v in self.er_inventory.items() if v.marked_to_delete)
         for key in deleted_keys:
             state = self.state_mgr.get_external_resource_state(key)
-            reconciliation = Reconciliation(
+            if state.resource_status == ResourceStatus.NOT_EXISTS:
+                logging.debug("Resource has already been removed. key: %s", key)
+                continue
+
+            r = Reconciliation(
                 key=key,
                 resource_hash=state.reconciliation.resource_hash,
                 module_configuration=state.reconciliation.module_configuration,
                 input=state.reconciliation.input,
                 action=Action.DESTROY,
             )
-            r.add(reconciliation)
-        return r
+            to_reconcile.add(r)
+        return to_reconcile
 
     def _update_in_progress_state(
         self, r: Reconciliation, state: ExternalResourceState
@@ -310,8 +312,8 @@ class ExternalResourcesManager:
         return resource
 
     def _serialize_resource_input(self, resource: ExternalResource) -> str:
-        return json.dumps(
-            resource.dict(exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2}})
+        return resource.json(
+            exclude={"data": {FLAG_RESOURCE_MANAGED_BY_ERV2, FLAG_DELETE_RESOURCE}}
         )
 
     def handle_resources(self) -> None:

reconcile/external_resources/meta.py

@@ -11,3 +11,6 @@ SECRET_ANN_PROVIDER = SECRET_ANN_PREFIX + "/provider"
 SECRET_ANN_IDENTIFIER = SECRET_ANN_PREFIX + "/identifier"
 SECRET_UPDATED_AT = SECRET_ANN_PREFIX + "/updated_at"
 SECRET_UPDATED_AT_TIMEFORMAT = "%Y-%m-%dT%H:%M:%SZ"
+
+FLAG_RESOURCE_MANAGED_BY_ERV2 = "managed_by_erv2"
+FLAG_DELETE_RESOURCE = "delete"

reconcile/external_resources/model.py

@@ -1,4 +1,3 @@
-import base64
 import hashlib
 import json
 from abc import (
@@ -88,7 +87,6 @@ class ExternalResourcesInventory(MutableMapping):
         ]
 
         for spec in desired_specs:
-            # self.set(ExternalResourceKey.from_spec(spec), spec)
             self._inventory[ExternalResourceKey.from_spec(spec)] = spec
 
     def __getitem__(self, key: ExternalResourceKey) -> ExternalResourceSpec | None:
@@ -239,6 +237,3 @@ class ExternalResource(BaseModel):
         return hashlib.md5(
             json.dumps(self.data, sort_keys=True).encode("utf-8")
         ).hexdigest()
-
-    def serialize_input(self) -> str:
-        return base64.b64encode(json.dumps(self.dict()).encode()).decode()

reconcile/gql_definitions/external_resources/external_resources_namespaces.py

@@ -108,6 +108,7 @@ query ExternalResourcesNamespaces {
                     loss_impact
                 }
                 managed_by_erv2
+                delete
             }
             ... on NamespaceTerraformResourceS3_v1 {
                 region
@@ -552,6 +553,7 @@ class NamespaceTerraformResourceRDSV1(NamespaceTerraformResourceAWSV1):
     event_notifications: Optional[list[AWSRDSEventNotificationV1]] = Field(..., alias="event_notifications")
     data_classification: Optional[AWSRDSDataClassificationV1] = Field(..., alias="data_classification")
     managed_by_erv2: Optional[bool] = Field(..., alias="managed_by_erv2")
+    delete: Optional[bool] = Field(..., alias="delete")
 
 
 class AWSS3EventNotificationV1(ConfiguredBaseModel):

reconcile/utils/external_resource_spec.py

@@ -92,6 +92,10 @@ class ExternalResourceSpec:
         init=False, compare=False, repr=False, hash=False, default_factory=lambda: {}
     )
 
+    @property
+    def marked_to_delete(self) -> bool:
+        return self.resource.get("delete") or False
+
     @property
     def provider(self) -> str:
         return self.resource["provider"]

reconcile/utils/jinja2/filters.py

@@ -6,6 +6,7 @@ from typing import Any
 from urllib import parse
 
 import jinja2
+import yaml
 
 from reconcile.utils.jsonpath import parse_jsonpath
 
@@ -20,6 +21,15 @@ def json_to_dict(input: str) -> Any:
     return data
 
 
+def yaml_to_dict(input: str) -> Any:
+    """Jinja2 filter to parse YAML strings into dictionaries.
+    :param input: yaml string
+    :return: dict with the parsed inputs contents
+    """
+    data = yaml.safe_load(input)
+    return data
+
+
 def urlescape(string: str, safe: str = "/", encoding: str | None = None) -> str:
     """Jinja2 filter that is a simple wrapper around urllib's URL quoting
     functions that takes a string value and makes it safe for use as URL

reconcile/utils/jinja2/utils.py

@@ -23,6 +23,7 @@ from reconcile.utils.jinja2.filters import (
     matches_jsonpath,
     urlescape,
     urlunescape,
+    yaml_to_dict,
 )
 from reconcile.utils.secret_reader import SecretNotFound, SecretReader, SecretReaderBase
 from reconcile.utils.vault import SecretFieldNotFound
@@ -81,6 +82,7 @@ def compile_jinja2_template(
     )
     jinja_env.filters.update({
         "json_to_dict": json_to_dict,
+        "yaml_to_dict": yaml_to_dict,
         "urlescape": urlescape,
         "urlunescape": urlunescape,
         "eval": eval_filter,