qontract-reconcile 0.10.2.dev19__py3-none-any.whl → 0.10.2.dev21__py3-none-any.whl

{qontract_reconcile-0.10.2.dev19.dist-info → qontract_reconcile-0.10.2.dev21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev19
+Version: 0.10.2.dev21
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev19.dist-info → qontract_reconcile-0.10.2.dev21.dist-info}/RECORD

@@ -197,12 +197,12 @@ reconcile/endpoints_discovery/merge_request_manager.py,sha256=wUMsumxv8RnWaRatta
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/external_resources/aws.py,sha256=NSaOeHqFEcMaMxNjJwuQZosolgsJ8XRVvwkEEBj9vrw,7730
 reconcile/external_resources/factories.py,sha256=TyJMaijDfPIFYks9i6dhKN7nSR1BoCkoBs1iPExKpcE,5493
-reconcile/external_resources/integration.py,sha256=gBVO5dE8JyZ3xYcYik-MTIp_18oU7_hpYc_oztyfElQ,6753
+reconcile/external_resources/integration.py,sha256=JF38M7R0Z4ADUTx57TZqSZH9k_xpPlbAxQAcGyIISuM,6925
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
-reconcile/external_resources/manager.py,sha256=eVaaGCaKDkc897xt5cA5-B4yYuS9VWR-Z7Uom0uSsG0,15971
+reconcile/external_resources/manager.py,sha256=q7Ezp-g3MhpH_t8zpNMH5wCzHvh-nERpHzvBPA1G1ng,17031
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=KiBjMUaN_z0cSkF_7Ar_a8RiuiwVqjyMcVdISlxhzXE,3898
-reconcile/external_resources/model.py,sha256=YJylbAhetN9szpLUFd9jFqxCRMvSWXVxSC9OMQNV-wg,11316
+reconcile/external_resources/model.py,sha256=KdB3eYlopWOLQmvL1aICjm-kAPIlY2N_zSikcCFBQpk,11751
 reconcile/external_resources/reconciler.py,sha256=K9QvbQCIOCuOHnPIxQE_P_jFtrkF3dGo8d_cCCh08Ys,8973
 reconcile/external_resources/secrets_sync.py,sha256=50fK4fzgSz-K8uy5_DQQWA_ju_rTDYAC2HRymgfY7TA,16344
 reconcile/external_resources/state.py,sha256=ye8yjMoCtTHSRhDH7skFLDIHIuYTjisWYCTJrwnmbEw,9565
@@ -657,7 +657,7 @@ reconcile/utils/clusterhealth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5N
 reconcile/utils/clusterhealth/providerbase.py,sha256=DXomGYogckBLqWtXn0PXU0hWYxB6K0F7ernldrkHhVY,1140
 reconcile/utils/clusterhealth/telemeter.py,sha256=PllSLsJXvGNatmTF4mxCNPVbDrpr_MPk0m5pWj-LT6g,1534
 reconcile/utils/dynatrace/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/utils/dynatrace/client.py,sha256=H8EjqmZlB1a2ionAjV8_R1ozs9lWbmPCYKLe0J8kZAs,2838
+reconcile/utils/dynatrace/client.py,sha256=RUk6KH-3CJyfJ1jolrdGQR4Hhz-tIWWJo9dsZ1IgJVw,3736
 reconcile/utils/glitchtip/__init__.py,sha256=FT6iBhGqoe7KExFdbgL8AYUb64iW_4snF5__Dcl7yt0,258
 reconcile/utils/glitchtip/client.py,sha256=ovh4tx-ajlihjvcq6nyY4chulbuMJYvzDPv9j9CuAKM,7867
 reconcile/utils/glitchtip/models.py,sha256=AJuGq4_A6G_T7asBKIw69-fOZLmT8HFrTKBEys7Tp00,6481
@@ -766,7 +766,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev19.dist-info/METADATA,sha256=wb_IQ6QESeh7TYBGvXCUJde_kGmYx8m3jIPVO7MsETA,24665
-qontract_reconcile-0.10.2.dev19.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev19.dist-info/entry_points.txt,sha256=JniHZPadNOILPyfSl0LF2YSp3Db7K2_W2CN7i9f3Gos,540
-qontract_reconcile-0.10.2.dev19.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev21.dist-info/METADATA,sha256=odjOqT6xlL94VSczTzQ6DNXDbhrppOG_UhaZk9wNnbg,24665
+qontract_reconcile-0.10.2.dev21.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev21.dist-info/entry_points.txt,sha256=JniHZPadNOILPyfSl0LF2YSp3Db7K2_W2CN7i9f3Gos,540
+qontract_reconcile-0.10.2.dev21.dist-info/RECORD,,

reconcile/external_resources/integration.py

@@ -3,6 +3,7 @@ from collections.abc import Callable
 from typing import Any
 
 from reconcile.external_resources.manager import (
+    ExternalResourceDryRunsValidator,
     ExternalResourcesInventory,
     ExternalResourcesManager,
     setup_factories,
@@ -89,6 +90,10 @@ def create_er_manager(
     m_inventory = load_module_inventory(get_modules())
     namespaces = [ns for ns in get_namespaces() if ns.external_resources]
     er_inventory = ExternalResourcesInventory(namespaces)
+    state_manager = ExternalResourcesStateDynamoDB(
+        aws_api=aws_api,
+        table_name=er_settings.state_dynamodb_table,
+    )
 
     if not workers_cluster:
         workers_cluster = er_settings.workers_cluster.name
@@ -104,10 +109,7 @@ def create_er_manager(
         ),
         er_inventory=er_inventory,
         module_inventory=m_inventory,
-        state_manager=ExternalResourcesStateDynamoDB(
-            aws_api=aws_api,
-            table_name=er_settings.state_dynamodb_table,
-        ),
+        state_manager=state_manager,
         reconciler=K8sExternalResourcesReconciler(
             controller=build_job_controller(
                 integration=QONTRACT_INTEGRATION,
@@ -128,6 +130,9 @@ def create_er_manager(
             thread_pool_size=thread_pool_size,
             dry_run=dry_run,
         ),
+        dry_runs_validator=ExternalResourceDryRunsValidator(
+            state_manager, er_inventory
+        ),
     )
 
 

reconcile/external_resources/manager.py

@@ -1,4 +1,5 @@
 import logging
+from collections import Counter
 from collections.abc import Iterable
 from datetime import UTC, datetime
 
@@ -19,6 +20,7 @@ from reconcile.external_resources.model import (
     ExternalResourceKey,
     ExternalResourceModuleConfiguration,
     ExternalResourceOrphanedResourcesError,
+    ExternalResourceOutputResourceNameDuplications,
     ExternalResourcesInventory,
     ExternalResourceValidationError,
     ModuleInventory,
@@ -73,6 +75,41 @@ def setup_factories(
     return of
 
 
+class ExternalResourceDryRunsValidator:
+    def __init__(
+        self,
+        state_manager: ExternalResourcesStateDynamoDB,
+        er_inventory: ExternalResourcesInventory,
+    ):
+        self.state_mgr = state_manager
+        self.er_inventory = er_inventory
+
+    def _check_output_resource_name_duplications(
+        self,
+    ) -> None:
+        specs = Counter(
+            (
+                spec.cluster_name,
+                spec.namespace_name,
+                spec.output_resource_name,
+            )
+            for spec in self.er_inventory.values()
+        )
+        if duplicates := [key for key, count in specs.items() if count > 1]:
+            raise ExternalResourceOutputResourceNameDuplications(duplicates)
+
+    def _check_orphaned_objects(self) -> None:
+        state_keys = self.state_mgr.get_all_resource_keys()
+        inventory_keys = set(self.er_inventory.keys())
+        orphans = state_keys - inventory_keys
+        if len(orphans) > 0:
+            raise ExternalResourceOrphanedResourcesError(orphans)
+
+    def validate(self) -> None:
+        self._check_orphaned_objects()
+        self._check_output_resource_name_duplications()
+
+
 class ExternalResourcesManager:
     def __init__(
         self,
@@ -84,6 +121,7 @@ class ExternalResourcesManager:
         er_inventory: ExternalResourcesInventory,
         factories: ObjectFactory[ExternalResourceFactory],
         secrets_reconciler: InClusterSecretsReconciler,
+        dry_runs_validator: ExternalResourceDryRunsValidator,
         thread_pool_size: int,
     ) -> None:
         self.state_mgr = state_manager
@@ -96,6 +134,7 @@ class ExternalResourcesManager:
         self.secrets_reconciler = secrets_reconciler
         self.errors: dict[ExternalResourceKey, ExternalResourceValidationError] = {}
         self.thread_pool_size = thread_pool_size
+        self.dry_runs_validator = dry_runs_validator
 
     def _get_reconcile_action(
         self, reconciliation: Reconciliation, state: ExternalResourceState
@@ -199,13 +238,6 @@ class ExternalResourcesManager:
             to_reconcile.add(r)
         return to_reconcile
 
-    def _check_orphaned_objects(self) -> None:
-        state_keys = self.state_mgr.get_all_resource_keys()
-        inventory_keys = set(self.er_inventory.keys())
-        orphans = state_keys - inventory_keys
-        if len(orphans) > 0:
-            raise ExternalResourceOrphanedResourcesError(orphans)
-
     def _get_reconciliation_status(
         self,
         r: Reconciliation,
@@ -374,7 +406,7 @@ class ExternalResourcesManager:
             self._sync_secrets(to_sync_keys=to_sync_keys | pending_sync_keys)
 
     def handle_dry_run_resources(self) -> None:
-        self._check_orphaned_objects()
+        self.dry_runs_validator.validate()
         desired_r = self._get_desired_objects_reconciliations()
         deleted_r = self._get_deleted_objects_reconciliations()
         reconciliations = desired_r.union(deleted_r)

reconcile/external_resources/model.py

@@ -45,6 +45,17 @@ class ExternalResourceOrphanedResourcesError(Exception):
         super().__init__("".join(msg))
 
 
+class ExternalResourceOutputResourceNameDuplications(Exception):
+    def __init__(self, duplicates: Iterable[tuple[str, str, str]]) -> None:
+        msg = [
+            "There are output_resource_name attribute duplications. ",
+            "output_resource_name must be unique within a cluster/namespace.\n"
+            "Duplications:\n",
+            "\n".join(map(str, duplicates)),
+        ]
+        super().__init__("".join(msg))
+
+
 class ExternalResourceValidationError(Exception):
     errors: list[str] = []
 

reconcile/utils/dynatrace/client.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 
 from collections.abc import Iterable
+from datetime import datetime
+from unittest.mock import patch
 
 from dynatrace import Dynatrace
 from dynatrace.environment_v2.tokens_api import ApiTokenUpdate
@@ -33,11 +35,32 @@ class DynatraceAPIToken(BaseModel):
     scopes: list[str]
 
 
+# TODO: Remove once APPSRE-11428 is resolved #######
+ISO_8601 = "%Y-%m-%dT%H:%M:%S.%fZ"
+FIXED_ISO_8601 = "%Y-%m-%dT%H:%M:%SZ"
+
+
+def custom_iso8601_to_datetime(timestamp: str | None) -> datetime | None:
+    if isinstance(timestamp, str):
+        try:
+            return datetime.strptime(timestamp, ISO_8601)
+        except ValueError:
+            return datetime.strptime(timestamp, FIXED_ISO_8601)
+    return timestamp
+
+
+################################################
+
+
 class DynatraceClient:
     def __init__(self, environment_url: str, api: Dynatrace) -> None:
         self._environment_url = environment_url
         self._api = api
 
+    @patch(
+        "dynatrace.environment_v2.tokens_api.iso8601_to_datetime",
+        custom_iso8601_to_datetime,
+    )
     def create_api_token(
         self, name: str, scopes: Iterable[str]
     ) -> DynatraceAPITokenCreated:
@@ -49,6 +72,10 @@ class DynatraceClient:
             ) from e
         return DynatraceAPITokenCreated(token=token.token, id=token.id)
 
+    @patch(
+        "dynatrace.environment_v2.tokens_api.iso8601_to_datetime",
+        custom_iso8601_to_datetime,
+    )
     def get_token_ids_map_for_name_prefix(self, prefix: str) -> dict[str, str]:
         try:
             dt_tokens = self._api.tokens.list()
@@ -60,6 +87,10 @@ class DynatraceClient:
             token.id: token.name for token in dt_tokens if token.name.startswith(prefix)
         }
 
+    @patch(
+        "dynatrace.environment_v2.tokens_api.iso8601_to_datetime",
+        custom_iso8601_to_datetime,
+    )
     def get_token_by_id(self, token_id: str) -> DynatraceAPIToken:
         try:
             token = self._api.tokens.get(token_id=token_id)