qontract-reconcile 0.10.1rc1144__py3-none-any.whl → 0.10.1rc1147__py3-none-any.whl

{qontract_reconcile-0.10.1rc1144.dist-info → qontract_reconcile-0.10.1rc1147.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1144
+Version: 0.10.1rc1147
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1144.dist-info → qontract_reconcile-0.10.1rc1147.dist-info}/RECORD

@@ -17,7 +17,7 @@ reconcile/dashdotdb_base.py,sha256=l34QDu1G96_Ctnh7ZXdxXgSeCE93GQMdLAkWxmN6vDA,4
 reconcile/dashdotdb_cso.py,sha256=IkI_KSZuH_kPn0cIQKXitJXiPPFSyHykrOuFy9h9ZpU,3643
 reconcile/dashdotdb_dora.py,sha256=YmfxD02tKUAQQzku2aj2DXv1oKkAr4V_2lrPVtTFGyI,17674
 reconcile/dashdotdb_dvo.py,sha256=lCkZ0iby6HrNQb-3kYb6xrt8wCjVUZYxKzz9SiStfHU,8946
-reconcile/dashdotdb_slo.py,sha256=QKKqLzA2f6zUjQvQ_6U4DObpSOB80C7h-NWXJhzQwME,7812
+reconcile/dashdotdb_slo.py,sha256=LWn0xmMLxpLXls6U4W2R40MqLH6VpBySolapdoK8T8E,8338
 reconcile/database_access_manager.py,sha256=FfyXnYcUdX54BYR_6B9PWFmhT8xdNrPCfoz3Q7q39tg,25646
 reconcile/deadmanssnitch.py,sha256=n-5W-djUgwzpmdDM4eQIZpkkDmHY0vndt-42LJXI4Y8,7491
 reconcile/email_sender.py,sha256=-5L-Ag_jaEYSzYRoMr52KQBRXz1E8yx9GqLbg2X4XFU,3533
@@ -196,13 +196,13 @@ reconcile/external_resources/aws.py,sha256=7W-6d-lXO6JGwaxtO1Uc3Lw0p8csJ1EVgz__O
 reconcile/external_resources/factories.py,sha256=nhdTqf1WEfRfgd5-70KAUJVz0ZvZ19C3Pz7wmotSdrs,4857
 reconcile/external_resources/integration.py,sha256=gBVO5dE8JyZ3xYcYik-MTIp_18oU7_hpYc_oztyfElQ,6753
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
-reconcile/external_resources/manager.py,sha256=cs6QEirz9EaLiuxybZ_1ugUn61vNWlKAC4NKouqpd5I,16148
+reconcile/external_resources/manager.py,sha256=SRrx44silyaKv_P8EWNW9JxpJADYh_CRwBmSE8DO6cs,17671
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
-reconcile/external_resources/metrics.py,sha256=ORZStR2Hg5dYSXpPnM_ZLz48d3VSbN_WY0TcNogvcWk,1003
+reconcile/external_resources/metrics.py,sha256=nMbyonGZEJDD1lYzpQY2eR9TNwvxYC4ZCcpi6wrExcM,1037
 reconcile/external_resources/model.py,sha256=UuQgrnv-SSkvSEQQGeCE2IZkhXjLTCVkP_mw8zBZsIQ,8349
 reconcile/external_resources/reconciler.py,sha256=3KFmkHsN7YAwJUSBpN1Xd_D2zM9Ea5_c2uMGWsfruZo,9707
 reconcile/external_resources/secrets_sync.py,sha256=6n0oDPLjd9Ql0lf6zsr1AZw8A6EEe3yCzl20XodtgkE,16229
-reconcile/external_resources/state.py,sha256=pjxqfjZ-DXmN_9dpfaAwhTtTDr4XrcPlwuQ7S3m25Eo,9326
+reconcile/external_resources/state.py,sha256=UupSa6tl4-73_J6Fhisn-qHal3v3uAUS5s5sk85LGDs,9343
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=XtewM9nfTPLnPSpYebP50GrveYOnhTvKNq3seSvL6u8,8343
 reconcile/glitchtip/reconciler.py,sha256=nUvDv7qG1ly0cA16MmlL6NV71yl1mJYLT2mui7lmi0Y,12402
@@ -871,8 +871,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1144.dist-info/METADATA,sha256=Yvs7ta0zvcAxgyoeqw-ebs-oL94279iPWlq3K_WqE2Q,2213
-qontract_reconcile-0.10.1rc1144.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1144.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1144.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1144.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1147.dist-info/METADATA,sha256=_cQIZE1MJivScvg5up_ZRMuua27rv6_UkjsTzX7WA5U,2213
+qontract_reconcile-0.10.1rc1147.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1147.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1147.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1147.dist-info/RECORD,,

reconcile/dashdotdb_slo.py

@@ -136,6 +136,7 @@ class DashdotdbSLO(DashdotdbBase):
                 template = jinja2.Template(expr)
                 window = slo.slo_parameters.window
                 promquery = template.render({"window": window})
+
                 try:
                     prom_response = self._promget(
                         url=promurl,
@@ -149,9 +150,19 @@ class DashdotdbSLO(DashdotdbBase):
                     # and some prometheus URL are openshift service names. The trick is to run
                     # with `oc port-forward` and update the local hosts file if we need to query those.
                     LOG.error(
-                        f"{self.logmarker} Could not reach prometheus at {promurl}: {error}. Skipping {slo.name}"
+                        f"{self.logmarker} Could not reach prometheus at {promurl}: {error}."
+                        f"Skipping SLOs from SLO doc {slo_document.name}"
                     )
+                    # cannot connect to this prometheus, skip all
                     raise
+                except requests.exceptions.HTTPError as error:
+                    LOG.error(
+                        f"{self.logmarker} Error wile querying {promurl}: {error}."
+                        f"Skipping SLO '{slo.name} from SLO doc {slo_document.name}"
+                    )
+                    # it could be a query issue, keep processing other SLOs from this doc
+                    continue
+
                 prom_result = prom_response["data"]["result"]
                 if not prom_result:
                     continue

reconcile/external_resources/manager.py

@@ -3,6 +3,7 @@ from collections.abc import Iterable
 from datetime import UTC, datetime
 from enum import StrEnum
 
+from pydantic import BaseModel
 from sretoolbox.utils import threaded
 
 from reconcile.external_resources.factories import (
@@ -88,6 +89,61 @@ class ReconcileAction(StrEnum):
     DESTROY_ERROR = "Resource status in ERROR state"
 
 
+class ReconciliationStatus(BaseModel):
+    reconcile_time: int = 0
+    resource_status: ResourceStatus
+
+    @property
+    def has_errors(self) -> bool:
+        return self.resource_status == ResourceStatus.ERROR
+
+    @property
+    def needs_secret_sync(self) -> bool:
+        return self.resource_status == ResourceStatus.PENDING_SECRET_SYNC
+
+    def publish_metrics(self, r: Reconciliation, spec: ExternalResourceSpec) -> None:
+        job_name = ReconciliationK8sJob(reconciliation=r).name()
+
+        metrics.set_gauge(
+            ExternalResourcesResourceStatus(
+                app=spec.namespace["app"]["name"],
+                environment=spec.namespace["environment"]["name"],
+                provision_provider=r.key.provision_provider,
+                provisioner_name=r.key.provisioner_name,
+                provider=r.key.provider,
+                identifier=r.key.identifier,
+                job_name=job_name,
+                status=self.resource_status,
+            ),
+            1,
+        )
+        metrics.set_gauge(
+            ExternalResourcesReconcileTimeGauge(
+                app=spec.namespace["app"]["name"],
+                environment=spec.namespace["environment"]["name"],
+                provision_provider=r.key.provision_provider,
+                provisioner_name=r.key.provisioner_name,
+                provider=r.key.provider,
+                identifier=r.key.identifier,
+                job_name=job_name,
+            ),
+            self.reconcile_time,
+        )
+
+        if self.has_errors:
+            metrics.inc_counter(
+                ExternalResourcesReconcileErrorsCounter(
+                    app=spec.namespace["app"]["name"],
+                    environment=spec.namespace["environment"]["name"],
+                    provision_provider=r.key.provision_provider,
+                    provisioner_name=r.key.provisioner_name,
+                    provider=r.key.provider,
+                    identifier=r.key.identifier,
+                    job_name=job_name,
+                )
+            )
+
+
 class ExternalResourcesManager:
     def __init__(
         self,
@@ -150,16 +206,17 @@ class ExternalResourcesManager:
         state: ExternalResourceState,
     ) -> bool:
         reconcile_action = self._get_reconcile_action(reconciliation, state)
-        reconcile = reconcile_action != ReconcileAction.NOOP
-        if reconcile:
-            logging.info(
-                "Reconciling: Status: [%s], Action: [%s], reason: [%s], key:[%s]",
-                state.resource_status.value,
-                reconciliation.action.value,
-                reconcile_action.value,
-                reconciliation.key,
-            )
-        return reconcile
+        if reconcile_action == ReconcileAction.NOOP:
+            return False
+
+        logging.info(
+            "Reconciling: Status: [%s], Action: [%s], reason: [%s], key:[%s]",
+            state.resource_status.value,
+            reconciliation.action.value,
+            reconcile_action.value,
+            reconciliation.key,
+        )
+        return True
 
     def get_all_reconciliations(self) -> dict[str, set[Reconciliation]]:
         """Returns all reconciliations in a dict. Useful to return all data
@@ -212,24 +269,27 @@ class ExternalResourcesManager:
             to_reconcile.add(r)
         return to_reconcile
 
-    def _update_in_progress_state(
-        self, r: Reconciliation, state: ExternalResourceState, job_name: str
-    ) -> bool:
+    def _get_reconciliation_status(
+        self,
+        r: Reconciliation,
+        state: ExternalResourceState,
+    ) -> ReconciliationStatus:
         """Gets the resource reconciliation state from the Job and updates the
         Resource state accordingly. It also returns if the target outputs secret needs
         to be reconciled.
 
         :param r: Reconciliation object
         :param state: State object
-        :return: True/False if target Secret needs to be reconciled.
+        :return: ReconciliationStatus
         """
-        need_secret_sync = False
+
+        status = ReconciliationStatus(resource_status=state.resource_status)
 
         if state.resource_status not in {
             ResourceStatus.DELETE_IN_PROGRESS,
             ResourceStatus.IN_PROGRESS,
         }:
-            return False
+            return status
 
         logging.info(
             "Reconciliation In progress. Action: %s, Key:%s",
@@ -239,7 +299,8 @@ class ExternalResourcesManager:
 
         # Need to check the reconciliation set in the state, not the desired one
         # as the reconciliation object might be from a previous desired state
-        error = False
+        status.resource_status = state.resource_status
+
         match self.reconciler.get_resource_reconcile_status(state.reconciliation):
             case ReconcileStatus.SUCCESS:
                 logging.info(
@@ -248,22 +309,11 @@ class ExternalResourcesManager:
                     r.key,
                 )
                 if r.action == Action.APPLY:
-                    state.resource_status = ResourceStatus.PENDING_SECRET_SYNC
-                    self.state_mgr.set_external_resource_state(state)
-                    need_secret_sync = True
+                    status.resource_status = ResourceStatus.PENDING_SECRET_SYNC
                 elif r.action == Action.DESTROY:
-                    state.resource_status = ResourceStatus.DELETED
-                    self.state_mgr.del_external_resource_state(r.key)
-
-                metrics.set_gauge(
-                    ExternalResourcesReconcileTimeGauge(
-                        provision_provider=r.key.provision_provider,
-                        provisioner_name=r.key.provisioner_name,
-                        provider=r.key.provider,
-                        identifier=r.key.identifier,
-                        job_name=job_name,
-                    ),
-                    self.reconciler.get_resource_reconcile_duration(r) or 0,
+                    status.resource_status = ResourceStatus.DELETED
+                status.reconcile_time = (
+                    self.reconciler.get_resource_reconcile_duration(r) or 0
                 )
             case ReconcileStatus.ERROR:
                 logging.info(
@@ -271,29 +321,40 @@ class ExternalResourcesManager:
                     r.action.value,
                     r.key,
                 )
-                error = True
+                status.resource_status = ResourceStatus.ERROR
             case ReconcileStatus.NOT_EXISTS:
                 logging.info(
                     "Reconciliation should exist but it doesn't. Marking as ERROR to retrigger: Action:%s, Key:%s",
                     r.action.value,
                     r.key,
                 )
-                error = True
-        if error:
-            state.resource_status = ResourceStatus.ERROR
+                status.resource_status = ResourceStatus.ERROR
+
+        return status
+
+    def _update_resource_state(
+        self,
+        r: Reconciliation,
+        state: ExternalResourceState,
+        status: ReconciliationStatus,
+    ) -> None:
+        # There is no need to update the state
+        # if it's marked in progress
+        if state.resource_status in {
+            ResourceStatus.DELETE_IN_PROGRESS,
+            ResourceStatus.IN_PROGRESS,
+        }:
+            return
+        state.ts = datetime.now(UTC)
+        if status.resource_status == ResourceStatus.DELETED:
+            self.state_mgr.del_external_resource_state(r.key)
+        else:
+            state.resource_status = status.resource_status
             self.state_mgr.set_external_resource_state(state)
-            metrics.inc_counter(
-                ExternalResourcesReconcileErrorsCounter(
-                    provision_provider=r.key.provision_provider,
-                    provisioner_name=r.key.provisioner_name,
-                    provider=r.key.provider,
-                    identifier=r.key.identifier,
-                    job_name=job_name,
-                )
-            )
-        return need_secret_sync
 
-    def _update_state(self, r: Reconciliation, state: ExternalResourceState) -> None:
+    def _set_resource_reconciliation_in_progress(
+        self, r: Reconciliation, state: ExternalResourceState
+    ) -> None:
         state.ts = datetime.now(UTC)
         if r.action == Action.APPLY:
             state.resource_status = ResourceStatus.IN_PROGRESS
@@ -352,26 +413,18 @@ class ExternalResourcesManager:
         to_sync_keys: set[ExternalResourceKey] = set()
         for r in desired_r.union(deleted_r):
             state = self.state_mgr.get_external_resource_state(r.key)
-            job_name = ReconciliationK8sJob(reconciliation=r).name()
-            need_sync = self._update_in_progress_state(r, state, job_name)
-            if need_sync:
+            status = self._get_reconciliation_status(r, state)
+            self._update_resource_state(r, state, status)
+
+            if status.needs_secret_sync:
                 to_sync_keys.add(r.key)
 
             if self._resource_needs_reconciliation(reconciliation=r, state=state):
                 self.reconciler.reconcile_resource(reconciliation=r)
-                self._update_state(r, state)
+                self._set_resource_reconciliation_in_progress(r, state)
 
-            metrics.set_gauge(
-                ExternalResourcesResourceStatus(
-                    provision_provider=r.key.provision_provider,
-                    provisioner_name=r.key.provisioner_name,
-                    provider=r.key.provider,
-                    identifier=r.key.identifier,
-                    job_name=job_name,
-                    status=state.resource_status,
-                ),
-                1,
-            )
+            if spec := self.er_inventory.get(r.key):
+                status.publish_metrics(r, spec)
 
         pending_sync_keys = self.state_mgr.get_keys_by_status(
             ResourceStatus.PENDING_SECRET_SYNC

reconcile/external_resources/metrics.py

@@ -10,6 +10,8 @@ from reconcile.utils.metrics import (
 
 class ExternalResourcesBaseMetric(BaseModel):
     integration = normalize_integration_name(QONTRACT_INTEGRATION)
+    app: str
+    environment: str
     provision_provider: str
     provisioner_name: str
     provider: str

reconcile/external_resources/state.py

@@ -74,7 +74,9 @@ class DynamoDBStateAdapter:
         return item[key][_type]
 
     def deserialize(
-        self, item: Mapping[str, Any], partial_data: bool = False
+        self,
+        item: Mapping[str, Any],
+        partial_data: bool = False,
     ) -> ExternalResourceState:
         _key = self._get_value(item, self.ER_KEY, _type="M")
         key = ExternalResourceKey(