qontract-reconcile 0.10.1rc1090__py3-none-any.whl → 0.10.1rc1091__py3-none-any.whl

{qontract_reconcile-0.10.1rc1090.dist-info → qontract_reconcile-0.10.1rc1091.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1090
+Version: 0.10.1rc1091
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1090.dist-info → qontract_reconcile-0.10.1rc1091.dist-info}/RECORD

@@ -196,13 +196,13 @@ reconcile/external_resources/aws.py,sha256=7W-6d-lXO6JGwaxtO1Uc3Lw0p8csJ1EVgz__O
 reconcile/external_resources/factories.py,sha256=nhdTqf1WEfRfgd5-70KAUJVz0ZvZ19C3Pz7wmotSdrs,4857
 reconcile/external_resources/integration.py,sha256=y1gJ16woMBC3J9qniMmS5y3lCkAs7V_ETZRUwjKqaO0,6628
 reconcile/external_resources/integration_secrets_sync.py,sha256=cMEZhgCvABAMf-DWF051L6CRnJQdfbsISA_b1xuS940,1670
-reconcile/external_resources/manager.py,sha256=FqOkwiWY6X8qlwE7yrwjiVxKKmZ3k0TdafQnqfihXF8,15107
+reconcile/external_resources/manager.py,sha256=AvBYpC4p2BaGrxYEV1ut3V2FtJfs1h2z0Vk8Px6dCaQ,15641
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
-reconcile/external_resources/metrics.py,sha256=m2TIOao2N7pD6k45driFbBGVCC_N7ai44m-lLPfa5qk,454
+reconcile/external_resources/metrics.py,sha256=Ihhe39BX0h2w7eLvvdijJuCXjj-f7w1E3MfDL0YiGNU,668
 reconcile/external_resources/model.py,sha256=UuQgrnv-SSkvSEQQGeCE2IZkhXjLTCVkP_mw8zBZsIQ,8349
-reconcile/external_resources/reconciler.py,sha256=E50X_lnOD0OWYXMzyZld1P6dCFJFYjHGyICWff9bxlc,9323
+reconcile/external_resources/reconciler.py,sha256=3KFmkHsN7YAwJUSBpN1Xd_D2zM9Ea5_c2uMGWsfruZo,9707
 reconcile/external_resources/secrets_sync.py,sha256=6n0oDPLjd9Ql0lf6zsr1AZw8A6EEe3yCzl20XodtgkE,16229
-reconcile/external_resources/state.py,sha256=fKU6PLYOZ2ZTaIwvt1BNlNOnIqcewLijOyT3Lgcd1NE,9677
+reconcile/external_resources/state.py,sha256=pjxqfjZ-DXmN_9dpfaAwhTtTDr4XrcPlwuQ7S3m25Eo,9326
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=XtewM9nfTPLnPSpYebP50GrveYOnhTvKNq3seSvL6u8,8343
 reconcile/glitchtip/reconciler.py,sha256=nUvDv7qG1ly0cA16MmlL6NV71yl1mJYLT2mui7lmi0Y,12402
@@ -767,7 +767,7 @@ reconcile/utils/jinja2/extensions.py,sha256=7K-uo6G2eCWa98MHT8fRPYIKCLQB_5D2keqQ
 reconcile/utils/jinja2/filters.py,sha256=JfO_14APySBPidsMvHXG-8dULNPddZCE15Umjk_aSBk,4830
 reconcile/utils/jinja2/utils.py,sha256=adxVFY4WvBGIToEEr8KwqLzp6uDxSipTBzZWwnRqbNQ,8700
 reconcile/utils/jobcontroller/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/utils/jobcontroller/controller.py,sha256=n9twSH_xZtp5vknM3qByp0eWCn9ED4U1a6yOFBS-GTg,14488
+reconcile/utils/jobcontroller/controller.py,sha256=Vh08lZuCSIIceWGSDhBB00iFwTI9eeKZW1sfHlkAvSo,15373
 reconcile/utils/jobcontroller/models.py,sha256=x9YIvWfYOOvXNKToFVx1H7qDrZb0Sa1KI_4Y0gl7rMM,6336
 reconcile/utils/membershipsources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/membershipsources/app_interface_resolver.py,sha256=MqDFvK3aXhmmMuMiIygC-onFVrrIopKHriaYJQ5jnuY,1988
@@ -870,8 +870,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1090.dist-info/METADATA,sha256=WP2Qi20ing6IiFnzyowq9CoR3Ds7qNU-ma6oiCJQspQ,2213
-qontract_reconcile-0.10.1rc1090.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1090.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1090.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1090.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1091.dist-info/METADATA,sha256=CLxB_EMfDQaVjFpxvKcZYESvRdIAyVLc9S3hlWpiDYM,2213
+qontract_reconcile-0.10.1rc1091.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1091.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1091.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1091.dist-info/RECORD,,

reconcile/external_resources/manager.py

@@ -13,7 +13,10 @@ from reconcile.external_resources.factories import (
     TerraformModuleProvisionDataFactory,
     setup_aws_resource_factories,
 )
-from reconcile.external_resources.metrics import ExternalResourcesReconcileErrorsGauge
+from reconcile.external_resources.metrics import (
+    ExternalResourcesReconcileErrorsCounter,
+    ExternalResourcesReconcileTimeGauge,
+)
 from reconcile.external_resources.model import (
     Action,
     ExternalResource,
@@ -24,7 +27,10 @@ from reconcile.external_resources.model import (
     ModuleInventory,
     Reconciliation,
 )
-from reconcile.external_resources.reconciler import ExternalResourcesReconciler
+from reconcile.external_resources.reconciler import (
+    ExternalResourcesReconciler,
+    ReconciliationK8sJob,
+)
 from reconcile.external_resources.secrets_sync import InClusterSecretsReconciler
 from reconcile.external_resources.state import (
     ExternalResourcesStateDynamoDB,
@@ -232,6 +238,7 @@ class ExternalResourcesManager:
 
         # Need to check the reconciliation set in the state, not the desired one
         # as the reconciliation object might be from a previous desired state
+        job_name = ReconciliationK8sJob(reconciliation=r).name()
         error = False
         match self.reconciler.get_resource_reconcile_status(state.reconciliation):
             case ReconcileStatus.SUCCESS:
@@ -242,12 +249,22 @@ class ExternalResourcesManager:
                 )
                 if r.action == Action.APPLY:
                     state.resource_status = ResourceStatus.PENDING_SECRET_SYNC
-                    state.reconciliation_errors = 0
                     self.state_mgr.set_external_resource_state(state)
                     need_secret_sync = True
                 elif r.action == Action.DESTROY:
                     state.resource_status = ResourceStatus.DELETED
                     self.state_mgr.del_external_resource_state(r.key)
+
+                metrics.set_gauge(
+                    ExternalResourcesReconcileTimeGauge(
+                        provision_provider=r.key.provision_provider,
+                        provisioner_name=r.key.provisioner_name,
+                        provider=r.key.provider,
+                        identifier=r.key.identifier,
+                        job_name=job_name,
+                    ),
+                    self.reconciler.get_resource_reconcile_duration(r) or 0,
+                )
             case ReconcileStatus.ERROR:
                 logging.info(
                     "Reconciliation ended with ERROR: Action:%s, Key:%s",
@@ -264,8 +281,16 @@ class ExternalResourcesManager:
                 error = True
         if error:
             state.resource_status = ResourceStatus.ERROR
-            state.reconciliation_errors += 1
             self.state_mgr.set_external_resource_state(state)
+            metrics.inc_counter(
+                ExternalResourcesReconcileErrorsCounter(
+                    provision_provider=r.key.provision_provider,
+                    provisioner_name=r.key.provisioner_name,
+                    provider=r.key.provider,
+                    identifier=r.key.identifier,
+                    job_name=job_name,
+                )
+            )
 
         return need_secret_sync
 
@@ -332,16 +357,6 @@ class ExternalResourcesManager:
             if need_sync:
                 to_sync_keys.add(r.key)
 
-            metrics.set_gauge(
-                ExternalResourcesReconcileErrorsGauge(
-                    provision_provider=r.key.provision_provider,
-                    provisioner_name=r.key.provisioner_name,
-                    provider=r.key.provider,
-                    identifier=r.key.identifier,
-                ),
-                float(state.reconciliation_errors),
-            )
-
             if self._resource_needs_reconciliation(reconciliation=r, state=state):
                 self.reconciler.reconcile_resource(reconciliation=r)
                 self._update_state(r, state)

reconcile/external_resources/metrics.py

@@ -1,20 +1,26 @@
 from pydantic import BaseModel
 
-from reconcile.utils.metrics import (
-    GaugeMetric,
-)
+from reconcile.utils.metrics import CounterMetric, GaugeMetric
 
 
 class ExternalResourcesBaseMetric(BaseModel):
     integration = "external_resources"
-
-
-class ExternalResourcesReconcileErrorsGauge(ExternalResourcesBaseMetric, GaugeMetric):
     provision_provider: str
     provisioner_name: str
     provider: str
     identifier: str
+    job_name: str
+
+
+class ExternalResourcesReconcileErrorsCounter(
+    ExternalResourcesBaseMetric, CounterMetric
+):
+    @classmethod
+    def name(cls) -> str:
+        return "external_resources_reconcile_errors"
+
 
+class ExternalResourcesReconcileTimeGauge(ExternalResourcesBaseMetric, GaugeMetric):
     @classmethod
     def name(cls) -> str:
-        return "external_resources_reconcile_status"
+        return "external_resources_reconcile_time"

reconcile/external_resources/reconciler.py

@@ -38,6 +38,11 @@ class ExternalResourcesReconciler(ABC):
         reconciliation: Reconciliation,
     ) -> ReconcileStatus: ...
 
+    @abstractmethod
+    def get_resource_reconcile_duration(
+        self, reconciliation: Reconciliation
+    ) -> int | None: ...
+
     @abstractmethod
     def reconcile_resource(self, reconciliation: Reconciliation) -> None: ...
 
@@ -199,6 +204,12 @@ class K8sExternalResourcesReconciler(ExternalResourcesReconciler):
         job_name = ReconciliationK8sJob(reconciliation=reconciliation).name()
         return ReconcileStatus(self.controller.get_job_status(job_name))
 
+    def get_resource_reconcile_duration(
+        self, reconciliation: Reconciliation
+    ) -> int | None:
+        job_name = ReconciliationK8sJob(reconciliation=reconciliation).name()
+        return self.controller.get_success_job_duration(job_name)
+
     def reconcile_resource(self, reconciliation: Reconciliation) -> None:
         concurrency_policy = (
             JobConcurrencyPolicy.REPLACE_FAILED | JobConcurrencyPolicy.REPLACE_FINISHED

reconcile/external_resources/state.py

@@ -44,7 +44,6 @@ class ExternalResourceState(BaseModel):
     ts: datetime
     resource_status: ResourceStatus
     reconciliation: Reconciliation
-    reconciliation_errors: int = 0
 
 
 class DynamoDBStateAdapter:
@@ -53,7 +52,6 @@ class DynamoDBStateAdapter:
 
     RESOURCE_STATUS = "resource_status"
     TIMESTAMP = "time_stamp"
-    RECONCILIATION_ERRORS = "reconciliation_errors"
 
     ER_KEY = "external_resource_key"
     ER_KEY_PROVISION_PROVIDER = "provision_provider"
@@ -120,9 +118,6 @@ class DynamoDBStateAdapter:
             ts=self._get_value(item, self.TIMESTAMP),
             resource_status=self._get_value(item, self.RESOURCE_STATUS),
             reconciliation=r,
-            reconciliation_errors=int(
-                self._get_value(item, self.RECONCILIATION_ERRORS, _type="N")
-            ),
         )
 
     def serialize(self, state: ExternalResourceState) -> dict[str, Any]:
@@ -130,7 +125,6 @@ class DynamoDBStateAdapter:
             self.ER_KEY_HASH: {"S": state.key.hash()},
             self.TIMESTAMP: {"S": state.ts.isoformat()},
             self.RESOURCE_STATUS: {"S": state.resource_status.value},
-            self.RECONCILIATION_ERRORS: {"N": str(state.reconciliation_errors)},
             self.ER_KEY: {
                 "M": {
                     self.ER_KEY_PROVISION_PROVIDER: {"S": state.key.provision_provider},
@@ -176,7 +170,6 @@ class ExternalResourcesStateDynamoDB:
         DynamoDBStateAdapter.ER_KEY,
         DynamoDBStateAdapter.TIMESTAMP,
         DynamoDBStateAdapter.RESOURCE_STATUS,
-        DynamoDBStateAdapter.RECONCILIATION_ERRORS,
         f"{DynamoDBStateAdapter.RECONC}.{DynamoDBStateAdapter.RECONC_RESOURCE_HASH}",
     ])
 

reconcile/utils/jobcontroller/controller.py

@@ -1,5 +1,6 @@
 import logging
 import time
+from datetime import datetime
 from typing import Protocol, TextIO
 
 from kubernetes.client import (  # type: ignore[attr-defined]
@@ -143,6 +144,25 @@ class K8sJobController:
             return JobStatus.ERROR
         return JobStatus.IN_PROGRESS
 
+    def get_success_job_duration(self, job_name: str) -> int | None:
+        """
+        Returns the number of seconds the job took to complete.
+        * If a job is not completed with success, returns None
+        """
+        if self.get_job_status(job_name) != JobStatus.SUCCESS:
+            return None
+        job_resource = self.cache.get(job_name)
+        if job_resource is None:
+            return None
+        status = job_resource.body.get("status") or {}
+        start_time = status.get("startTime", None)
+        completion_time = status.get("completionTime", None)
+        if not completion_time or not start_time:
+            return None
+        dt_start_time = datetime.fromisoformat(start_time)
+        dt_completion_time = datetime.fromisoformat(completion_time)
+        return int((dt_completion_time - dt_start_time).total_seconds())
+
     def wait_for_job_list_completion(
         self, job_names: set[str], check_interval_seconds: int, timeout_seconds: int
     ) -> dict[str, JobStatus]: