qontract-reconcile 0.10.2.dev85__py3-none-any.whl → 0.10.2.dev87__py3-none-any.whl

{qontract_reconcile-0.10.2.dev85.dist-info → qontract_reconcile-0.10.2.dev87.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qontract-reconcile
-Version: 0.10.2.dev85
+Version: 0.10.2.dev87
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Project-URL: homepage, https://github.com/app-sre/qontract-reconcile
 Project-URL: repository, https://github.com/app-sre/qontract-reconcile

{qontract_reconcile-0.10.2.dev85.dist-info → qontract_reconcile-0.10.2.dev87.dist-info}/RECORD

@@ -199,22 +199,22 @@ reconcile/external_resources/aws.py,sha256=wzN3GHxyqVa4Lqqg5HdogqNW2RM532t0ZiKaQ
 reconcile/external_resources/factories.py,sha256=C0QHT0soEv6z99-ELAAE19S5MaMHhV0t1fSiQn0Coc4,5970
 reconcile/external_resources/integration.py,sha256=JF38M7R0Z4ADUTx57TZqSZH9k_xpPlbAxQAcGyIISuM,6925
 reconcile/external_resources/integration_secrets_sync.py,sha256=dX09O3r6KURziUYYfiki10orNjOGVma-XojhVqd0ww4,1667
-reconcile/external_resources/manager.py,sha256=ZagwLn6YQ1XmgmMN3qpuDzQsQxa4VOYl-IQPZBwCDqM,17103
+reconcile/external_resources/manager.py,sha256=mvFfcXPzvNqDWDgKTK8eiSe6C_FUvBtben3bSqrqSoc,18246
 reconcile/external_resources/meta.py,sha256=noaytFzmShpzLA_ebGh7wuP45mOfHIOnnoUxivjDa1I,672
 reconcile/external_resources/metrics.py,sha256=KiBjMUaN_z0cSkF_7Ar_a8RiuiwVqjyMcVdISlxhzXE,3898
-reconcile/external_resources/model.py,sha256=dxwiyI3J9xyLeue8_W9NJoap-CkKLMAoY0S0ml5-NbU,13450
+reconcile/external_resources/model.py,sha256=HrSkmAjOqw0aT3CsZKXKMLyocM__8XO5qHb-dXcoagw,13747
 reconcile/external_resources/reconciler.py,sha256=-0trp1K-iUgOQn3mm1ZUSmfaReRrUT0eHzPkUhNPolQ,9583
 reconcile/external_resources/secrets_sync.py,sha256=ZDxzGZ6wC4zxLhA7-L39xDRH6rzUM285gytuzmRQdlw,16208
 reconcile/external_resources/state.py,sha256=gF3ACdl7YiUlbQ4uEGrD6i_Txxqr6mT9f8IFlTQ-8dY,13176
 reconcile/fleet_labeler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/fleet_labeler/dependencies.py,sha256=Ta-SLnrHRN4OBAmhE_mTk1P7y1X7AInIiQsIYaY6hY0,2910
-reconcile/fleet_labeler/integration.py,sha256=gDK97QvsJ5IEo1t1ci4yxTRM7VSDj4uP9l30zHi_rWo,7700
+reconcile/fleet_labeler/dependencies.py,sha256=ZOpmtwgxEPBWU2yHRc6rhPwlvqhwYSsNMJQ_jVq1dLI,2993
+reconcile/fleet_labeler/integration.py,sha256=jm2wvkxsxqGLsX_vE874crgce_YsDWyggL3Q3olMNJM,8753
 reconcile/fleet_labeler/merge_request.py,sha256=VA_XSIob4LWYb02dRUr9coXjIa8_0prObI1v5Gqi_mY,1513
 reconcile/fleet_labeler/meta.py,sha256=DF7O4T9wvQ7-xTWXvuNw1OG_F0SBmRrjFBtVy9wWh9U,146
 reconcile/fleet_labeler/metrics.py,sha256=wx9BmXLsN67m-aSsf81iB7Ehj5SzUsS2WB75isUReZg,662
 reconcile/fleet_labeler/ocm.py,sha256=GGsz-bq1g8BJVVMCfI2kSwZCyngbQoZ3i3k8fO608KA,2506
 reconcile/fleet_labeler/validate.py,sha256=gzc2tt7h9F60h7dcyJfEmsnjnfuux5Jtc_WzrIqr-5k,2541
-reconcile/fleet_labeler/vcs.py,sha256=v4e_3l8F6aquVfe-ItLv2WJtS0kjMiRZ6DQ4mzCLOlE,726
+reconcile/fleet_labeler/vcs.py,sha256=6UHUQ08AGAHXF7629I6X-T_E1pvx96LxjS66EeOzve4,1108
 reconcile/glitchtip/README.md,sha256=rfXT6jNP9khJW65jL7I2PgoxvxgcGGuJF8NpbzufEQ4,4335
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=vCyg8W4ZUGxjU8tB1Gkre_auSpzo83n05mmO8_-7al0,8263
@@ -786,7 +786,7 @@ tools/saas_promotion_state/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJ
 tools/saas_promotion_state/saas_promotion_state.py,sha256=UfwwRLS5Ya4_Nh1w5n1dvoYtchQvYE9yj1VANt2IKqI,3925
 tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOyS39I,145
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
-qontract_reconcile-0.10.2.dev85.dist-info/METADATA,sha256=vrGwAn3HqOrv1hAUy75U95p-JGo8Qf35bTOhuopBq0U,24565
-qontract_reconcile-0.10.2.dev85.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-qontract_reconcile-0.10.2.dev85.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
-qontract_reconcile-0.10.2.dev85.dist-info/RECORD,,
+qontract_reconcile-0.10.2.dev87.dist-info/METADATA,sha256=nVRpZDkDyzcxrgE5yIHcfaAKhAQBbpzHtq_p3vStPr8,24565
+qontract_reconcile-0.10.2.dev87.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+qontract_reconcile-0.10.2.dev87.dist-info/entry_points.txt,sha256=5i9l54La3vQrDLAdwDKQWC0iG4sV9RRfOb1BpvzOWLc,698
+qontract_reconcile-0.10.2.dev87.dist-info/RECORD,,

reconcile/external_resources/manager.py

@@ -133,6 +133,40 @@ class ExternalResourcesManager:
         self.thread_pool_size = thread_pool_size
         self.dry_runs_validator = dry_runs_validator
 
+    def _resource_spec_changed(
+        self, reconciliation: Reconciliation, state: ExternalResourceState
+    ) -> bool:
+        return reconciliation.resource_hash != state.reconciliation.resource_hash
+
+    def _resource_drift_detection_ttl_expired(
+        self, reconciliation: Reconciliation, state: ExternalResourceState
+    ) -> bool:
+        return (datetime.now(state.ts.tzinfo) - state.ts).total_seconds() > (
+            reconciliation.module_configuration.reconcile_drift_interval_minutes * 60
+        )
+
+    def _reconciliation_module_config_overridden(
+        self, reconciliation: Reconciliation, state: ExternalResourceState
+    ) -> bool:
+        return reconciliation.module_configuration.overridden and (
+            reconciliation.module_configuration.image_version
+            != state.reconciliation.module_configuration.image_version
+        )
+
+    def _reconciliation_needs_dry_run_run(
+        self, reconciliation: Reconciliation, state: ExternalResourceState
+    ) -> bool:
+        return (
+            reconciliation.action == Action.APPLY
+            and (
+                self._resource_spec_changed(reconciliation, state)
+                or self._reconciliation_module_config_overridden(reconciliation, state)
+            )
+        ) or (
+            reconciliation.action == Action.DESTROY
+            and not state.resource_status.is_in_progress
+        )
+
     def _get_reconcile_action(
         self, reconciliation: Reconciliation, state: ExternalResourceState
     ) -> ReconcileAction:
@@ -145,17 +179,16 @@ class ExternalResourcesManager:
                 case ResourceStatus.ERROR:
                     return ReconcileAction.APPLY_ERROR
                 case ResourceStatus.CREATED | ResourceStatus.PENDING_SECRET_SYNC:
-                    if (
-                        reconciliation.resource_hash
-                        != state.reconciliation.resource_hash
-                    ):
+                    if self._resource_spec_changed(reconciliation, state):
                         return ReconcileAction.APPLY_SPEC_CHANGED
-                    elif (
-                        (datetime.now(state.ts.tzinfo) - state.ts).total_seconds()
-                        > reconciliation.module_configuration.reconcile_drift_interval_minutes
-                        * 60
+                    elif self._resource_drift_detection_ttl_expired(
+                        reconciliation, state
                     ):
                         return ReconcileAction.APPLY_DRIFT_DETECTION
+                    elif self._reconciliation_module_config_overridden(
+                        reconciliation, state
+                    ):
+                        return ReconcileAction.APPLY_MODULE_CONFIG_OVERRIDDEN
         elif reconciliation.action == Action.DESTROY:
             match state.resource_status:
                 case ResourceStatus.CREATED:
@@ -411,18 +444,13 @@ class ExternalResourcesManager:
         self.dry_runs_validator.validate()
         desired_r = self._get_desired_objects_reconciliations()
         deleted_r = self._get_deleted_objects_reconciliations()
-        reconciliations = desired_r.union(deleted_r)
-        triggered: set[Reconciliation] = set()
-
-        for r in reconciliations:
-            state = self.state_mgr.get_external_resource_state(key=r.key)
-            if (
-                r.action == Action.APPLY
-                and state.reconciliation.resource_hash != r.resource_hash
-            ) or (
-                r.action == Action.DESTROY and not state.resource_status.is_in_progress
-            ):
-                triggered.add(r)
+        triggered = {
+            r
+            for r in desired_r.union(deleted_r)
+            if self._reconciliation_needs_dry_run_run(
+                r, self.state_mgr.get_external_resource_state(key=r.key)
+            )
+        }
 
         threaded.run(
             self.reconciler.reconcile_resource,

reconcile/external_resources/model.py

@@ -7,7 +7,7 @@ from collections.abc import ItemsView, Iterable, Iterator, MutableMapping
 from enum import StrEnum
 from typing import Any
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 from reconcile.external_resources.meta import (
     FLAG_DELETE_RESOURCE,
@@ -263,6 +263,7 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
     outputs_secret_image: str = ""
     outputs_secret_version: str = ""
     resources: Resources = Resources()
+    overridden: bool = Field(default=False, exclude=True)
 
     @property
     def image_version(self) -> str:
@@ -278,17 +279,19 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
         spec: ExternalResourceSpec,
         settings: ExternalResourcesSettingsV1,
     ) -> "ExternalResourceModuleConfiguration":
-        module_overrides = spec.metadata.get(
-            "module_overrides"
-        ) or ExternalResourcesModuleOverrides(
-            module_type=None,
-            image=None,
-            version=None,
-            reconcile_timeout_minutes=None,
-            outputs_secret_image=None,
-            outputs_secret_version=None,
-            resources=None,
-        )
+        module_overrides = spec.metadata.get("module_overrides")
+        overridden = module_overrides is not None
+
+        if module_overrides is None:
+            module_overrides = ExternalResourcesModuleOverrides(
+                module_type=None,
+                image=None,
+                version=None,
+                reconcile_timeout_minutes=None,
+                outputs_secret_image=None,
+                outputs_secret_version=None,
+                resources=None,
+            )
 
         return ExternalResourceModuleConfiguration(
             image=module_overrides.image or module.image,
@@ -307,6 +310,7 @@ class ExternalResourceModuleConfiguration(BaseModel, frozen=True):
                 or module.resources
                 or settings.module_default_resources
             ),
+            overridden=overridden,
         )
 
 
@@ -358,6 +362,7 @@ class ReconcileAction(StrEnum):
     APPLY_SPEC_CHANGED = "Resource spec has changed"
     APPLY_DRIFT_DETECTION = "Resource drift detection run"
     APPLY_USER_REQUESTED = "Resource reconciliation requested"
+    APPLY_MODULE_CONFIG_OVERRIDDEN = "Module configuration overridden"
     DESTROY_CREATED = "Resource no longer exists in the configuration"
     DESTROY_ERROR = "Resource status in ERROR state"
 

reconcile/fleet_labeler/dependencies.py

@@ -27,10 +27,12 @@ class Dependencies:
         label_specs_by_name: Mapping[str, FleetLabelsSpecV1],
         ocm_clients_by_label_spec_name: Mapping[str, OCMClient],
         vcs: VCS,
+        dry_run: bool,
     ):
         self.label_specs_by_name = label_specs_by_name
         self.ocm_clients_by_label_spec_name = ocm_clients_by_label_spec_name
         self.vcs = vcs
+        self.dry_run = dry_run
 
     @classmethod
     def create(
@@ -42,6 +44,7 @@ class Dependencies:
             label_specs_by_name=_label_specs(),
             ocm_clients_by_label_spec_name=_ocm_clients(secret_reader=secret_reader),
             vcs=_vcs(secret_reader=secret_reader, dry_run=dry_run),
+            dry_run=dry_run,
         )
 
 

reconcile/fleet_labeler/integration.py

@@ -16,7 +16,7 @@ from reconcile.fleet_labeler.meta import (
 from reconcile.fleet_labeler.metrics import FleetLabelerDuplicateClusterMatchesGauge
 from reconcile.fleet_labeler.ocm import OCMClient
 from reconcile.fleet_labeler.validate import validate_label_specs
-from reconcile.fleet_labeler.vcs import VCS
+from reconcile.fleet_labeler.vcs import VCS, Gitlab404Error
 from reconcile.gql_definitions.fleet_labeler.fleet_labels import (
     FleetLabelDefaultV1,
     FleetLabelsSpecV1,
@@ -60,7 +60,10 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
         validate_label_specs(specs=dependencies.label_specs_by_name)
         for spec_name, ocm in dependencies.ocm_clients_by_label_spec_name.items():
             self._sync_cluster_inventory(
-                ocm, dependencies.label_specs_by_name[spec_name], dependencies.vcs
+                ocm=ocm,
+                spec=dependencies.label_specs_by_name[spec_name],
+                vcs=dependencies.vcs,
+                dry_run=dependencies.dry_run,
             )
 
     def _render_default_labels(
@@ -112,7 +115,11 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
             return stream.getvalue()
 
     def _sync_cluster_inventory(
-        self, ocm: OCMClient, spec: FleetLabelsSpecV1, vcs: VCS
+        self,
+        ocm: OCMClient,
+        spec: FleetLabelsSpecV1,
+        vcs: VCS,
+        dry_run: bool,
     ) -> None:
         class ClusterData(BaseModel):
             """
@@ -189,7 +196,23 @@ class FleetLabelerIntegration(QontractReconcileIntegration[NoParams]):
                 f"[{spec.name}] Deleting cluster {cluster_id=} from inventory."
             )
 
-        current_content = vcs.get_file_content_from_main(path=spec.path)
+        # When adding a new label spec file, then we dont have any existing content in main yet.
+        # This is a chicken-egg problem, but if we are in dry-run we can skip these steps on 404s.
+        # Note, that the diff is already printed above, so we can make a good decision if desired
+        # content fits.
+        # If the content exists in main, then it doesnt harm to also run the rendering procedure.
+        try:
+            current_content = vcs.get_file_content_from_main(path=spec.path)
+        except Gitlab404Error:
+            if dry_run:
+                logging.info(
+                    f"The file data{spec.path} does not exist in main branch yet. This is likely because it is being created with this MR. We are skipping rendering steps."
+                )
+                return
+            # 404 must never happen on non-dry-run, as the file must have already passed
+            # MR check and must have been merged to main
+            raise
+
         # Lets make sure we are deterministic when adding new clusters
         # The overhead is neglectable and it makes testing easier
         sorted_clusters_to_add = sorted(clusters_to_add, key=lambda c: c.name)

reconcile/fleet_labeler/vcs.py

@@ -1,7 +1,13 @@
+from gitlab.exceptions import GitlabGetError
+
 from reconcile.fleet_labeler.merge_request import FleetLabelerUpdates
 from reconcile.utils.vcs import VCS as VCSBase
 
 
+class Gitlab404Error(Exception):
+    pass
+
+
 class VCS:
     """
     Thin abstractions of reconcile.utils.vcs module to reduce coupling and simplify tests.
@@ -11,9 +17,16 @@ class VCS:
         self._vcs = vcs
 
     def get_file_content_from_main(self, path: str) -> str:
-        return self._vcs.get_file_content_from_app_interface_ref(
-            file_path=path, ref="main"
-        )
+        try:
+            return self._vcs.get_file_content_from_app_interface_ref(
+                file_path=path, ref="main"
+            )
+        except GitlabGetError as e:
+            if e.response_code != 404:
+                raise
+            raise Gitlab404Error(
+                f"File at ${path} does not exist yet in main branch. Maybe it is just being created with this MR?"
+            ) from e
 
     def open_merge_request(self, path: str, content: str) -> None:
         mr = FleetLabelerUpdates(path=path, content=content)