qontract-reconcile 0.10.1rc863__py3-none-any.whl → 0.10.1rc865__py3-none-any.whl

{qontract_reconcile-0.10.1rc863.dist-info → qontract_reconcile-0.10.1rc865.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc863
+Version: 0.10.1rc865
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc863.dist-info → qontract_reconcile-0.10.1rc865.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=1x1ZIR1wKgnGh799yPcV8IQD1RHyHwWlDg9Llh1diOw,104736
+reconcile/cli.py,sha256=zGgAkhK8Pl8NIKtnPr8fz5ewHa1_SSMJYtrvPa-ip6I,105151
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=R2JuwiXAEYAFiCtnztM_IIr1rtVzPpaWAmgxuDa2FgY,4813
@@ -35,7 +35,7 @@ reconcile/gitlab_labeler.py,sha256=IxE1XM5o4rDOFuR4cM2yAHTy4Uzdg3Nyz2mp7b8Fx1g,4
 reconcile/gitlab_members.py,sha256=M6LwFOrwgvl1NNdOJa1mrQFUon-bEVv1AyhGeLed454,8443
 reconcile/gitlab_mr_sqs_consumer.py,sha256=O46mdziPgGOndbU-0_UJKJVUaiEoVzJPEgKm4_UvYoI,2571
 reconcile/gitlab_owners.py,sha256=sn9njaKOtqcvnhi2qtm-faAfAR4zNqflbSuusA9RUuI,13456
-reconcile/gitlab_permissions.py,sha256=LcO5tVND3Rh-onU6WFh5ZWPCqZ-U4AVuFTZIs_bmPwA,3153
+reconcile/gitlab_permissions.py,sha256=YaLiMqrn9YOM5e7o2NJVg6vtILyBXTryTScA4RhTyPo,3279
 reconcile/gitlab_projects.py,sha256=K3tFf_aD1W4Ijp5q-9Qek3kwFGEWPcZ1kd7tzFJ4GyQ,1781
 reconcile/integrations_manager.py,sha256=J_VV-HINI7YNav2NPIolePZkll-7VBuBXWAyMNhsM_Q,9535
 reconcile/jenkins_base.py,sha256=0Gocu3fU2YTltaxBlbDQOUvP-7CP2OSQV1ZRwtWeVXw,875
@@ -184,14 +184,15 @@ reconcile/dynatrace_token_provider/metrics.py,sha256=xiKkl8fTEBQaXJelGCPNTZhHAWd
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG-zlHOM,2956
 reconcile/external_resources/factories.py,sha256=bLboXX5Dq0xN60mtDGNjCOLC6HlKofXMWQxVbRwMMwo,4485
-reconcile/external_resources/integration.py,sha256=rIfDVnTUy3qdmrh7NRwvZ7l1hOiNfk0a57J1ZsRSNrg,4959
-reconcile/external_resources/manager.py,sha256=APszaw9PRIiHnFyCffHZjIFseIwhlYROIPLt18pHUTQ,13685
-reconcile/external_resources/meta.py,sha256=SA4Km1r7ePdcNqHn2GA4ByQp4ZnIeo_n8qOOd-11IEg,151
+reconcile/external_resources/integration.py,sha256=p07tnD4pww9QgFlHT18RRn929BL7zKzRHRvHQ_ETRAU,5113
+reconcile/external_resources/integration_secrets_sync.py,sha256=cMEZhgCvABAMf-DWF051L6CRnJQdfbsISA_b1xuS940,1670
+reconcile/external_resources/manager.py,sha256=d1YuCZWOnUV3nfZ0HJI-IIG5qef7eQ4ct7hSTXcFK4M,14576
+reconcile/external_resources/meta.py,sha256=Z5guBceyaGyAzzA9kVb0-WaNpSli368NVUnWulxtMb4,551
 reconcile/external_resources/metrics.py,sha256=m2TIOao2N7pD6k45driFbBGVCC_N7ai44m-lLPfa5qk,454
 reconcile/external_resources/model.py,sha256=FJUb7rHU2l7YSAv-t4QaacL9pqheFBxhPydWSPqu3vY,7413
 reconcile/external_resources/reconciler.py,sha256=E50X_lnOD0OWYXMzyZld1P6dCFJFYjHGyICWff9bxlc,9323
-reconcile/external_resources/secrets_sync.py,sha256=g-ksvzmTlCTwo3PM3FgYXm0LUBcnwfAxcvisuR1jAMY,7982
-reconcile/external_resources/state.py,sha256=LrZ_-9DRmKOdPlYgxgaYlGAOa6lzKDgcmn-Zc4AQDX0,9333
+reconcile/external_resources/secrets_sync.py,sha256=xFQ_ObWl29btbxzEJSkndvHBPcsVpSTkmUlWcUGzZ70,15132
+reconcile/external_resources/state.py,sha256=fA_CzT4oNie4wnaImwW-W1duWEOKFyS1omcnMyYwx2Q,9644
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=Y7ofQg_xCt3dOln3pjeXp7rAnwohCgD2zcUAb-Hciis,8375
 reconcile/glitchtip/reconciler.py,sha256=nUvDv7qG1ly0cA16MmlL6NV71yl1mJYLT2mui7lmi0Y,12402
@@ -274,9 +275,9 @@ reconcile/gql_definitions/dynatrace_token_provider/__init__.py,sha256=47DEQpj8HB
 reconcile/gql_definitions/dynatrace_token_provider/dynatrace_bootstrap_tokens.py,sha256=5gTuAnR2rnx2k6Rn7FMEAzw6GCZ6F5HZbqkmJ9-3NI4,2244
 reconcile/gql_definitions/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/external_resources/aws_accounts.py,sha256=XR69j9dpTQ0gv8y-AZN7AJ0dPvO-wbHscyCDgrax6Bk,2046
-reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=2VIb3hC2MRNhYDZ1al4PvudT2oSfd3fPGpzv4CEJNiw,2341
+reconcile/gql_definitions/external_resources/external_resources_modules.py,sha256=g2KB2wRnb8zF7xCmDJJFmiRdE4z4aYa9HtY3vCBVwMA,2441
 reconcile/gql_definitions/external_resources/external_resources_namespaces.py,sha256=UyOAUY1rROenjTz6y-uSEFjrEwhh-lPsIQPbi6EQLFg,40915
-reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=989_pG9NWKB5BPvdwqjqZUYp_2qf-xYmJ9c9kq8Kmfw,2886
+reconcile/gql_definitions/external_resources/external_resources_settings.py,sha256=Hw9n_90BPG6Lnt2PT3mHc6p0KEm2CxKxvSGRFc_Dhus,2982
 reconcile/gql_definitions/fragments/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/gql_definitions/fragments/aus_organization.py,sha256=uBKbTuBa3CZmTXR5HOcGhRcu2U9kM93KbYmoWTxcpB0,4767
 reconcile/gql_definitions/fragments/aws_account_common.py,sha256=3-7ZAP6GSff7Z2Syz2VQCLY4IySqBOSVmceaRiVNQpw,2385
@@ -647,7 +648,7 @@ reconcile/utils/environ.py,sha256=VnW3zp6Un_UJn5BU4FU8RfhuqtZp0s-VeuuHnqC_WcQ,51
 reconcile/utils/exceptions.py,sha256=DwfnWUpVOotpP79RWZ2pycmG6nKCL00RBIeZLYkQPW4,635
 reconcile/utils/expiration.py,sha256=BXwKE50sNIV-Lszke97fxitNkLxYszoOLW1LBgp_yqg,1246
 reconcile/utils/extended_early_exit.py,sha256=QSktrmfw37zSRMNk930tDbQsVeKxaPPPD43e79DGwZw,6754
-reconcile/utils/external_resource_spec.py,sha256=IRY8MCsyWKzt-Qj_hXiFKgkCvZu6VVyj6IYtqEb05BA,6618
+reconcile/utils/external_resource_spec.py,sha256=OqEwhMgdpiWm9CnlpZjCMzqFmBDVfFDNYkcxvLmBwXM,6922
 reconcile/utils/external_resources.py,sha256=ZnBjQlMpiuCX0Ivm77eIMB4Um_RuAsoMtc8IyuvIxI4,7590
 reconcile/utils/filtering.py,sha256=zZnHH0u0SaTDyzuFXZ_mREURGLvjEqQIQy4z-7QBVlc,419
 reconcile/utils/git.py,sha256=BdxXFgQ1XOZpS-4qb3qMsKTCFDG8MlE26rv1jAhvCkM,1560
@@ -678,7 +679,7 @@ reconcile/utils/oc_connection_parameters.py,sha256=85slrnDigYwYmzhyceVkMElWzFArp
 reconcile/utils/oc_filters.py,sha256=R2Lf3fo0jQCeE62Ygeo_KN24XbAosq0QbjimYG6qHI4,1402
 reconcile/utils/oc_map.py,sha256=nT69J5pdPeIDnIYjD9fwY6GkE3BMQCf-AF0rmHJuUNw,9068
 reconcile/utils/ocm_base_client.py,sha256=X8qkPXfpfJdBKBtFv7zyGD33HNAEBJL8owf-ykrt-Ts,6469
-reconcile/utils/openshift_resource.py,sha256=l5VLvwZ1LRi3l8InMdJS0oc1w5juhgwpWD2Yg7rXubc,24763
+reconcile/utils/openshift_resource.py,sha256=zA2hOhhvAb6v_NAcge_pem2EL1G7JMGolEfvFoHMgvM,24952
 reconcile/utils/openssl.py,sha256=QVvhzhpChq_4Daf_5wE1qeZJr4thg3DDjJPn4bOPD4E,365
 reconcile/utils/output.py,sha256=I_kXYyPcN1mlZmX16ZnLNGkhhwnal640GIdIaGJd4wE,2026
 reconcile/utils/pagerduty_api.py,sha256=fcSAUez6w51woDvbm0plJW2qSw6_NXQs1Fit_KTNitc,7653
@@ -836,8 +837,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jrss,4941
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc863.dist-info/METADATA,sha256=y5iyh4zPDNPYn2dGYtX6BC-ak-Fy21lC_RxGqXh0BjA,2273
-qontract_reconcile-0.10.1rc863.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc863.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc863.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc863.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc865.dist-info/METADATA,sha256=hKbGCp2EG6vMoZ5lyFHsA9xtBs6aXsG0AINh0fD9w5A,2273
+qontract_reconcile-0.10.1rc865.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc865.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc865.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc865.dist-info/RECORD,,

reconcile/cli.py

@@ -3712,6 +3712,24 @@ def external_resources(
     )
 
 
+@integration.command(
+    short_help="Syncs External Resources Secrets from Vault to Clusters"
+)
+@click.pass_context
+@threaded(default=5)
+def external_resources_secrets_sync(
+    ctx,
+    thread_pool_size: int,
+):
+    import reconcile.external_resources.integration_secrets_sync
+
+    run_integration(
+        reconcile.external_resources.integration_secrets_sync,
+        ctx.obj,
+        thread_pool_size,
+    )
+
+
 def get_integration_cli_meta() -> dict[str, IntegrationMeta]:
     """
     returns all integrations known to cli.py via click introspection

reconcile/external_resources/integration.py

@@ -124,7 +124,12 @@ def run(
                 dry_run_job_suffix=dry_run_job_suffix,
             ),
             secrets_reconciler=build_incluster_secrets_reconciler(
-                workers_cluster, workers_namespace, secret_reader, vault_path="app-sre"
+                workers_cluster,
+                workers_namespace,
+                secret_reader,
+                vault_path=er_settings.vault_secrets_path,
+                thread_pool_size=thread_pool_size,
+                dry_run=dry_run,
             ),
         )
 

reconcile/external_resources/integration_secrets_sync.py

@@ -0,0 +1,47 @@
+from reconcile.external_resources.model import (
+    ExternalResourcesInventory,
+    load_module_inventory,
+)
+from reconcile.external_resources.secrets_sync import VaultSecretsReconciler
+from reconcile.typed_queries.app_interface_vault_settings import (
+    get_app_interface_vault_settings,
+)
+from reconcile.typed_queries.external_resources import (
+    get_modules,
+    get_namespaces,
+    get_settings,
+)
+from reconcile.utils.openshift_resource import ResourceInventory
+from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.semver_helper import make_semver
+
+QONTRACT_INTEGRATION = "external_resources_secrets_sync"
+QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
+
+
+def run(dry_run: bool, thread_pool_size: int) -> None:
+    """Integration that syncs External Resources Outputs Secrets from Vault into
+    the target clusters
+    """
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    er_settings = get_settings()[0]
+
+    namespaces = [ns for ns in get_namespaces() if ns.external_resources]
+    er_inventory = ExternalResourcesInventory(namespaces)
+    m_inventory = load_module_inventory(get_modules())
+
+    to_sync_specs = [
+        spec
+        for key, spec in er_inventory.items()
+        if m_inventory.get_from_external_resource_key(key).outputs_secret_sync
+    ]
+
+    reconciler = VaultSecretsReconciler(
+        ri=ResourceInventory(),
+        secrets_reader=secret_reader,
+        vault_path=er_settings.vault_secrets_path,
+        thread_pool_size=thread_pool_size,
+        dry_run=dry_run,
+    )
+    reconciler.sync_secrets(to_sync_specs)

reconcile/external_resources/manager.py

@@ -212,7 +212,7 @@ class ExternalResourcesManager:
             ResourceStatus.DELETE_IN_PROGRESS,
             ResourceStatus.IN_PROGRESS,
         ]):
-            return need_secret_sync
+            return False
 
         logging.info(
             "Reconciliation In progress. Action: %s, Key:%s",
@@ -231,7 +231,7 @@ class ExternalResourcesManager:
                     r.key,
                 )
                 if r.action == Action.APPLY:
-                    state.resource_status = ResourceStatus.CREATED
+                    state.resource_status = ResourceStatus.PENDING_SECRET_SYNC
                     state.reconciliation_errors = 0
                     self.state_mgr.set_external_resource_state(state)
                     need_secret_sync = True
@@ -275,9 +275,31 @@ class ExternalResourcesManager:
             r.action == Action.APPLY and state.resource_status == ResourceStatus.CREATED
         )
 
-    def _sync_secrets(self, keys: Iterable[ExternalResourceKey]) -> None:
-        specs = [spec for key in keys if (spec := self.er_inventory.get(key))]
-        self.secrets_reconciler.sync_secrets(specs=specs)
+    def _sync_secrets(
+        self,
+        to_sync_keys: Iterable[ExternalResourceKey],
+    ) -> None:
+        specs = [
+            spec for key in set(to_sync_keys) if (spec := self.er_inventory.get(key))
+        ]
+
+        sync_error_spec_keys = {
+            ExternalResourceKey.from_spec(spec)
+            for spec in self.secrets_reconciler.sync_secrets(specs=specs)
+        }
+
+        for key in to_sync_keys:
+            if key in sync_error_spec_keys:
+                logging.error(
+                    "Outputs secret for key can not be reconciled. Key: %s", key
+                )
+            else:
+                logging.debug(
+                    "Outputs secret for key has been reconciled. Marking resource as %s. Key: %s",
+                    ResourceStatus.CREATED,
+                    key,
+                )
+                self.state_mgr.update_resource_status(key, ResourceStatus.CREATED)
 
     def _build_external_resource(
         self, spec: ExternalResourceSpec, er_inventory: ExternalResourcesInventory
@@ -295,12 +317,12 @@ class ExternalResourcesManager:
     def handle_resources(self) -> None:
         desired_r = self._get_desired_objects_reconciliations()
         deleted_r = self._get_deleted_objects_reconciliations()
-        to_sync: set[ExternalResourceKey] = set()
+        to_sync_keys: set[ExternalResourceKey] = set()
         for r in desired_r.union(deleted_r):
             state = self.state_mgr.get_external_resource_state(r.key)
             need_sync = self._update_in_progress_state(r, state)
             if need_sync:
-                to_sync.add(r.key)
+                to_sync_keys.add(r.key)
 
             metrics.set_gauge(
                 ExternalResourcesReconcileErrorsGauge(
@@ -316,8 +338,12 @@ class ExternalResourcesManager:
                 self.reconciler.reconcile_resource(reconciliation=r)
                 self._update_state(r, state)
 
-        if to_sync:
-            self._sync_secrets(keys=to_sync)
+        pending_sync_keys = self.state_mgr.get_keys_by_status(
+            ResourceStatus.PENDING_SECRET_SYNC
+        )
+
+        if to_sync_keys or pending_sync_keys:
+            self._sync_secrets(to_sync_keys=to_sync_keys | pending_sync_keys)
 
     def handle_dry_run_resources(self) -> None:
         desired_r = self._get_desired_objects_reconciliations()

reconcile/external_resources/meta.py

@@ -2,3 +2,12 @@ from reconcile.utils.semver_helper import make_semver
 
 QONTRACT_INTEGRATION = "external_resources"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
+
+
+SECRET_ANN_PREFIX = "external-resources"
+SECRET_ANN_PROVISION_PROVIDER = SECRET_ANN_PREFIX + "/provision_provider"
+SECRET_ANN_PROVISIONER = SECRET_ANN_PREFIX + "/provisioner_name"
+SECRET_ANN_PROVIDER = SECRET_ANN_PREFIX + "/provider"
+SECRET_ANN_IDENTIFIER = SECRET_ANN_PREFIX + "/identifier"
+SECRET_UPDATED_AT = SECRET_ANN_PREFIX + "/updated_at"
+SECRET_UPDATED_AT_TIMEFORMAT = "%Y-%m-%dT%H:%M:%SZ"

reconcile/external_resources/secrets_sync.py

@@ -1,17 +1,27 @@
 import base64
 import json
+import logging
 from abc import abstractmethod
 from collections.abc import Iterable, Mapping
+from datetime import datetime, timezone
 from hashlib import shake_128
-from typing import Any, Optional, cast
+from typing import Any, Optional
 
 from pydantic import BaseModel
-from sretoolbox.utils import retry
+from sretoolbox.utils import threaded
 
 from reconcile.external_resources.meta import (
     QONTRACT_INTEGRATION,
     QONTRACT_INTEGRATION_VERSION,
+    SECRET_ANN_IDENTIFIER,
+    SECRET_ANN_PROVIDER,
+    SECRET_ANN_PROVISION_PROVIDER,
+    SECRET_ANN_PROVISIONER,
+    SECRET_UPDATED_AT,
+    SECRET_UPDATED_AT_TIMEFORMAT,
 )
+from reconcile.external_resources.model import ExternalResourceKey
+from reconcile.openshift_base import ApplyOptions, apply_action
 from reconcile.typed_queries.clusters_minimal import get_clusters_minimal
 from reconcile.utils.differ import diff_mappings
 from reconcile.utils.external_resource_spec import (
@@ -22,7 +32,7 @@ from reconcile.utils.oc import (
 )
 from reconcile.utils.oc_map import OCMap, init_oc_map_from_clusters
 from reconcile.utils.openshift_resource import OpenshiftResource, ResourceInventory
-from reconcile.utils.secret_reader import SecretReaderBase
+from reconcile.utils.secret_reader import SecretNotFound, SecretReaderBase
 from reconcile.utils.three_way_diff_strategy import three_way_diff_using_hash
 from reconcile.utils.vault import (
     VaultClient,
@@ -39,54 +49,103 @@ class VaultSecret(BaseModel):
     q_format: Optional[str]
 
 
+class SecretHelper:
+    @staticmethod
+    def get_comparable_secret(resource: OpenshiftResource) -> OpenshiftResource:
+        metadata = {k: v for k, v in resource.body["metadata"].items()}
+        metadata["annotations"] = {
+            k: v
+            for k, v in metadata.get("annotations", {}).items()
+            if k != SECRET_UPDATED_AT
+        }
+        new = OpenshiftResource(
+            body=resource.body | {"metadata": metadata},
+            integration=resource.integration,
+            integration_version=resource.integration_version,
+            error_details=resource.error_details,
+            caller_name=resource.caller_name,
+            validate_k8s_object=False,
+        )
+
+        return new
+
+    @staticmethod
+    def is_newer(a: OpenshiftResource, b: OpenshiftResource) -> bool:
+        try:
+            # ISO8601 with the same TZ can be compared as strings.
+            return a.annotations[SECRET_UPDATED_AT] > b.annotations[SECRET_UPDATED_AT]
+        except (KeyError, ValueError) as e:
+            logging.debug("Error comparing timestamps %s", e)
+            return False
+
+    @staticmethod
+    def compare(current: OpenshiftResource, desired: OpenshiftResource) -> bool:
+        if SECRET_UPDATED_AT not in current.annotations:
+            logging.debug(
+                "Current does not have the optimistic locking annotation. Apply"
+            )
+            return False
+        # if current is newer; don't apply
+        if SecretHelper.is_newer(current, desired):
+            logging.debug("Current Secret is newer than Desired: Don't Apply")
+            return True
+        cmp_current = SecretHelper.get_comparable_secret(current)
+        cmp_desired = SecretHelper.get_comparable_secret(desired)
+
+        return three_way_diff_using_hash(cmp_current, cmp_desired)
+
+
 class SecretsReconciler:
     def __init__(
         self,
         ri: ResourceInventory,
         secrets_reader: SecretReaderBase,
-        vault_path: str,
-        vault_client: VaultClient,
+        thread_pool_size: int,
+        dry_run: bool,
     ) -> None:
         self.secrets_reader = secrets_reader
         self.ri = ri
-        self.vault_path = vault_path
-        self.vault_client = cast(_VaultClient, vault_client)
+        self.thread_pool_size = thread_pool_size
+        self.dry_run = dry_run
 
     @abstractmethod
     def _populate_secret_data(self, specs: Iterable[ExternalResourceSpec]) -> None:
         raise NotImplementedError()
 
-    def _populate_annotations(self, spec: ExternalResourceSpec) -> None:
+    def _annotate(self, spec: ExternalResourceSpec) -> None:
         try:
             annotations = json.loads(spec.resource["annotations"])
         except Exception:
             annotations = {}
-
-        annotations["provision_provider"] = spec.provision_provider
-        annotations["provisioner"] = spec.provisioner_name
-        annotations["provider"] = spec.provider
-        annotations["identifier"] = spec.identifier
-
+        annotations[SECRET_ANN_PROVISION_PROVIDER] = spec.provision_provider
+        annotations[SECRET_ANN_PROVISIONER] = spec.provisioner_name
+        annotations[SECRET_ANN_PROVIDER] = spec.provider
+        annotations[SECRET_ANN_IDENTIFIER] = spec.identifier
+        annotations[SECRET_UPDATED_AT] = spec.metadata[SECRET_UPDATED_AT]
         spec.resource["annotations"] = json.dumps(annotations)
 
-    def _initialize_ri(
+    def _specs_with_secret(
         self,
-        ri: ResourceInventory,
         specs: Iterable[ExternalResourceSpec],
+    ) -> Iterable[ExternalResourceSpec]:
+        return [spec for spec in specs if spec.secret]
+
+    def _add_secret_to_ri(
+        self,
+        spec: ExternalResourceSpec,
     ) -> None:
-        for spec in specs:
-            ri.initialize_resource_type(
-                spec.cluster_name, spec.namespace_name, "Secret"
-            )
-            ri.add_desired(
-                spec.cluster_name,
-                spec.namespace_name,
-                "Secret",
-                name=spec.output_resource_name,
-                value=spec.build_oc_secret(
-                    QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION
-                ),
-            )
+        self.ri.initialize_resource_type(
+            spec.cluster_name, spec.namespace_name, "Secret"
+        )
+        self.ri.add_desired(
+            spec.cluster_name,
+            spec.namespace_name,
+            "Secret",
+            name=spec.output_resource_name,
+            value=spec.build_oc_secret(
+                QONTRACT_INTEGRATION, QONTRACT_INTEGRATION_VERSION
+            ).annotate(),
+        )
 
     def _init_ocmap(self, specs: Iterable[ExternalResourceSpec]) -> OCMap:
         return init_oc_map_from_clusters(
@@ -99,54 +158,117 @@ class SecretsReconciler:
             integration=QONTRACT_INTEGRATION,
         )
 
-    @retry()
-    def _write_secrets_to_vault(self, spec: ExternalResourceSpec) -> None:
-        if spec.secret:
-            secret_path = f"{self.vault_path}/{QONTRACT_INTEGRATION}/{spec.cluster_name}/{spec.namespace_name}/{spec.output_resource_name}"
-            stringified_secret = {k: str(v) for k, v in spec.secret.items()}
-            desired_secret = {"path": secret_path, "data": stringified_secret}
-            self.vault_client.write(desired_secret, decode_base64=False)
+    def sync_secrets(
+        self, specs: Iterable[ExternalResourceSpec]
+    ) -> list[ExternalResourceSpec]:
+        """Sync outputs secrets to target clusters.
+        Logic:
+        Vault To Cluster:
+            If current is newer; don't apply.
+            If other changes; apply and Recycle Pods
+            Desired can not be newer than current.
+        External reosurce to Cluster (last reconciliation):
+            If updated_at annotation is the only change; Don't update
+            If other changes; Update Secret and Recycle Pods
+            Current can not be newer then Desired
 
-    def sync_secrets(self, specs: Iterable[ExternalResourceSpec]) -> None:
+        :param specs: Specs that need sync the outputs secret to the target cluster
+        :return: specs that produced errors when syncing secrets to clusters.
+        """
         self._populate_secret_data(specs)
-        ri = ResourceInventory()
-        self._initialize_ri(ri, specs)
-        ocmap = self._init_ocmap(specs)
-        for item in ri:
-            self.reconcile_data(item, ri, ocmap)
+
+        to_sync_specs = [spec for spec in self._specs_with_secret(specs)]
+        ocmap = self._init_ocmap(to_sync_specs)
+
+        for spec in to_sync_specs:
+            self._annotate(spec)
+            self._add_secret_to_ri(spec)
+
+        threaded.run(
+            self.reconcile_data,
+            self.ri,
+            thread_pool_size=self.thread_pool_size,
+            ocmap=ocmap,
+        )
+
+        if self.ri.has_error_registered():
+            # Return all specs as error if there are errors.
+            # There is no a clear way to kwno which specs failed.
+            return list(specs)
+        else:
+            return []
 
     def reconcile_data(
         self,
         ri_item: tuple[str, str, str, Mapping[str, Any]],
-        ri: ResourceInventory,
         ocmap: OCMap,
     ) -> None:
         cluster, namespace, kind, data = ri_item
         oc = ocmap.get_cluster(cluster)
         names = list(data["desired"].keys())
 
+        logging.debug(
+            "Getting Secrets from cluster/namespace %s/%s", cluster, namespace
+        )
         items = oc.get_items("Secret", namespace=namespace, resource_names=names)
+
         for item in items:
-            obj = OpenshiftResource(
+            current = OpenshiftResource(
                 body=item,
                 integration=QONTRACT_INTEGRATION,
                 integration_version=QONTRACT_INTEGRATION_VERSION,
             )
-            ri.add_current(cluster, namespace, kind, name=obj.name, value=obj)
+
+            self.ri.add_current(
+                cluster, namespace, kind, name=current.name, value=current
+            )
 
         diff = diff_mappings(
-            data["current"], data["desired"], equal=three_way_diff_using_hash
+            data["current"], data["desired"], equal=SecretHelper.compare
         )
-        items_to_update = [i.desired for i in diff.change.values()] + list(
+
+        items_to_update = [item.desired for item in diff.change.values()] + list(
             diff.add.values()
         )
-        self.apply_action(oc, namespace, items_to_update)
+
+        self.apply_action(ocmap, cluster, namespace, items_to_update)
 
     def apply_action(
-        self, oc: OCCli, namespace: str, items: Iterable[OpenshiftResource]
+        self,
+        ocmap: OCMap,
+        cluster: str,
+        namespace: str,
+        items: Iterable[OpenshiftResource],
     ) -> None:
-        for i in items:
-            oc.apply(namespace, resource=i)
+        options = ApplyOptions(
+            dry_run=self.dry_run,
+            no_dry_run_skip_compare=False,
+            wait_for_namespace=False,
+            recycle_pods=True,
+            take_over=False,
+            override_enable_deletion=False,
+            caller=None,
+            all_callers=None,
+            privileged=None,
+            enable_deletion=None,
+        )
+        for item in items:
+            logging.debug(
+                "Updating Secret Cluster: %s, Namespace: %s, Secret: %s",
+                cluster,
+                namespace,
+                item.name,
+            )
+
+            apply_action(
+                ocmap,
+                self.ri,
+                cluster,
+                namespace,
+                "Secret",
+                item,
+                options=options,
+            )
 
 
 class InClusterSecretsReconciler(SecretsReconciler):
@@ -159,56 +281,105 @@ class InClusterSecretsReconciler(SecretsReconciler):
         cluster: str,
         namespace: str,
         oc: OCCli,
+        thread_pool_size: int,
+        dry_run: bool,
     ):
-        super().__init__(ri, secrets_reader, vault_path, vault_client)
+        super().__init__(ri, secrets_reader, thread_pool_size, dry_run)
 
         self.cluster = cluster
         self.namespace = namespace
         self.oc = oc
         self.source_secrets: list[str] = []
+        self.vault_client = vault_client
+        self.vault_path = vault_path
 
     def _get_spec_hash(self, spec: ExternalResourceSpec) -> str:
         secret_key = f"{spec.provision_provider}-{spec.provisioner_name}-{spec.provider}-{spec.identifier}"
         return shake_128(secret_key.encode("utf-8")).hexdigest(16)
 
+    def _get_spec_outputs_secret_name(self, spec: ExternalResourceSpec) -> str:
+        return "external-resources-output-" + self._get_spec_hash(spec)
+
     def _populate_secret_data(self, specs: Iterable[ExternalResourceSpec]) -> None:
         if not specs:
             return
-        secrets_map = {
-            "external-resources-output-" + self._get_spec_hash(spec): spec
-            for spec in specs
-        }
+
+        secrets_map = {self._get_spec_outputs_secret_name(spec): spec for spec in specs}
+
         secrets = self.oc.get_items(
             "Secret", namespace=self.namespace, resource_names=list(secrets_map.keys())
         )
+
         for secret in secrets:
             secret_name = secret["metadata"]["name"]
             spec = secrets_map[secret_name]
             data = dict[str, str]()
             for k, v in secret["data"].items():
                 decoded = base64.b64decode(v).decode("utf-8")
+
                 if decoded.startswith("__vault__:"):
                     _secret_ref = json.loads(decoded.replace("__vault__:", ""))
                     secret_ref = VaultSecret(**_secret_ref)
                     data[k] = self.secrets_reader.read_secret(secret_ref)
                 else:
                     data[k] = decoded
+
+            spec.metadata[SECRET_UPDATED_AT] = datetime.now(timezone.utc).strftime(
+                SECRET_UPDATED_AT_TIMEFORMAT
+            )
             spec.secret = data
 
-        self.source_secrets = list(secrets_map.keys())
+    def _delete_source_secret(self, spec: ExternalResourceSpec) -> None:
+        secret_name = self._get_spec_outputs_secret_name(spec)
+        logging.debug("Deleting secret " + secret_name)
+        self.oc.delete(namespace=self.namespace, kind="Secret", name=secret_name)
+
+    def _write_secret_to_vault(self, spec: ExternalResourceSpec) -> None:
+        secret_path = f"{self.vault_path}/{spec.cluster_name}/{spec.namespace_name}/{spec.identifier}"
+        secret = {k: str(v) for k, v in spec.secret.items()}
+        secret[SECRET_UPDATED_AT] = spec.metadata[SECRET_UPDATED_AT]
+        desired_secret = {"path": secret_path, "data": secret}
+        self.vault_client.write(desired_secret, decode_base64=False)  # type: ignore[attr-defined]
 
-    def _delete_source_secrets(self) -> None:
-        for secret_name in self.source_secrets:
-            print("Deleting secret " + secret_name)
-            self.oc.delete(namespace=self.namespace, kind="Secret", name=secret_name)
+    def sync_secrets(
+        self, specs: Iterable[ExternalResourceSpec]
+    ) -> list[ExternalResourceSpec]:
+        try:
+            specs_with_error = super().sync_secrets(specs)
+        except Exception as e:
+            # There is no an easy way to map which secrets have not been reconciled with the specs. If the sync
+            # fails at this stage all the involved specs will be retried in the next iteration
+            logging.error(
+                "Error syncing Secrets to clusters. "
+                "All specs reconciled in this iteration are marked as pending secret synchronization\n%s",
+                e,
+            )
+            return list(specs)
+
+        for spec in self._specs_with_secret(specs):
+            try:
+                self._write_secret_to_vault(spec)
+                self._delete_source_secret(spec)
+            except Exception as e:
+                key = ExternalResourceKey.from_spec(spec)
+                logging.error(
+                    "Error writing Secret to Vault or deleting the source secret: Key: %s, Secret: %s\n%s",
+                    key,
+                    self._get_spec_outputs_secret_name(spec),
+                    e,
+                )
+                specs_with_error.append(spec)
 
-    def sync_secrets(self, specs: Iterable[ExternalResourceSpec]) -> None:
-        super().sync_secrets(specs)
-        self._delete_source_secrets()
+        return specs_with_error
 
 
 def build_incluster_secrets_reconciler(
-    cluster: str, namespace: str, secrets_reader: SecretReaderBase, vault_path: str
+    cluster: str,
+    namespace: str,
+    secrets_reader: SecretReaderBase,
+    vault_path: str,
+    thread_pool_size: int,
+    dry_run: bool,
 ) -> InClusterSecretsReconciler:
     ri = ResourceInventory()
     ocmap = init_oc_map_from_clusters(
@@ -217,13 +388,42 @@ def build_incluster_secrets_reconciler(
         integration=QONTRACT_INTEGRATION,
     )
     oc = ocmap.get_cluster(cluster)
-    vault_client = VaultClient()
     return InClusterSecretsReconciler(
         cluster=cluster,
         namespace=namespace,
         ri=ri,
         oc=oc,
         vault_path=vault_path,
-        vault_client=vault_client,
+        vault_client=VaultClient(),
         secrets_reader=secrets_reader,
+        thread_pool_size=thread_pool_size,
+        dry_run=dry_run,
     )
+
+
+class VaultSecretsReconciler(SecretsReconciler):
+    def __init__(
+        self,
+        ri: ResourceInventory,
+        secrets_reader: SecretReaderBase,
+        vault_path: str,
+        thread_pool_size: int,
+        dry_run: bool,
+    ):
+        super().__init__(ri, secrets_reader, thread_pool_size, dry_run)
+        self.secrets_reader = secrets_reader
+        self.vault_path = vault_path
+
+    def _populate_secret_data(self, specs: Iterable[ExternalResourceSpec]) -> None:
+        threaded.run(self._read_secret, specs, self.thread_pool_size)
+
+    def _read_secret(self, spec: ExternalResourceSpec) -> None:
+        secret_path = f"{self.vault_path}/{spec.cluster_name}/{spec.namespace_name}/{spec.identifier}"
+        try:
+            logging.debug("Reading Secret %s", secret_path)
+            data = self.secrets_reader.read_all({"path": secret_path})
+            spec.metadata[SECRET_UPDATED_AT] = data[SECRET_UPDATED_AT]
+            del data[SECRET_UPDATED_AT]
+            spec.secret = data
+        except SecretNotFound:
+            logging.info("Error getting secret from vault, skipping. [%s]", secret_path)

reconcile/external_resources/state.py

@@ -35,6 +35,7 @@ class ResourceStatus(str, Enum):
     IN_PROGRESS: str = "IN_PROGRESS"
     DELETE_IN_PROGRESS: str = "DELETE_IN_PROGRESS"
     ERROR: str = "ERROR"
+    PENDING_SECRET_SYNC: str = "PENDING_SECRET_SYNC"
 
 
 class ExternalResourceState(BaseModel):
@@ -236,6 +237,15 @@ class ExternalResourcesStateDynamoDB:
     def get_all_resource_keys(self) -> set[ExternalResourceKey]:
         return {k for k in self.partial_resources.keys()}
 
+    def get_keys_by_status(
+        self, resource_status: ResourceStatus
+    ) -> set[ExternalResourceKey]:
+        return {
+            k
+            for k, v in self.partial_resources.items()
+            if v.resource_status == resource_status
+        }
+
     def update_resource_status(
         self, key: ExternalResourceKey, status: ResourceStatus
     ) -> None:

reconcile/gitlab_permissions.py

@@ -77,9 +77,11 @@ def share_project_with_group_members(
 
 
 def share_project_with_group(gl: GitLabApi, repos: list[str], dry_run: bool) -> None:
+    # get repos not owned by app-sre
+    non_app_sre_projects = {repo for repo in repos if "/app-sre/" not in repo}
     group_id, shared_projects = gl.get_group_id_and_shared_projects(APP_SRE_GROUP_NAME)
     shared_project_repos = {project["web_url"] for project in shared_projects}
-    repos_to_share = set(repos) - shared_project_repos
+    repos_to_share = non_app_sre_projects - shared_project_repos
     for repo in repos_to_share:
         gl.share_project_with_group(repo_url=repo, group_id=group_id, dry_run=dry_run)
 

reconcile/gql_definitions/external_resources/external_resources_modules.py

@@ -28,6 +28,7 @@ query ExternalResourcesModules {
         default_version
         reconcile_drift_interval_minutes
         reconcile_timeout_minutes
+        outputs_secret_sync
     }
 }
 """
@@ -47,6 +48,7 @@ class ExternalResourcesModuleV1(ConfiguredBaseModel):
     default_version: str = Field(..., alias="default_version")
     reconcile_drift_interval_minutes: str = Field(..., alias="reconcile_drift_interval_minutes")
     reconcile_timeout_minutes: str = Field(..., alias="reconcile_timeout_minutes")
+    outputs_secret_sync: bool = Field(..., alias="outputs_secret_sync")
 
 
 class ExternalResourcesModulesQueryData(ConfiguredBaseModel):

reconcile/gql_definitions/external_resources/external_resources_settings.py

@@ -35,6 +35,7 @@ query ExternalResourcesSettings {
         tf_state_bucket
         tf_state_region
         tf_state_dynamodb_table
+        vault_secrets_path
     }
 }
 """
@@ -67,6 +68,7 @@ class ExternalResourcesSettingsV1(ConfiguredBaseModel):
     tf_state_bucket: Optional[str] = Field(..., alias="tf_state_bucket")
     tf_state_region: Optional[str] = Field(..., alias="tf_state_region")
     tf_state_dynamodb_table: Optional[str] = Field(..., alias="tf_state_dynamodb_table")
+    vault_secrets_path: str = Field(..., alias="vault_secrets_path")
 
 
 class ExternalResourcesSettingsQueryData(ConfiguredBaseModel):

reconcile/utils/external_resource_spec.py

@@ -87,6 +87,11 @@ class ExternalResourceSpec:
     resource: MutableMapping[str, Any]
     namespace: Mapping[str, Any]
     secret: Mapping[str, str] = field(init=False, default_factory=lambda: {})
+    # Metadata is used for processing data that shuold not be included in the secret data
+    # e.g: ERV2 adds a updated_at attribute that acts as optimistic lock.
+    metadata: MutableMapping[str, str] = field(
+        init=False, compare=False, repr=False, hash=False, default_factory=lambda: {}
+    )
 
     @property
     def provider(self) -> str:

reconcile/utils/openshift_resource.py

@@ -14,6 +14,7 @@ from typing import (
 import semver
 from pydantic import BaseModel
 
+from reconcile.external_resources.meta import SECRET_UPDATED_AT
 from reconcile.utils.metrics import GaugeMetric
 
 SECRET_MAX_KEY_LENGTH = 253
@@ -526,6 +527,9 @@ class OpenshiftResource:
         # remove qontract specific params
         for a in QONTRACT_ANNOTATIONS:
             annotations.pop(a, None)
+
+        # Remove external resources annotation used for optimistic locking
+        annotations.pop(SECRET_UPDATED_AT, None)
         return body
 
     @staticmethod