qontract-reconcile 0.10.1rc995__py3-none-any.whl → 0.10.1rc997__py3-none-any.whl

{qontract_reconcile-0.10.1rc995.dist-info → qontract_reconcile-0.10.1rc997.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc995
+Version: 0.10.1rc997
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc995.dist-info → qontract_reconcile-0.10.1rc997.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=q96mwr2KeEQ5bpNniGlgIMZTxiuLSodcYfX-t
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=O1wFp52EyF538c6txaWBs8eMtUIy19gyHZ6VzJ6QXS8,3512
 reconcile/checkpoint.py,sha256=_JhMxrye5BgkRMxWYuf7Upli6XayPINKSsuo3ynHTRc,5010
-reconcile/cli.py,sha256=7-GNgB05b5MGrnEuD0resVNL3hOsQNSYcAAHrR68sS0,105755
+reconcile/cli.py,sha256=lLVw-FxEUR8zU6UAKZzJk7XwRbjsXPaGUAQqXuBYSaU,105825
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=rLh16BOlBOxTmJ8Si3wWyyEpmMlhh4Znx1Gc36qsmOc,4865
 reconcile/cluster_deployment_mapper.py,sha256=5gumAaRCcFXsabUJ1dnuUy9WrP_FEEM5JnOnE8ch9sE,2326
 reconcile/dashdotdb_base.py,sha256=l34QDu1G96_Ctnh7ZXdxXgSeCE93GQMdLAkWxmN6vDA,4775
@@ -187,14 +187,14 @@ reconcile/dynatrace_token_provider/ocm.py,sha256=iHMsgbsLs-dlrB9UXmWNDF7E4UDe49J
 reconcile/external_resources/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/external_resources/aws.py,sha256=JvjKaABy2Pg8u8Lq82Acv4zMvpE3_qGKes7OG-zlHOM,2956
 reconcile/external_resources/factories.py,sha256=DXgaLxoO87zZ76VOpRpu2GeYGhsbfOnOx5mrzgo4Gf4,4767
-reconcile/external_resources/integration.py,sha256=PJOpz-wUf7NzWvqCDalRWu5OxKjgi5RwiiB6HZfJs0k,5122
+reconcile/external_resources/integration.py,sha256=y1gJ16woMBC3J9qniMmS5y3lCkAs7V_ETZRUwjKqaO0,6628
 reconcile/external_resources/integration_secrets_sync.py,sha256=cMEZhgCvABAMf-DWF051L6CRnJQdfbsISA_b1xuS940,1670
-reconcile/external_resources/manager.py,sha256=5X9HjANMUGkZgC5RUA0r2TvZIbHw0UID1odn2QWrND4,14675
+reconcile/external_resources/manager.py,sha256=8lHOFyA_xF8thQXbmJp8zzA2-oJ0MUhfnCcFpkexwjU,15089
 reconcile/external_resources/meta.py,sha256=cMT9OsKcUY26qwEjlQ02EkorvOBNqWj0JVMwfJa3Mg0,634
 reconcile/external_resources/metrics.py,sha256=m2TIOao2N7pD6k45driFbBGVCC_N7ai44m-lLPfa5qk,454
 reconcile/external_resources/model.py,sha256=oXxJkjhV53lwwAuxUCBrjJ8aCJmQdgcKWv68ugJPK4k,7229
 reconcile/external_resources/reconciler.py,sha256=E50X_lnOD0OWYXMzyZld1P6dCFJFYjHGyICWff9bxlc,9323
-reconcile/external_resources/secrets_sync.py,sha256=WeoUANltYOjzr_Pn_pZ1ormGof9yRy2DiSB7LoPAQqM,15076
+reconcile/external_resources/secrets_sync.py,sha256=6n0oDPLjd9Ql0lf6zsr1AZw8A6EEe3yCzl20XodtgkE,16229
 reconcile/external_resources/state.py,sha256=bWq51xPK4-BHVXWsRu6Y-vn69yg9Dse4x1RNNF7qw84,9614
 reconcile/glitchtip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/glitchtip/integration.py,sha256=XtewM9nfTPLnPSpYebP50GrveYOnhTvKNq3seSvL6u8,8343
@@ -854,8 +854,8 @@ tools/test/test_qontract_cli.py,sha256=_D61RFGAN5x44CY1tYbouhlGXXABwYfxKSWSQx3Jr
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc995.dist-info/METADATA,sha256=O45HKoKnz-vWM-qm9KKu_RKXfZNVmO9fhTPKN4PKROk,2262
-qontract_reconcile-0.10.1rc995.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc995.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc995.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc995.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc997.dist-info/METADATA,sha256=zxGaYZt6G_4RJZBpfsYs23oXou1bP0MNIEkxLY83WxA,2262
+qontract_reconcile-0.10.1rc997.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc997.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc997.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc997.dist-info/RECORD,,

reconcile/cli.py

@@ -3714,20 +3714,20 @@ def deadmanssnitch(ctx):
 )
 def external_resources(
     ctx,
-    workers_cluster: str,
-    workers_namespace: str,
     dry_run_job_suffix: str,
     thread_pool_size: int,
+    workers_cluster: str,
+    workers_namespace: str,
 ):
     import reconcile.external_resources.integration
 
     run_integration(
         reconcile.external_resources.integration,
         ctx.obj,
-        dry_run_job_suffix,
-        thread_pool_size,
-        workers_cluster,
-        workers_namespace,
+        dry_run_job_suffix=dry_run_job_suffix,
+        thread_pool_size=thread_pool_size,
+        workers_cluster=workers_cluster,
+        workers_namespace=workers_namespace,
     )
 
 

reconcile/external_resources/integration.py

@@ -1,5 +1,6 @@
 import logging
 from collections.abc import Callable
+from typing import Any
 
 from reconcile.external_resources.manager import (
     ExternalResourcesInventory,
@@ -73,6 +74,62 @@ def get_aws_api(
     return AWSApi(aws_credentials)
 
 
+def create_er_manager(
+    aws_api: AWSApi,
+    workers_cluster: str | None,
+    workers_namespace: str | None,
+    thread_pool_size: int,
+    dry_run: bool,
+    dry_run_job_suffix: str,
+) -> ExternalResourcesManager:
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    er_settings = get_settings()[0]
+    m_inventory = load_module_inventory(get_modules())
+    namespaces = [ns for ns in get_namespaces() if ns.external_resources]
+    er_inventory = ExternalResourcesInventory(namespaces)
+
+    if not workers_cluster:
+        workers_cluster = er_settings.workers_cluster.name
+    if not workers_namespace:
+        workers_namespace = er_settings.workers_namespace.name
+
+    return ExternalResourcesManager(
+        thread_pool_size=thread_pool_size,
+        settings=er_settings,
+        secret_reader=secret_reader,
+        factories=setup_factories(
+            er_settings, m_inventory, er_inventory, secret_reader
+        ),
+        er_inventory=er_inventory,
+        module_inventory=m_inventory,
+        state_manager=ExternalResourcesStateDynamoDB(
+            aws_api=aws_api,
+            table_name=er_settings.state_dynamodb_table,
+        ),
+        reconciler=K8sExternalResourcesReconciler(
+            controller=build_job_controller(
+                integration=QONTRACT_INTEGRATION,
+                integration_version=QONTRACT_INTEGRATION_VERSION,
+                cluster=workers_cluster,
+                namespace=workers_namespace,
+                secret_reader=secret_reader,
+                dry_run=dry_run,
+            ),
+            dry_run=dry_run,
+            dry_run_job_suffix=dry_run_job_suffix,
+        ),
+        secrets_reconciler=build_incluster_secrets_reconciler(
+            workers_cluster,
+            workers_namespace,
+            secret_reader,
+            vault_path=er_settings.vault_secrets_path,
+            thread_pool_size=thread_pool_size,
+            dry_run=dry_run,
+        ),
+    )
+
+
 def run(
     dry_run: bool,
     dry_run_job_suffix: str,
@@ -83,9 +140,6 @@ def run(
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
     er_settings = get_settings()[0]
-    m_inventory = load_module_inventory(get_modules())
-    namespaces = [ns for ns in get_namespaces() if ns.external_resources]
-    er_inventory = ExternalResourcesInventory(namespaces)
 
     if not workers_cluster:
         workers_cluster = er_settings.workers_cluster.name
@@ -98,41 +152,14 @@ def run(
         region=er_settings.state_dynamodb_region,
         secret_reader=secret_reader,
     ) as aws_api:
-        er_mgr = ExternalResourcesManager(
-            thread_pool_size=thread_pool_size,
-            settings=er_settings,
-            secret_reader=secret_reader,
-            factories=setup_factories(
-                er_settings, m_inventory, er_inventory, secret_reader
-            ),
-            er_inventory=er_inventory,
-            module_inventory=m_inventory,
-            state_manager=ExternalResourcesStateDynamoDB(
-                aws_api=aws_api,
-                table_name=er_settings.state_dynamodb_table,
-            ),
-            reconciler=K8sExternalResourcesReconciler(
-                controller=build_job_controller(
-                    integration=QONTRACT_INTEGRATION,
-                    integration_version=QONTRACT_INTEGRATION_VERSION,
-                    cluster=workers_cluster,
-                    namespace=workers_namespace,
-                    secret_reader=secret_reader,
-                    dry_run=dry_run,
-                ),
-                dry_run=dry_run,
-                dry_run_job_suffix=dry_run_job_suffix,
-            ),
-            secrets_reconciler=build_incluster_secrets_reconciler(
-                workers_cluster,
-                workers_namespace,
-                secret_reader,
-                vault_path=er_settings.vault_secrets_path,
-                thread_pool_size=thread_pool_size,
-                dry_run=dry_run,
-            ),
+        er_mgr = create_er_manager(
+            aws_api,
+            workers_cluster,
+            workers_namespace,
+            thread_pool_size,
+            dry_run,
+            dry_run_job_suffix,
         )
-
         if dry_run:
             er_mgr.handle_dry_run_resources()
             if er_mgr.errors:
@@ -141,3 +168,25 @@ def run(
                     logging.error("ExternalResourceKey: %s, Error: %s" % (k, e))
         else:
             er_mgr.handle_resources()
+
+
+def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
+    er_settings = get_settings()[0]
+
+    with get_aws_api(
+        query_func=gql.get_api().query,
+        account_name=er_settings.state_dynamodb_account.name,
+        region=er_settings.state_dynamodb_region,
+        secret_reader=secret_reader,
+    ) as aws_api:
+        er_mgr = create_er_manager(
+            aws_api,
+            workers_cluster=kwargs["workers_cluster"],
+            workers_namespace=kwargs["workers_namespace"],
+            thread_pool_size=kwargs["thread_pool_size"],
+            dry_run=True,
+            dry_run_job_suffix=kwargs["dry_run_job_suffix"],
+        )
+        return er_mgr.get_all_reconciliations()

reconcile/external_resources/manager.py

@@ -116,7 +116,7 @@ class ExternalResourcesManager:
                     return ReconcileAction.APPLY_NOT_EXISTS
                 case ResourceStatus.ERROR:
                     return ReconcileAction.APPLY_ERROR
-                case ResourceStatus.CREATED:
+                case ResourceStatus.CREATED | ResourceStatus.PENDING_SECRET_SYNC:
                     if (
                         reconciliation.resource_hash
                         != state.reconciliation.resource_hash
@@ -154,6 +154,14 @@ class ExternalResourcesManager:
             )
         return reconcile
 
+    def get_all_reconciliations(self) -> dict[str, set[Reconciliation]]:
+        """Returns all reconciliations in a dict. Useful to return all data
+        from app-interface to make comparisions (early-exit)"""
+        return {
+            "desired": self._get_desired_objects_reconciliations(),
+            "deleted": self._get_deleted_objects_reconciliations(),
+        }
+
     def _get_desired_objects_reconciliations(self) -> set[Reconciliation]:
         r: set[Reconciliation] = set()
         for key, spec in self.er_inventory.items():

reconcile/external_resources/secrets_sync.py

@@ -95,6 +95,43 @@ class SecretHelper:
         return three_way_diff_using_hash(cmp_current, cmp_desired)
 
 
+class OutputSecretsFormatter:
+    """Class to format Module output keys/values into suitable values for K8s Secrets. It currently implements the same
+    behavior as Terraform-Resources."""
+
+    def __init__(self, secret_reader: SecretReaderBase) -> None:
+        self.secret_reader = secret_reader
+
+    def _key_must_be_populated(self, key: str) -> bool:
+        "Only keys containing '__' must be populated to Secrets"
+        return "__" in key
+
+    def _format_key(self, key: str) -> str:
+        if "__" not in key:
+            return key
+        k_split = key.split("__")
+        output_key = k_split[1]
+        if output_key.startswith("db"):
+            output_key = output_key.replace("db_", "db.")
+        return output_key
+
+    def _format_value(self, value: str) -> str:
+        decoded_value = base64.b64decode(value).decode("utf-8")
+        if decoded_value.startswith("__vault__:"):
+            _secret_ref = json.loads(decoded_value.replace("__vault__:", ""))
+            secret_ref = VaultSecret(**_secret_ref)
+            return self.secret_reader.read_secret(secret_ref)
+        else:
+            return decoded_value
+
+    def format(self, data: Mapping[str, str]) -> dict[str, str]:
+        return {
+            self._format_key(key): self._format_value(value)
+            for key, value in data.items()
+            if self._key_must_be_populated(key)
+        }
+
+
 class SecretsReconciler:
     def __init__(
         self,
@@ -281,6 +318,7 @@ class InClusterSecretsReconciler(SecretsReconciler):
         cluster: str,
         namespace: str,
         oc: OCCli,
+        output_secrets_formatter: OutputSecretsFormatter,
         thread_pool_size: int,
         dry_run: bool,
     ):
@@ -292,6 +330,7 @@ class InClusterSecretsReconciler(SecretsReconciler):
         self.source_secrets: list[str] = []
         self.vault_client = vault_client
         self.vault_path = vault_path
+        self.output_secrets_formatter = output_secrets_formatter
 
     def _get_spec_hash(self, spec: ExternalResourceSpec) -> str:
         secret_key = f"{spec.provision_provider}-{spec.provisioner_name}-{spec.provider}-{spec.identifier}"
@@ -313,21 +352,10 @@ class InClusterSecretsReconciler(SecretsReconciler):
         for secret in secrets:
             secret_name = secret["metadata"]["name"]
             spec = secrets_map[secret_name]
-            data = dict[str, str]()
-            for k, v in secret["data"].items():
-                decoded = base64.b64decode(v).decode("utf-8")
-
-                if decoded.startswith("__vault__:"):
-                    _secret_ref = json.loads(decoded.replace("__vault__:", ""))
-                    secret_ref = VaultSecret(**_secret_ref)
-                    data[k] = self.secrets_reader.read_secret(secret_ref)
-                else:
-                    data[k] = decoded
-
+            spec.secret = self.output_secrets_formatter.format(secret["data"])
             spec.metadata[SECRET_UPDATED_AT] = datetime.now(UTC).strftime(
                 SECRET_UPDATED_AT_TIMEFORMAT
             )
-            spec.secret = data
 
     def _delete_source_secret(self, spec: ExternalResourceSpec) -> None:
         secret_name = self._get_spec_outputs_secret_name(spec)
@@ -396,6 +424,7 @@ def build_incluster_secrets_reconciler(
         vault_path=vault_path,
         vault_client=VaultClient(),
         secrets_reader=secrets_reader,
+        output_secrets_formatter=OutputSecretsFormatter(secrets_reader),
         thread_pool_size=thread_pool_size,
         dry_run=dry_run,
     )