qontract-reconcile 0.10.1rc536__py3-none-any.whl → 0.10.1rc538__py3-none-any.whl

{qontract_reconcile-0.10.1rc536.dist-info → qontract_reconcile-0.10.1rc538.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc536
+Version: 0.10.1rc538
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc536.dist-info → qontract_reconcile-0.10.1rc538.dist-info}/RECORD

@@ -1,5 +1,5 @@
 reconcile/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-reconcile/acs_policies.py,sha256=Mop44Z5T0DxXghme13ZVvy1hr5KphZKzc_ZevxAJBQ0,8784
+reconcile/acs_policies.py,sha256=e_kVyP4rGm3fJzq10Anr8scZoPenlmTcoS5REK0d2T0,9144
 reconcile/acs_rbac.py,sha256=YoKu5wTRTtb3EGT0PV3r279LDgvw2ECb-0_0j4suScg,23032
 reconcile/aws_ami_share.py,sha256=eeu0TI3M5yyUaozyAq_aW3tir-9be4YFguOXvIvKHSo,3757
 reconcile/aws_ecr_image_pull_secrets.py,sha256=TGEc_0nv8oxV2HqA8VdcM4HHP-B1YqmNOOU6FPwVFTY,2328
@@ -9,7 +9,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=ek511mirANFglUXz_lrnIjmHZ48ZDJOx-53OFf-fSO4,84491
+reconcile/cli.py,sha256=LFsNTM36MINQ_iDqIpXlJPrUcLUL7BUEE4GpPG2LmaQ,85546
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -111,7 +111,7 @@ reconcile/terraform_cloudflare_dns.py,sha256=auU4bzeLwd4S8D8oqpqJbrCUoEdELXrgi7v
 reconcile/terraform_cloudflare_resources.py,sha256=EbQQaoDnZ7brvRCpbFtwlD7KLk2hDVNcjhrJGaAywEk,15023
 reconcile/terraform_cloudflare_users.py,sha256=1EbTHwJgiPkJpMP-Ag340QNgGK3mXn3dcC3DpLakudM,13987
 reconcile/terraform_repo.py,sha256=c0GZFuY3rCm6VHjHqYbsgOHrEkRWKF_1LrMThsn2XDw,16127
-reconcile/terraform_resources.py,sha256=x5Do4xBBhjJdIVRi0Gy4h-ryCCZ6kU7bT_iB0_mGing,17105
+reconcile/terraform_resources.py,sha256=3-q0WyCzBbfgvfDjCEKGp09bNGgxtbzUs9jEeoLr6u4,19176
 reconcile/terraform_tgw_attachments.py,sha256=_g7QSHM03YZzTU7O189S4HYtUn7WmwOBq67G4AieU24,15298
 reconcile/terraform_users.py,sha256=kXRUxCUchKCP2dbXXOzctynqMii4oyCP6bYZHQTrlTg,10202
 reconcile/terraform_vpc_peerings.py,sha256=rnDH1u93OyzrBM8Hib0HwSnlxZtx4ScRQaZAcn3mx-k,25402
@@ -375,7 +375,7 @@ reconcile/templates/jira-checkpoint-missinginfo.j2,sha256=c_Vvg-lEENsB3tgxm9B6Y9
 reconcile/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/test/conftest.py,sha256=rQousYrxUz-EwAIbsYO6bIwR1B4CrOz9y_zaUVo2lfI,4466
 reconcile/test/fixtures.py,sha256=9SDWAUlSd1rCx7z3GhULHcpr-I6FyCsXxaFAZIqYQsQ,591
-reconcile/test/test_acs_policies.py,sha256=2hioxQ1AlbtW-jerw9YiQ8aIb0YjFpzDFs1j6pvn-Nc,15133
+reconcile/test/test_acs_policies.py,sha256=hMnCX9KdLRKb53gYXK4JUR5yJwhczJRyyUPywDeLeLg,15716
 reconcile/test/test_acs_rbac.py,sha256=lvNd8GY0-GHzcOdOn13QWdrqbBXXKzNT7EEDHNH7cjM,28272
 reconcile/test/test_aggregated_list.py,sha256=iiWitQuNYC58aimWaiBoE4NROHjr1NCgQ91MnHEG_Ro,6412
 reconcile/test/test_amtool.py,sha256=vxRhGieeydMBOb9UI2ziMHjJa8puMeGNsUhGhy-yMnk,1032
@@ -440,7 +440,7 @@ reconcile/test/test_terraform_cloudflare_dns.py,sha256=aQTXX8Vr4h9aWvJZTnpZEhMGY
 reconcile/test/test_terraform_cloudflare_resources.py,sha256=NK_uktyWihkQ3gMN4bCaKerpi43CXAVYGIKTfcz05rY,13550
 reconcile/test/test_terraform_cloudflare_users.py,sha256=RAFtMMdqZha3jNnNNsqbNQQUDSqUzdoM63rCw7fs4Fo,27456
 reconcile/test/test_terraform_repo.py,sha256=soKFJfF8tWIimDs39RQl3Hnh-Od-bR4PfnEA2s1UprM,11552
-reconcile/test/test_terraform_resources.py,sha256=1ny_QSFuRjV9jxZY8EeT4NVJ5dMv7cLrEEIx_cBpjgk,9075
+reconcile/test/test_terraform_resources.py,sha256=O8kCYxGKqILbbCP86eJ3ESjatdPa1m3wQYXxrVfc_eo,15082
 reconcile/test/test_terraform_tgw_attachments.py,sha256=cAq6exc-K-jtLla1CZUZQzVnBkyDnIlL7jybnddhLKc,36861
 reconcile/test/test_terraform_users.py,sha256=Xn4y6EcxnNQb6XcPoOhz_Ikxmh9Nrsu88OM1scN9hzY,5434
 reconcile/test/test_terraform_vpc_peerings.py,sha256=ubcsKh0TrUIwuI1-W3ETIgzsFvzAyeoFmEJFC-IK6JY,20538
@@ -524,13 +524,14 @@ reconcile/utils/defer.py,sha256=SniUsbgOEs9Pa8JkecLu0F94O63yQPByKXaElDYe0FI,377
 reconcile/utils/differ.py,sha256=kJmUp9ZffFPSUEviaAw3s9c92ErwRJeHaRexGPai7wA,7643
 reconcile/utils/disabled_integrations.py,sha256=avdDsFyl_LdTsrPVzlcIhWzT_V4C4MXw1ZC__aOtluE,1126
 reconcile/utils/dnsutils.py,sha256=VX4gDQXpiMYVuT0pvNbzzSgfqmsWOM2qtjNQHUyIYF8,370
-reconcile/utils/early_exit_cache.py,sha256=EOZK-Z3w9SXooa42p-mwCa6LbIxZFJ7qWlyUcXsus7w,2344
+reconcile/utils/early_exit_cache.py,sha256=2uaiYIILA64D8mqqPj-GSvVBa80bN5XQRKz9x33iIpc,2304
 reconcile/utils/elasticsearch_exceptions.py,sha256=UY5Z3y2hw7T73sPJ6dHmUybegiIophrKFdTfdsOa6UY,379
 reconcile/utils/environ.py,sha256=VnW3zp6Un_UJn5BU4FU8RfhuqtZp0s-VeuuHnqC_WcQ,515
 reconcile/utils/exceptions.py,sha256=DwfnWUpVOotpP79RWZ2pycmG6nKCL00RBIeZLYkQPW4,635
 reconcile/utils/expiration.py,sha256=BXwKE50sNIV-Lszke97fxitNkLxYszoOLW1LBgp_yqg,1246
+reconcile/utils/extended_early_exit.py,sha256=gLWRtgzRB584iX4pVfGjfZxbIDd_AQPYu8MkQkReA3U,5688
 reconcile/utils/external_resource_spec.py,sha256=OGPKH3IKXgJszRTgE5U_QKgU-s4BHQnx97Lj-Krz46k,6655
-reconcile/utils/external_resources.py,sha256=eF9Wup8zbLWx56WoA0FlqguT6BRXRYgoyN3cbmpT_Dk,7443
+reconcile/utils/external_resources.py,sha256=a2CkJ3KLociYBnc_9F2VWfZGWMhzDl6fDNhwo2U-MWU,7501
 reconcile/utils/filtering.py,sha256=zZnHH0u0SaTDyzuFXZ_mREURGLvjEqQIQy4z-7QBVlc,419
 reconcile/utils/git.py,sha256=Qad7mfPuS9s7eKODeWSewehwSGgJPCbQuLda1qg_6GA,1522
 reconcile/utils/git_secrets.py,sha256=0wGNL5mvDtVPRuu3vEQgld1Am64gIDJHtmu1_ZKxMAI,1973
@@ -553,7 +554,7 @@ reconcile/utils/keycloak.py,sha256=UqOsAcHKmmIunroWB5YzC1fUZ3S3aq6L7trn6vLRmXY,3
 reconcile/utils/ldap_client.py,sha256=ho4veSrHqQWs0YhLFyKeD-duCwY8Nc5gUIA5qLENuMY,2502
 reconcile/utils/lean_terraform_client.py,sha256=zReyNPJbr2uOdrdh8Qfe-OZQBoRwxb5Za_ddeoUCYVk,4064
 reconcile/utils/make.py,sha256=QaEwucrzbl8-VHS66Wfdjfo0ubmAcvt_hZGpiGsKU50,231
-reconcile/utils/metrics.py,sha256=r93j_9WmsR4jC9uqgGrkzk9gycolhZhbsomHcpmJv8g,18297
+reconcile/utils/metrics.py,sha256=7nXdctmZ0UtGMHPpS3V55sfH4xpMPqdYaJ3JKAUc_sM,18474
 reconcile/utils/models.py,sha256=R3wF68yiaT0H2sQVMmHhHSTGI3JKKYEVjoizhHkySn8,4533
 reconcile/utils/oc.py,sha256=iAoM7zSm72zBOmCNW19ttAvm76tUgS26vlpb8SvVYwU,64058
 reconcile/utils/oc_connection_parameters.py,sha256=85slrnDigYwYmzhyceVkMElWzFArp4ge1d-fHXVqh0w,9729
@@ -580,8 +581,8 @@ reconcile/utils/sqs_gateway.py,sha256=gFl9DM4DmGnptuxTOe4lS3YTyE80eSAvK42ljS8h4d
 reconcile/utils/state.py,sha256=SAa6QLHu9lr0yqLCBy2AypNx1IPCJWlrRBrvlzAKsOU,14505
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
-reconcile/utils/terraform_client.py,sha256=V7AMQOEU4tvUOT-LQN2cXLqcphD5L93PMGMfurQQyPY,31753
-reconcile/utils/terrascript_aws_client.py,sha256=wA6AcAqbS2tIuqh8soMfpdpuf0IbAJWjONMmffJ-0vA,265467
+reconcile/utils/terraform_client.py,sha256=_jBriLBwU005bDxWlq7CRByOkVCfiH47oBzB0ArNAY8,31901
+reconcile/utils/terrascript_aws_client.py,sha256=Ht2akaR4bRERLoyN_Zh2JBbN1-p-ofXICqW-oXzGcFk,265789
 reconcile/utils/three_way_diff_strategy.py,sha256=nyqeQsLCoPI6e16k2CF3b9KNgQLU-rPf5RtfdUfVMwE,4468
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/unleash.py,sha256=1D56CsZfE3ShDtN3IErE1T2eeIwNmxhK-yYbCotJ99E,3601
@@ -590,7 +591,7 @@ reconcile/utils/vaultsecretref.py,sha256=3Ed2uBy36TzSvL0B-l4FoWQqB2SbBKDKEuUPIO6
 reconcile/utils/vcs.py,sha256=o1r0n_IrU2El75CED_6sjR2GZGM-exuWsj5F7jONaMU,6779
 reconcile/utils/acs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/acs/base.py,sha256=Qih-xZ3RBJZEE291iHHlv7lUY6ShcAvSj1PA3_aTTnM,2276
-reconcile/utils/acs/policies.py,sha256=utDmFKb6pbBN3W6JFxRrlr9yFwL3aQurGDywaFGM6w0,5196
+reconcile/utils/acs/policies.py,sha256=_jAz6cv8KRYtDsXjGoJgNbD8_9PUa5LSwwVlpK4A_cQ,5505
 reconcile/utils/acs/rbac.py,sha256=ugsLM9Pb7FbUbdq85E3VzXGMaB9ZovXob7tdWCxwqZ8,8808
 reconcile/utils/cloud_resource_best_practice/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 reconcile/utils/cloud_resource_best_practice/aws_rds.py,sha256=EvE6XKLsrZ531MJptKqPht2lOETrOjySTHXk6CzMgo0,2279
@@ -657,7 +658,7 @@ tools/app_interface_metrics_exporter.py,sha256=zkwkxdAUAxjdc-pzx2_oJXG25fo0Fnyd5
 tools/app_interface_reporter.py,sha256=upA-J-n-HXHKVDINRuMR7vTt-iJvQORKUVi9D3leQto,17738
 tools/glitchtip_access_reporter.py,sha256=oPBnk_YoDuljU3v0FaChzOwwnk4vap1xEE67QEjzdqs,2948
 tools/glitchtip_access_revalidation.py,sha256=8kbBJk04mkq28kWoRDDkfCGIF3GRg3pJrFAh1sW0dbk,2821
-tools/qontract_cli.py,sha256=bw7JVS9zaob2hvb-0MkGovTEK5HnxAy5ms-MEAU9WQU,103543
+tools/qontract_cli.py,sha256=mN5oXM_lmFmCGwoGU77qqO8nzlfKnNGlP6y-ILFyHIQ,103423
 tools/sd_app_sre_alert_report.py,sha256=e9vAdyenUz2f5c8-z-5WY0wv-SJ9aePKDH2r4IwB6pc,5063
 tools/cli_commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/cli_commands/gpg_encrypt.py,sha256=w8hl4jIEWk5wKbEFN6fVEOwUJGmdlvOqYodW3XSN7mU,4978
@@ -665,11 +666,11 @@ tools/sre_checkpoints/__init__.py,sha256=CDaDaywJnmRCLyl_NCcvxi-Zc0hTi_3OdwKiFOy
 tools/sre_checkpoints/util.py,sha256=zEDbGr18ZeHNQwW8pUsr2JRjuXIPz--WAGJxZo9sv_Y,894
 tools/test/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvftCWEEf-g1mfXOtgCog-g,1271
-tools/test/test_qontract_cli.py,sha256=d18KrdhtUGqoC7_kWZU128U0-VJEj-0rjFkLVufcI6I,2755
+tools/test/test_qontract_cli.py,sha256=se-YG_YVCWRFrnCPvBVHDBT_59CkbIoEni-4SJa8_MU,2755
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc536.dist-info/METADATA,sha256=BZPcvTsEmz0tR4NJmj-JZM7l_J2qAMSglos5FuteOCo,2349
-qontract_reconcile-0.10.1rc536.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
-qontract_reconcile-0.10.1rc536.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
-qontract_reconcile-0.10.1rc536.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc536.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc538.dist-info/METADATA,sha256=TkmlNiYqcISDkpP1_R0KR4m709lpVpeeJUAkfjVEFVc,2349
+qontract_reconcile-0.10.1rc538.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+qontract_reconcile-0.10.1rc538.dist-info/entry_points.txt,sha256=rTjAv28I_CHLM8ID3OPqMI_suoQ9s7tFbim4aYjn9kk,376
+qontract_reconcile-0.10.1rc538.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc538.dist-info/RECORD,,

reconcile/acs_policies.py

@@ -58,7 +58,10 @@ class AcsPoliciesIntegration(QontractReconcileIntegration[NoParams]):
         return self.qontract_integration.replace("_", "-")
 
     def _build_policy(
-        self, gql_policy: AcsPolicyV1, notifier_name_to_id: dict[str, str]
+        self,
+        gql_policy: AcsPolicyV1,
+        notifier_name_to_id: dict[str, str],
+        cluster_name_to_id: dict[str, str],
     ) -> Policy:
         conditions = [
             pc for c in gql_policy.conditions if (pc := self._build_policy_condition(c))
@@ -72,7 +75,7 @@ class AcsPoliciesIntegration(QontractReconcileIntegration[NoParams]):
             severity=f"{gql_policy.severity.upper()}_SEVERITY",  # align with acs api severity value format
             scope=sorted(
                 [
-                    Scope(cluster=cs.name, namespace="")
+                    Scope(cluster=cluster_name_to_id[cs.name], namespace="")
                     for cs in cast(
                         gql_acs_policies.AcsPolicyScopeClusterV1,
                         gql_policy.scope,
@@ -83,7 +86,9 @@ class AcsPoliciesIntegration(QontractReconcileIntegration[NoParams]):
             if gql_policy.scope.level == "cluster"
             else sorted(
                 [
-                    Scope(cluster=ns.cluster.name, namespace=ns.name)
+                    Scope(
+                        cluster=cluster_name_to_id[ns.cluster.name], namespace=ns.name
+                    )
                     for ns in cast(
                         gql_acs_policies.AcsPolicyScopeNamespaceV1,
                         gql_policy.scope,
@@ -158,7 +163,10 @@ class AcsPoliciesIntegration(QontractReconcileIntegration[NoParams]):
                 return None
 
     def get_desired_state(
-        self, query_func: Callable, notifiers: list[AcsPolicyApi.NotifierIdentifiers]
+        self,
+        query_func: Callable,
+        notifiers: list[AcsPolicyApi.NotifierIdentifiers],
+        clusters: list[AcsPolicyApi.ClusterIdentifiers],
     ) -> list[Policy]:
         """
         Get desired ACS security policies and convert to acs api policy object format
@@ -167,8 +175,9 @@ class AcsPoliciesIntegration(QontractReconcileIntegration[NoParams]):
         :return: list of utils.acs.policies.Policy derived from acs-policy-1 definitions
         """
         notifier_name_to_id = {n.name: n.id for n in notifiers}
+        cluster_name_to_id = {c.name: c.id for c in clusters}
         return [
-            self._build_policy(gql_policy, notifier_name_to_id)
+            self._build_policy(gql_policy, notifier_name_to_id, cluster_name_to_id)
             for gql_policy in gql_acs_policies.query(query_func=query_func).acs_policies
             or []
         ]
@@ -225,7 +234,8 @@ class AcsPoliciesIntegration(QontractReconcileIntegration[NoParams]):
             instance={"url": instance.url, "token": token[instance.credentials.field]}
         ) as acs_api:
             notifiers = acs_api.list_notifiers()
-            desired = self.get_desired_state(gqlapi.query, notifiers)
+            clusters = acs_api.list_clusters()
+            desired = self.get_desired_state(gqlapi.query, notifiers, clusters)
             current = acs_api.get_custom_policies()
             self.reconcile(
                 desired=desired, current=current, acs=acs_api, dry_run=dry_run

reconcile/cli.py

@@ -507,6 +507,30 @@ def trigger_integration(function):
     return function
 
 
+def enable_extended_early_exit(function):
+    return click.option(
+        "--enable-extended-early-exit/--no-enable-extended-early-exit",
+        default=False,
+        help="enable extended early exit.",
+    )(function)
+
+
+def extended_early_exit_cache_ttl_seconds(function):
+    return click.option(
+        "--extended-early-exit-cache-ttl-seconds",
+        default=3600,
+        help="TTL of extended early exit cache in seconds.",
+    )(function)
+
+
+def log_cached_log_output(function):
+    return click.option(
+        "--log-cached-log-output/--no-log-cached-log-output",
+        default=False,
+        help="log the cached log output.",
+    )(function)
+
+
 def register_faulthandler(fileobj=sys.__stderr__):
     if fileobj:
         if not faulthandler.is_enabled():
@@ -1738,6 +1762,9 @@ def terraform_repo(ctx, output_file, gitlab_project_id, gitlab_merge_request_id)
 @enable_deletion(default=False)
 @account_name_multiple
 @exclude_aws_accounts
+@enable_extended_early_exit
+@extended_early_exit_cache_ttl_seconds
+@log_cached_log_output
 @click.option(
     "--light/--full",
     default=False,
@@ -1755,6 +1782,9 @@ def terraform_resources(
     vault_output_path,
     account_name,
     exclude_accounts,
+    enable_extended_early_exit,
+    extended_early_exit_cache_ttl_seconds,
+    log_cached_log_output,
 ):
     import reconcile.terraform_resources
 
@@ -1772,6 +1802,9 @@ def terraform_resources(
         vault_output_path,
         account_name=account_name,
         exclude_accounts=exclude_accounts,
+        enable_extended_early_exit=enable_extended_early_exit,
+        extended_early_exit_cache_ttl_seconds=extended_early_exit_cache_ttl_seconds,
+        log_cached_log_output=log_cached_log_output,
     )
 
 

reconcile/terraform_resources.py

@@ -1,6 +1,4 @@
 import logging
-import shutil
-import sys
 from collections.abc import (
     Callable,
     Iterable,
@@ -11,6 +9,7 @@ from typing import (
     Collection,
     Optional,
     Sequence,
+    TypedDict,
     cast,
 )
 
@@ -33,6 +32,10 @@ from reconcile.typed_queries.terraform_namespaces import get_namespaces
 from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.defer import defer
+from reconcile.utils.extended_early_exit import (
+    ExtendedEarlyExitRunnerResult,
+    extended_early_exit_run,
+)
 from reconcile.utils.external_resource_spec import (
     ExternalResourceSpec,
     ExternalResourceSpecInventory,
@@ -52,10 +55,12 @@ from reconcile.utils.ocm import OCMMap
 from reconcile.utils.openshift_resource import OpenshiftResource as OR
 from reconcile.utils.openshift_resource import ResourceInventory
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
-from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.secret_reader import SecretReaderBase, create_secret_reader
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.terraform_client import TerraformClient as Terraform
+from reconcile.utils.terrascript_aws_client import TerrascriptClient
 from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
+from reconcile.utils.unleash import get_feature_toggle_state
 from reconcile.utils.vault import (
     VaultClient,
     _VaultClient,
@@ -117,18 +122,14 @@ def populate_oc_resources(
 
 
 def fetch_current_state(
-    dry_run: bool,
     namespaces: Iterable[NamespaceV1],
     thread_pool_size: int,
     internal: Optional[bool],
     use_jump_host: bool,
     account_names: Optional[Iterable[str]],
-) -> tuple[ResourceInventory, Optional[OCMap]]:
+    secret_reader: SecretReaderBase,
+) -> tuple[ResourceInventory, OCMap]:
     ri = ResourceInventory()
-    if dry_run:
-        return ri, None
-    vault_settings = get_app_interface_vault_settings()
-    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
     oc_map = init_oc_map_from_namespaces(
         namespaces=namespaces,
         integration=QONTRACT_INTEGRATION,
@@ -172,64 +173,74 @@ def init_working_dirs(
 
 
 def filter_accounts_by_name(
-    accounts: Iterable[Mapping[str, Any]], filter: Iterable[str]
-) -> Collection[Mapping[str, Any]]:
-    return [ac for ac in accounts if ac["name"] in filter]
+    accounts: Iterable[dict[str, Any]], names: Iterable[str]
+) -> list[dict[str, Any]]:
+    return [ac for ac in accounts if ac["name"] in names]
 
 
 def exclude_accounts_by_name(
-    accounts: Iterable[Mapping[str, Any]], filter: Iterable[str]
-) -> Collection[Mapping[str, Any]]:
-    return [ac for ac in accounts if ac["name"] not in filter]
+    accounts: Iterable[dict[str, Any]], names: Iterable[str]
+) -> list[dict[str, Any]]:
+    return [ac for ac in accounts if ac["name"] not in names]
 
 
 def validate_account_names(
     accounts: Collection[Mapping[str, Any]], names: Collection[str]
 ) -> None:
-    if len(accounts) != len(names):
-        missing_names = set(names) - {a["name"] for a in accounts}
+    if missing_names := set(names) - {a["name"] for a in accounts}:
         raise ValueError(
             f"Accounts {missing_names} were provided as arguments, but not found in app-interface. Check your input for typos or for missing AWS account definitions."
         )
 
 
-def setup(
+def get_aws_accounts(
     dry_run: bool,
-    print_to_file: Optional[str],
-    thread_pool_size: int,
-    internal: Optional[bool],
-    use_jump_host: bool,
     include_accounts: Optional[Collection[str]],
     exclude_accounts: Optional[Collection[str]],
-) -> tuple[
-    ResourceInventory, Optional[OCMap], Terraform, ExternalResourceSpecInventory
-]:
+) -> list[dict[str, Any]]:
+    if exclude_accounts and not dry_run:
+        message = "--exclude-accounts is only supported in dry-run mode"
+        logging.error(message)
+        raise ExcludeAccountsAndDryRunException(message)
+
+    if exclude_accounts and include_accounts:
+        message = "Using --exclude-accounts and --account-name at the same time is not allowed"
+        logging.error(message)
+        raise ExcludeAccountsAndAccountNameException(message)
+
+    # If we are not running in dry run we don't want to run with more than one account
+    if include_accounts and len(include_accounts) > 1 and not dry_run:
+        message = "Running with multiple accounts is only supported in dry-run mode"
+        logging.error(message)
+        raise MultipleAccountNamesInDryRunException(message)
+
     accounts = queries.get_aws_accounts(terraform_state=True)
-    if not include_accounts and exclude_accounts:
-        excluding = filter_accounts_by_name(accounts, exclude_accounts)
-        validate_account_names(excluding, exclude_accounts)
-        accounts = exclude_accounts_by_name(accounts, exclude_accounts)
-        if len(accounts) == 0:
+
+    if exclude_accounts:
+        validate_account_names(accounts, exclude_accounts)
+        filtered_accounts = exclude_accounts_by_name(accounts, exclude_accounts)
+        if not filtered_accounts:
             raise ValueError("You have excluded all aws accounts, verify your input")
-        account_names = tuple(ac["name"] for ac in accounts)
-    elif include_accounts:
-        accounts = filter_accounts_by_name(accounts, include_accounts)
+        return filtered_accounts
+
+    if include_accounts:
         validate_account_names(accounts, include_accounts)
-    account_names = tuple(a["name"] for a in accounts)
-    settings = queries.get_app_interface_settings()
+        return filter_accounts_by_name(accounts, include_accounts)
 
-    # build a resource inventory for all the kube secrets managed by the
-    # app-interface managed terraform resources
-    tf_namespaces = get_tf_namespaces(account_names)
-    if not tf_namespaces:
-        logging.warning(
-            "No terraform namespaces found, consider disabling this integration, account names: "
-            f"{', '.join(account_names)}"
-        )
-    ri, oc_map = fetch_current_state(
-        dry_run, tf_namespaces, thread_pool_size, internal, use_jump_host, account_names
-    )
+    return accounts
+
+
+def setup(
+    accounts: list[dict[str, Any]],
+    account_names: set[str],
+    tf_namespaces: list[NamespaceV1],
+    print_to_file: Optional[str],
+    thread_pool_size: int,
+) -> tuple[Terraform, TerrascriptClient, SecretReaderBase]:
+    vault_settings = get_app_interface_vault_settings()
+    secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
+    settings = queries.get_app_interface_settings()
     # initialize terrascript (scripting engine to generate terraform manifests)
     ts, working_dirs = init_working_dirs(accounts, thread_pool_size, settings=settings)
 
@@ -260,7 +271,7 @@ def setup(
     ts.populate_resources(ocm_map=ocm_map)
     ts.dump(print_to_file, existing_dirs=working_dirs)
 
-    return ri, oc_map, tf, ts.resource_spec_inventory
+    return tf, ts, secret_reader
 
 
 def filter_tf_namespaces(
@@ -290,21 +301,6 @@ def filter_tf_namespaces(
     return tf_namespaces
 
 
-def cleanup_and_exit(
-    tf: Optional[Terraform] = None,
-    status: bool = False,
-    working_dirs: Optional[Mapping[str, str]] = None,
-) -> None:
-    if working_dirs is None:
-        working_dirs = {}
-    if tf is None:
-        for wd in working_dirs.values():
-            shutil.rmtree(wd)
-    else:
-        tf.cleanup()
-    sys.exit(status)
-
-
 @retry()
 def write_outputs_to_vault(
     vault_path: str, resource_specs: ExternalResourceSpecInventory
@@ -366,65 +362,125 @@ def run(
     vault_output_path: str = "",
     account_name: Optional[Sequence[str]] = None,
     exclude_accounts: Optional[Sequence[str]] = None,
+    enable_extended_early_exit: bool = False,
+    extended_early_exit_cache_ttl_seconds: int = 3600,
+    log_cached_log_output: bool = False,
     defer: Optional[Callable] = None,
 ) -> None:
-    if exclude_accounts and not dry_run:
-        message = "--exclude-accounts is only supported in dry-run mode"
-        logging.error(message)
-        raise ExcludeAccountsAndDryRunException(message)
-
-    if exclude_accounts and account_name:
-        message = "Using --exclude-accounts and --account-name at the same time is not allowed"
-        logging.error(message)
-        raise ExcludeAccountsAndAccountNameException(message)
-
     # account_name is a tuple of account names for more detail go to
     # https://click.palletsprojects.com/en/8.1.x/options/#multiple-options
-    account_names = account_name
-
-    # acc_name will prevent type error since account_name is not a str
-    acc_name: Optional[str] = account_names[0] if account_names else None
-
-    # If we are not running in dry run we don't want to run with more than one account
-    if account_names and len(account_names) > 1 and not dry_run:
-        message = "Running with multiple accounts is only supported in dry-run mode"
-        logging.error(message)
-        raise MultipleAccountNamesInDryRunException(message)
+    accounts = get_aws_accounts(dry_run, account_name, exclude_accounts)
+    account_names = {a["name"] for a in accounts}
+    tf_namespaces = get_tf_namespaces(account_names)
+    if not tf_namespaces:
+        logging.warning(
+            "No terraform namespaces found, consider disabling this integration, account names: "
+            f"{', '.join(account_names)}"
+        )
 
-    ri, oc_map, tf, resource_specs = setup(
-        dry_run,
+    tf, ts, secret_reader = setup(
+        accounts,
+        account_names,
+        tf_namespaces,
         print_to_file,
         thread_pool_size,
-        internal,
-        use_jump_host,
-        account_names,
-        exclude_accounts,
     )
-    publish_metrics(resource_specs, QONTRACT_INTEGRATION)
+    if defer:
+        defer(tf.cleanup)
 
-    if not dry_run and oc_map and defer:
-        defer(oc_map.cleanup)
+    publish_metrics(ts.resource_spec_inventory, QONTRACT_INTEGRATION)
 
     if print_to_file:
-        cleanup_and_exit(tf)
-    if tf is None:
-        err = True
-        cleanup_and_exit(tf, err)
+        return
+
+    runner_params: RunnerParams = dict(
+        accounts=accounts,
+        account_names=account_names,
+        tf_namespaces=tf_namespaces,
+        tf=tf,
+        ts=ts,
+        secret_reader=secret_reader,
+        dry_run=dry_run,
+        enable_deletion=enable_deletion,
+        thread_pool_size=thread_pool_size,
+        internal=internal,
+        use_jump_host=use_jump_host,
+        light=light,
+        vault_output_path=vault_output_path,
+        defer=defer,
+    )
 
+    if enable_extended_early_exit and get_feature_toggle_state(
+        "terraform-resources-extended-early-exit",
+        default=False,
+    ):
+        extended_early_exit_run(
+            integration=QONTRACT_INTEGRATION,
+            integration_version=QONTRACT_INTEGRATION_VERSION,
+            dry_run=dry_run,
+            cache_source=ts.terraform_configurations(),
+            ttl_seconds=extended_early_exit_cache_ttl_seconds,
+            logger=logging.getLogger(),
+            runner=runner,
+            runner_params=runner_params,
+            secret_reader=secret_reader,
+            log_cached_log_output=log_cached_log_output,
+        )
+    else:
+        runner(**runner_params)
+
+
+class RunnerParams(TypedDict):
+    accounts: list[dict[str, Any]]
+    account_names: set[str]
+    tf_namespaces: list[NamespaceV1]
+    tf: Terraform
+    ts: Terrascript
+    secret_reader: SecretReaderBase
+    dry_run: bool
+    enable_deletion: bool
+    thread_pool_size: int
+    internal: Optional[bool]
+    use_jump_host: bool
+    light: bool
+    vault_output_path: str
+    defer: Optional[Callable]
+
+
+def runner(
+    accounts: list[dict[str, Any]],
+    account_names: set[str],
+    tf_namespaces: list[NamespaceV1],
+    tf: Terraform,
+    ts: Terrascript,
+    secret_reader: SecretReaderBase,
+    dry_run: bool,
+    enable_deletion: bool = False,
+    thread_pool_size: int = 10,
+    internal: Optional[bool] = None,
+    use_jump_host: bool = True,
+    light: bool = False,
+    vault_output_path: str = "",
+    defer: Optional[Callable] = None,
+) -> ExtendedEarlyExitRunnerResult:
     if not light:
         disabled_deletions_detected, err = tf.plan(enable_deletion)
         if err:
-            cleanup_and_exit(tf, err)
+            raise RuntimeError("Terraform plan has errors")
         if disabled_deletions_detected:
-            cleanup_and_exit(tf, disabled_deletions_detected)
+            raise RuntimeError("Terraform plan has disabled deletions detected")
 
     if dry_run:
-        cleanup_and_exit(tf)
+        return ExtendedEarlyExitRunnerResult(
+            payload=ts.terraform_configurations(),
+            applied_count=0,
+        )
 
-    if not light and tf.should_apply:
+    acc_name = accounts[0]["name"] if accounts else None
+    if not light and tf.should_apply():
         err = tf.apply()
         if err:
-            cleanup_and_exit(tf, err)
+            raise RuntimeError("Terraform apply has errors")
 
         if defer:
             defer(
@@ -437,26 +493,41 @@ def run(
 
     # refresh output data after terraform apply
     tf.populate_terraform_output_secrets(
-        resource_specs=resource_specs, init_rds_replica_source=True
+        resource_specs=ts.resource_spec_inventory, init_rds_replica_source=True
     )
+
+    ri, oc_map = fetch_current_state(
+        tf_namespaces,
+        thread_pool_size,
+        internal,
+        use_jump_host,
+        account_names,
+        secret_reader=secret_reader,
+    )
+    if defer:
+        defer(oc_map.cleanup)
     # populate the resource inventory with latest output data
-    populate_desired_state(ri, resource_specs)
+    populate_desired_state(ri, ts.resource_spec_inventory)
 
     ob.publish_metrics(ri, QONTRACT_INTEGRATION)
-    actions = []
-    if oc_map:
-        actions = ob.realize_data(
-            dry_run, oc_map, ri, thread_pool_size, caller=acc_name
-        )
+    actions = ob.realize_data(
+        dry_run,
+        oc_map,
+        ri,
+        thread_pool_size,
+        caller=acc_name,
+    )
 
     if actions and vault_output_path:
-        write_outputs_to_vault(vault_output_path, resource_specs)
+        write_outputs_to_vault(vault_output_path, ts.resource_spec_inventory)
 
     if ri.has_error_registered():
-        err = True
-        cleanup_and_exit(tf, err)
+        raise RuntimeError("Resource inventory has errors registered")
 
-    cleanup_and_exit(tf)
+    return ExtendedEarlyExitRunnerResult(
+        payload=ts.terraform_configurations(),
+        applied_count=tf.apply_count + len(actions),
+    )
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]: