qontract-reconcile 0.10.1rc781__py3-none-any.whl → 0.10.1rc783__py3-none-any.whl

{qontract_reconcile-0.10.1rc781.dist-info → qontract_reconcile-0.10.1rc783.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc781
+Version: 0.10.1rc783
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc781.dist-info → qontract_reconcile-0.10.1rc783.dist-info}/RECORD

@@ -10,7 +10,7 @@ reconcile/aws_iam_password_reset.py,sha256=NwErtrqgBiXr7eGCAHdtGGOx0S7-4JnSc29Ie
 reconcile/aws_support_cases_sos.py,sha256=Jk6_XjDeJSYxgRGqcEAOcynt9qJF2r5HPIPcSKmoBv8,2974
 reconcile/blackbox_exporter_endpoint_monitoring.py,sha256=W_VJagnsJR1v5oqjlI3RJJE0_nhtJ0m81RS8zWA5u5c,3538
 reconcile/checkpoint.py,sha256=R2WFXUXLTB4sWMi4GeA4eegsuf_1-Q4vH8M0Toh3Ij4,5036
-reconcile/cli.py,sha256=5x9LADlVvoEB6Wen6NZeLZnGIj439voLz5L6ccxRxds,99574
+reconcile/cli.py,sha256=uMjslj9XkN1UFJ3k-CuHlTz4MeyXMk6gK1BkWfg6Kt4,99967
 reconcile/closedbox_endpoint_monitoring_base.py,sha256=SMhkcQqprWvThrIJa3U_3uh5w1h-alleW1QnCJFY4Qw,4909
 reconcile/cluster_deployment_mapper.py,sha256=2Ah-nu-Mdig0pjuiZl_XLrmVAjYzFjORR3dMlCgkmw0,2352
 reconcile/dashdotdb_base.py,sha256=a5aPLVxyqPSbjdB0Ty-uliOtxwvEbbEljHJKxdK3-Zk,4813
@@ -114,7 +114,7 @@ reconcile/terraform_cloudflare_resources.py,sha256=EbQQaoDnZ7brvRCpbFtwlD7KLk2hD
 reconcile/terraform_cloudflare_users.py,sha256=1EbTHwJgiPkJpMP-Ag340QNgGK3mXn3dcC3DpLakudM,13987
 reconcile/terraform_repo.py,sha256=xkp5EiRQ7cz-IquXiBq5plvBQLf910tqywKK0B_QlPM,16271
 reconcile/terraform_resources.py,sha256=BN8XuJwjOt1ztruEAHydkd0YiBlb3fHZ7n0snZtRhck,19356
-reconcile/terraform_tgw_attachments.py,sha256=k9Lf0ST65gmI6aUV6HnvxSGcKL7MGx_lN22OXuRGH9Y,16224
+reconcile/terraform_tgw_attachments.py,sha256=S5IP7RmnVuVSOIPXFlUX2srGBrBBeH2uM4J6ewHCvOQ,18797
 reconcile/terraform_users.py,sha256=9rgbM572LfmOSnV3uCP20G_Cw6T7due94g8rhhiz904,10225
 reconcile/terraform_vpc_peerings.py,sha256=rnDH1u93OyzrBM8Hib0HwSnlxZtx4ScRQaZAcn3mx-k,25402
 reconcile/vault_replication.py,sha256=79GZ_kCimPoQcxkdhkWTQxPOAa46E0mNhf05s_Mk5so,17385
@@ -523,7 +523,7 @@ reconcile/test/test_terraform_cloudflare_resources.py,sha256=NK_uktyWihkQ3gMN4bC
 reconcile/test/test_terraform_cloudflare_users.py,sha256=RAFtMMdqZha3jNnNNsqbNQQUDSqUzdoM63rCw7fs4Fo,27456
 reconcile/test/test_terraform_repo.py,sha256=j9mLfwiK707U2KRxYpvzAbOYywk__pL9SXAH3xbP1t8,12184
 reconcile/test/test_terraform_resources.py,sha256=EFCqPI5_G8hPRh1zmnU91o8wMeT2qK1CabDUa_X1rSk,15283
-reconcile/test/test_terraform_tgw_attachments.py,sha256=cAq6exc-K-jtLla1CZUZQzVnBkyDnIlL7jybnddhLKc,36861
+reconcile/test/test_terraform_tgw_attachments.py,sha256=rHZHUtDxewpKsRj3nfm2bZ2JoQ4CWiN2nQM-SWkMopg,41047
 reconcile/test/test_terraform_users.py,sha256=XOAfGvITCJPI1LTlISmHbA4ONMQMkxYUMTsny7pQCFw,4319
 reconcile/test/test_terraform_vpc_peerings.py,sha256=ubcsKh0TrUIwuI1-W3ETIgzsFvzAyeoFmEJFC-IK6JY,20538
 reconcile/test/test_terraform_vpc_peerings_build_desired_state.py,sha256=DAfpb12I0PlqnuVUHK2vh4LH4d1OylT3H2GE_3TGZZI,47852
@@ -785,8 +785,8 @@ tools/test/test_app_interface_metrics_exporter.py,sha256=SX7qL3D1SIRKFo95FoQztvf
 tools/test/test_qontract_cli.py,sha256=w2l4BHB09k1d-BGJ1jBUNCqDv7zkqYrMHojQXg-21kQ,4155
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc781.dist-info/METADATA,sha256=U8z82KU9PoP9inuu8tIIXkd2XaKyf1VWzyrbI2seSYQ,2382
-qontract_reconcile-0.10.1rc781.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-qontract_reconcile-0.10.1rc781.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
-qontract_reconcile-0.10.1rc781.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc781.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc783.dist-info/METADATA,sha256=SwZsjjF0UyVKhYaUdO-pvRDKF2nPhywOgRc0OcA-XLA,2382
+qontract_reconcile-0.10.1rc783.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+qontract_reconcile-0.10.1rc783.dist-info/entry_points.txt,sha256=rIxI5zWtHNlfpDeq1a7pZXAPoqf7HG32KMTN3MeWK_8,429
+qontract_reconcile-0.10.1rc783.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc783.dist-info/RECORD,,

reconcile/cli.py

@@ -2318,6 +2318,9 @@ def vpc_peerings_validator(ctx):
 @threaded()
 @binary(["terraform", "git"])
 @binary_version("terraform", ["version"], TERRAFORM_VERSION_REGEX, TERRAFORM_VERSION)
+@enable_extended_early_exit
+@extended_early_exit_cache_ttl_seconds
+@log_cached_log_output
 @enable_deletion(default=False)
 @account_name
 @click.pass_context
@@ -2327,6 +2330,9 @@ def terraform_tgw_attachments(
     enable_deletion,
     thread_pool_size,
     account_name,
+    enable_extended_early_exit,
+    extended_early_exit_cache_ttl_seconds,
+    log_cached_log_output,
 ):
     import reconcile.terraform_tgw_attachments
 
@@ -2339,6 +2345,9 @@ def terraform_tgw_attachments(
         enable_deletion,
         thread_pool_size,
         account_name=account_name,
+        enable_extended_early_exit=enable_extended_early_exit,
+        extended_early_exit_cache_ttl_seconds=extended_early_exit_cache_ttl_seconds,
+        log_cached_log_output=log_cached_log_output,
     )
 
 

reconcile/terraform_tgw_attachments.py

@@ -8,6 +8,7 @@ from collections.abc import (
 from typing import (
     Any,
     Optional,
+    TypedDict,
     Union,
     cast,
 )
@@ -40,15 +41,20 @@ from reconcile.utils import gql
 from reconcile.utils.aws_api import AWSApi
 from reconcile.utils.defer import defer
 from reconcile.utils.disabled_integrations import integration_is_enabled
+from reconcile.utils.extended_early_exit import (
+    ExtendedEarlyExitRunnerResult,
+    extended_early_exit_run,
+)
 from reconcile.utils.ocm import (
     OCM,
     OCMMap,
 )
 from reconcile.utils.runtime.integration import DesiredStateShardConfig
-from reconcile.utils.secret_reader import create_secret_reader
+from reconcile.utils.secret_reader import SecretReaderBase, create_secret_reader
 from reconcile.utils.semver_helper import make_semver
 from reconcile.utils.terraform_client import TerraformClient as Terraform
 from reconcile.utils.terrascript_aws_client import TerrascriptClient as Terrascript
+from reconcile.utils.unleash import get_feature_toggle_state
 
 QONTRACT_INTEGRATION = "terraform_tgw_attachments"
 QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
@@ -103,6 +109,17 @@ class DesiredStateDataSource(BaseModel):
     accounts: list[AWSAccountV1]
 
 
+class CacheSource(TypedDict):
+    terraform_configurations: dict[str, str]
+
+
+class RunnerParams(TypedDict):
+    terraform_client: Terraform
+    terrascript_client: Terrascript
+    dry_run: bool
+    enable_deletion: bool
+
+
 def _build_desired_state_tgw_attachments(
     clusters: Iterable[ClusterV1],
     ocm_map: Optional[OCMMap],
@@ -402,37 +419,19 @@ def _fetch_desired_state_data_source(
     )
 
 
-@defer
-def run(
-    dry_run: bool,
-    print_to_file: Optional[str] = None,
-    enable_deletion: bool = False,
+def setup(
+    account_name: Optional[str],
+    desired_state_data_source: DesiredStateDataSource,
+    tgw_accounts: list[dict[str, Any]],
     thread_pool_size: int = 10,
-    account_name: Optional[str] = None,
-    defer: Optional[Callable] = None,
-) -> None:
-    desired_state_data_source = _fetch_desired_state_data_source(account_name)
+    print_to_file: Optional[str] = None,
+) -> tuple[SecretReaderBase, AWSApi, Terraform, Terrascript]:
     tgw_clusters = desired_state_data_source.clusters
     all_accounts = [a.dict(by_alias=True) for a in desired_state_data_source.accounts]
     account_by_name = {a["name"]: a for a in all_accounts}
-    tgw_accounts = [
-        a.dict(by_alias=True)
-        for a in _filter_tgw_accounts(desired_state_data_source.accounts, tgw_clusters)
-        if not account_name or account_name == a.name
-    ]
-
-    if not tgw_accounts:
-        logging.warning(
-            f"No participating AWS accounts found, consider disabling this integration, account name: {account_name}"
-        )
-        return
-
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(vault_settings.vault)
     aws_api = AWSApi(1, all_accounts, secret_reader=secret_reader, init_users=False)
-    if defer:
-        defer(aws_api.cleanup)
-
     ocm_map = _build_ocm_map(desired_state_data_source.clusters, vault_settings)
     desired_state, err = _build_desired_state_tgw_attachments(
         desired_state_data_source.clusters,
@@ -463,10 +462,6 @@ def run(
         desired_state,
         print_to_file,
     )
-
-    if print_to_file:
-        return
-
     tf = Terraform(
         QONTRACT_INTEGRATION,
         QONTRACT_INTEGRATION_VERSION,
@@ -476,22 +471,95 @@ def run(
         thread_pool_size,
         aws_api,
     )
+    return secret_reader, aws_api, tf, ts
 
-    if defer:
-        defer(tf.cleanup)
 
-    disabled_deletions_detected, err = tf.plan(enable_deletion)
+def runner(
+    dry_run: bool,
+    terraform_client: Terraform,
+    terrascript_client: Terrascript,
+    enable_deletion: bool = False,
+) -> ExtendedEarlyExitRunnerResult:
+    disabled_deletions_detected, err = terraform_client.plan(enable_deletion)
     if err:
         raise RuntimeError("Error running terraform plan")
     if disabled_deletions_detected:
         raise RuntimeError("Disabled deletions detected running terraform plan")
+    if not dry_run:
+        err = terraform_client.apply()
+        if err:
+            raise RuntimeError("Error running terraform apply")
+    return ExtendedEarlyExitRunnerResult(
+        payload=terrascript_client.terraform_configurations(),
+        applied_count=terraform_client.apply_count,
+    )
 
-    if dry_run:
-        return
 
-    err = tf.apply()
-    if err:
-        raise RuntimeError("Error running terraform apply")
+@defer
+def run(
+    dry_run: bool,
+    print_to_file: Optional[str] = None,
+    enable_deletion: bool = False,
+    thread_pool_size: int = 10,
+    account_name: Optional[str] = None,
+    defer: Optional[Callable] = None,
+    enable_extended_early_exit: bool = False,
+    extended_early_exit_cache_ttl_seconds: int = 3600,
+    log_cached_log_output: bool = False,
+) -> None:
+    desired_state_data_source = _fetch_desired_state_data_source(account_name)
+    tgw_accounts = [
+        a.dict(by_alias=True)
+        for a in _filter_tgw_accounts(
+            desired_state_data_source.accounts, desired_state_data_source.clusters
+        )
+        if not account_name or account_name == a.name
+    ]
+    if not tgw_accounts:
+        logging.warning(
+            f"No participating AWS accounts found, consider disabling this integration, account name: {account_name}"
+        )
+        return
+    secret_reader, aws_api, tf, ts = setup(
+        desired_state_data_source=desired_state_data_source,
+        account_name=account_name,
+        tgw_accounts=tgw_accounts,
+        thread_pool_size=thread_pool_size,
+        print_to_file=print_to_file,
+    )
+    if defer:
+        defer(aws_api.cleanup)
+        defer(tf.cleanup)
+    if print_to_file:
+        return
+    runner_params: RunnerParams = dict(
+        terraform_client=tf,
+        terrascript_client=ts,
+        enable_deletion=enable_deletion,
+        dry_run=dry_run,
+    )
+    if enable_extended_early_exit and get_feature_toggle_state(
+        "terraform-tgw-attachments-extended-early-exit",
+        default=False,
+    ):
+        cache_source = CacheSource(
+            terraform_configurations=ts.terraform_configurations(),
+        )
+        extended_early_exit_run(
+            integration=QONTRACT_INTEGRATION,
+            integration_version=QONTRACT_INTEGRATION_VERSION,
+            dry_run=dry_run,
+            cache_source=cache_source,
+            shard="_".join(account_name) if account_name else "",
+            ttl_seconds=extended_early_exit_cache_ttl_seconds,
+            logger=logging.getLogger(),
+            runner=runner,
+            runner_params=runner_params,
+            secret_reader=secret_reader,
+            log_cached_log_output=log_cached_log_output,
+        )
+    else:
+        runner(**runner_params)
 
 
 def early_exit_desired_state(*args: Any, **kwargs: Any) -> dict[str, Any]:

reconcile/test/test_terraform_tgw_attachments.py

@@ -509,6 +509,7 @@ def _setup_mocks(
     vpc_details: Optional[Mapping] = None,
     tgws: Optional[Iterable] = None,
     assume_role: Optional[str] = None,
+    feature_toggle_state: bool = True,
 ) -> dict:
     mocked_gql_api = create_autospec(GqlApi)
     mocker.patch(
@@ -564,7 +565,14 @@ def _setup_mocks(
     ).return_value
     mocked_tf.plan.return_value = (False, False)
     mocked_tf.apply.return_value = False
-
+    mocked_tf.apply_count = 0
+    get_feature_toggle_state = mocker.patch(
+        "reconcile.terraform_tgw_attachments.get_feature_toggle_state",
+        return_value=feature_toggle_state,
+    )
+    mock_extended_early_exit_run = mocker.patch(
+        "reconcile.terraform_tgw_attachments.extended_early_exit_run"
+    )
     mocked_logging = mocker.patch("reconcile.terraform_tgw_attachments.logging")
 
     return {
@@ -578,9 +586,126 @@ def _setup_mocks(
         "aws_api": mocked_aws_api,
         "gql_api": mocked_gql_api,
         "logging": mocked_logging,
+        "extended_early_exit_run": mock_extended_early_exit_run,
+        "get_feature_toggle_state": get_feature_toggle_state,
     }
 
 
+def test_with_extended_early_exit_enabled(
+    mocker: MockerFixture,
+    app_interface_vault_settings: AppInterfaceSettingsV1,
+    cluster_with_tgw_connection: ClusterV1,
+    tgw_account: AWSAccountV1,
+    tgw: Mapping,
+    vpc_details: Mapping,
+    assume_role: str,
+) -> None:
+    mocks = _setup_mocks(
+        mocker,
+        vault_settings=app_interface_vault_settings,
+        clusters=[cluster_with_tgw_connection],
+        accounts=[tgw_account],
+        vpc_details=vpc_details,
+        tgws=[tgw],
+        assume_role=assume_role,
+    )
+    expected_params = integ.RunnerParams(
+        terraform_client=mocks["tf"],
+        terrascript_client=mocks["ts"],
+        dry_run=False,
+        enable_deletion=False,
+    )
+
+    integ.run(
+        False,
+        enable_deletion=False,
+        enable_extended_early_exit=True,
+        extended_early_exit_cache_ttl_seconds=40,
+        log_cached_log_output=True,
+    )
+
+    mocks["extended_early_exit_run"].assert_called_once_with(
+        integration=integ.QONTRACT_INTEGRATION,
+        integration_version=integ.QONTRACT_INTEGRATION_VERSION,
+        dry_run=False,
+        shard="",
+        cache_source=integ.CacheSource(
+            terraform_configurations=mocks["ts"].terraform_configurations.return_value
+        ),
+        ttl_seconds=40,
+        logger=mocks["logging"].getLogger.return_value,
+        runner=integ.runner,
+        runner_params=expected_params,
+        secret_reader=mocks["secret_reader"],
+        log_cached_log_output=True,
+    )
+
+
+def test_with_extended_early_exit_disabled(
+    mocker: MockerFixture,
+    app_interface_vault_settings: AppInterfaceSettingsV1,
+    cluster_with_tgw_connection: ClusterV1,
+    tgw_account: AWSAccountV1,
+    tgw: Mapping,
+    vpc_details: Mapping,
+    assume_role: str,
+) -> None:
+    mocks = _setup_mocks(
+        mocker,
+        vault_settings=app_interface_vault_settings,
+        clusters=[cluster_with_tgw_connection],
+        accounts=[tgw_account],
+        vpc_details=vpc_details,
+        tgws=[tgw],
+        assume_role=assume_role,
+    )
+    integ.run(
+        False,
+        enable_deletion=False,
+        enable_extended_early_exit=False,
+    )
+    mocks["extended_early_exit_run"].assert_not_called()
+    mocks["get_app_interface_vault_settings"].assert_called_once_with()
+    mocks["get_clusters_with_peering"].assert_called_once_with(mocks["gql_api"])
+    mocks["get_aws_accounts"].assert_called_once_with(mocks["gql_api"])
+    mocks["tf"].plan.assert_called_once_with(False)
+    mocks["tf"].apply.assert_called_once()
+
+
+def test_with_feature_flag_disabled(
+    mocker: MockerFixture,
+    app_interface_vault_settings: AppInterfaceSettingsV1,
+    cluster_with_tgw_connection: ClusterV1,
+    tgw_account: AWSAccountV1,
+    tgw: Mapping,
+    vpc_details: Mapping,
+    assume_role: str,
+) -> None:
+    mocks = _setup_mocks(
+        mocker,
+        vault_settings=app_interface_vault_settings,
+        clusters=[cluster_with_tgw_connection],
+        accounts=[tgw_account],
+        vpc_details=vpc_details,
+        tgws=[tgw],
+        assume_role=assume_role,
+        feature_toggle_state=False,
+    )
+    integ.run(
+        False,
+        enable_deletion=False,
+        enable_extended_early_exit=True,
+        extended_early_exit_cache_ttl_seconds=40,
+        log_cached_log_output=True,
+    )
+    mocks["extended_early_exit_run"].assert_not_called()
+    mocks["get_app_interface_vault_settings"].assert_called_once_with()
+    mocks["get_clusters_with_peering"].assert_called_once_with(mocks["gql_api"])
+    mocks["get_aws_accounts"].assert_called_once_with(mocks["gql_api"])
+    mocks["tf"].plan.assert_called_once_with(False)
+    mocks["tf"].apply.assert_called_once()
+
+
 def test_empty_run(
     mocker: MockerFixture,
     app_interface_vault_settings: AppInterfaceSettingsV1,