qontract-reconcile 0.10.1rc1157__py3-none-any.whl → 0.10.1rc1158__py3-none-any.whl

{qontract_reconcile-0.10.1rc1157.dist-info → qontract_reconcile-0.10.1rc1158.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: qontract-reconcile
-Version: 0.10.1rc1157
+Version: 0.10.1rc1158
 Summary: Collection of tools to reconcile services with their desired state as defined in the app-interface DB.
 Home-page: https://github.com/app-sre/qontract-reconcile
 Author: Red Hat App-SRE Team

{qontract_reconcile-0.10.1rc1157.dist-info → qontract_reconcile-0.10.1rc1158.dist-info}/RECORD

@@ -94,7 +94,7 @@ reconcile/quay_mirror.py,sha256=mFp4Z5Nwl-DcFbbsJBOB8f9ldohFT-V67o868d5ux1s,1536
 reconcile/quay_mirror_org.py,sha256=utrJpJaKCs7U6WX6DODdfCeB0EmX-lUC8Y5fkmpgFSs,10764
 reconcile/quay_permissions.py,sha256=9KOutS1w4RFQqkvMSy54VtsKNx56-phzP6yI_rEW-B8,4244
 reconcile/quay_repos.py,sha256=cuEYG0HUe0ut5yvLdEwOF5-CmccpXQHRb_wDazvDrvQ,6895
-reconcile/queries.py,sha256=HAhD9d9TYRfkJMA5PQpv_eXuO2_wLt2Mlu2PaaBU3Xc,50201
+reconcile/queries.py,sha256=VaBYi8IU01Tjrncs_Ik0D4Q-cQvlinWFhH2_8cq5x7U,50812
 reconcile/query_validator.py,sha256=MSh5pKLBksws4AqfuvT8nrIGucIbqX-IOzYyPYTLO7k,1491
 reconcile/requests_sender.py,sha256=914iluuF4UVgG3VyxxtnHOu4yf6YKS2fIy6PViSsFTQ,3875
 reconcile/resource_scraper.py,sha256=znXCHrU7YwPfKuxGBiUrV7T1tYtn4vlz9qmZlfy6Flg,2307
@@ -113,7 +113,7 @@ reconcile/terraform_cloudflare_dns.py,sha256=-aLEe2QnH5cJPu7HWqs-R9NmQ1NlFbcVUm0
 reconcile/terraform_cloudflare_resources.py,sha256=pq8Ieo5NmB-dYQ9X2F0s6iEoINMzhiqGw2yQK4ovok4,14980
 reconcile/terraform_cloudflare_users.py,sha256=iyTG5sj20Jg4J4qWJ144KVptfIHGOSfH8wQKxu0imq0,13942
 reconcile/terraform_repo.py,sha256=TKqlodhQGoAtQ6nDm04TNlpx4wpgJ_n4atoUK5Rfd7o,16444
-reconcile/terraform_resources.py,sha256=-sgMMHDtNvnQyNR05-MKebI_pSiyxSWAg8LmeA2_Ntk,19326
+reconcile/terraform_resources.py,sha256=jpBtp6vezq79jQ7rWdk49_mW-PIUFVzFK54ilVSEZFM,19564
 reconcile/terraform_tgw_attachments.py,sha256=09svJG9pAiwWp4aY0xRoQRV90T4ZNwHG3r8flI-ZS_s,18810
 reconcile/terraform_users.py,sha256=HqSm3ev3b8dZ9J6F_phDZB-FQsnlsdeKp9RPoY1cU94,10188
 reconcile/terraform_vpc_peerings.py,sha256=VLSfuO7FvHN5McopRiKoKJDHCmIhYtlJEHv_hxV5kcM,27669
@@ -674,7 +674,7 @@ reconcile/utils/exceptions.py,sha256=DwfnWUpVOotpP79RWZ2pycmG6nKCL00RBIeZLYkQPW4
 reconcile/utils/expiration.py,sha256=3JaXH4psksR7z262k7FmdyREjCLqm66OpVMEbcfdWRo,1213
 reconcile/utils/extended_early_exit.py,sha256=QSktrmfw37zSRMNk930tDbQsVeKxaPPPD43e79DGwZw,6754
 reconcile/utils/external_resource_spec.py,sha256=bhH_xneFwATdFumTPkiQmcVKYI0gcaWuqV6FpFdf_P0,7006
-reconcile/utils/external_resources.py,sha256=n3c7GyFb0o5sgQ67dAsGWV40ELHJH0vMKyJMkvMHEok,7659
+reconcile/utils/external_resources.py,sha256=y7Wz32cOAmCsUhQ6T-1N6lktnLikGkaHQ0SdDpMse1w,7585
 reconcile/utils/filtering.py,sha256=S4PbMHuFr3ED0P2Q_ea5CAaB7FimI62B-F5YTaKrphA,402
 reconcile/utils/git.py,sha256=wzVIYAeKlMGW538U1mkJWUI6h_mFRUY4lawh2AR8hw4,2345
 reconcile/utils/github_api.py,sha256=R8OvqyPdnRqvP-Efnv9RvIcbBlb4M0KC4RlbnJMD0Tg,2426
@@ -727,7 +727,7 @@ reconcile/utils/state.py,sha256=W0_awkLAPX18hNOF_60o73tkPxDUylqbzYNHfl_sDsk,1638
 reconcile/utils/structs.py,sha256=LcbLEg8WxfRqM6nW7NhcWN0YeqF7SQzxOgntmLs1SgY,352
 reconcile/utils/template.py,sha256=wTvRU4AnAV_o042tD4Mwls2dwWMuk7MKnde3MaCjaYg,331
 reconcile/utils/terraform_client.py,sha256=LjX2U2E0Dglt2S_KA5jWQ_dVC8sPn4FEAh0xW_d6JTk,35953
-reconcile/utils/terrascript_aws_client.py,sha256=0xHh8NbMbISv9zIxwcXxuYVer8dTM_2t7f6Jf4TuU2I,281362
+reconcile/utils/terrascript_aws_client.py,sha256=aWaxp-lomSdlWz-yPvqAsaE-3W_jnxOIRvJuRXDGokY,283449
 reconcile/utils/three_way_diff_strategy.py,sha256=oQcHXd9LVhirJfoaOBoHUYuZVGfyL2voKr6KVI34zZE,4833
 reconcile/utils/throughput.py,sha256=iP4UWAe2LVhDo69mPPmgo9nQ7RxHD6_GS8MZe-aSiuM,344
 reconcile/utils/vault.py,sha256=pi0PuyopvCq1gW0cldvy1-Ff6bqLUlCKC2MW0sifvSE,15043
@@ -880,8 +880,8 @@ tools/test/test_qontract_cli.py,sha256=iuzKbQ6ahinvjoQmQLBrG4shey0z-1rB6qCgS8T6d
 tools/test/test_saas_promotion_state.py,sha256=dy4kkSSAQ7bC0Xp2CociETGN-2aABEfL6FU5D9Jl00Y,6056
 tools/test/test_sd_app_sre_alert_report.py,sha256=v363r9zM7__0kR5K6mvJoGFcM9BvE33fWAayrqkpojA,2116
 tools/test/test_sre_checkpoints.py,sha256=SKqPPTl9ua0RFdSSofnoQX-JZE6dFLO3LRhfQzqtfh8,2607
-qontract_reconcile-0.10.1rc1157.dist-info/METADATA,sha256=xO5D-srjk2bIJiQ0wi7YxeO45fY7JKWzpBl4ktpNok8,2213
-qontract_reconcile-0.10.1rc1157.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
-qontract_reconcile-0.10.1rc1157.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
-qontract_reconcile-0.10.1rc1157.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
-qontract_reconcile-0.10.1rc1157.dist-info/RECORD,,
+qontract_reconcile-0.10.1rc1158.dist-info/METADATA,sha256=eMmlXcbujY4OpUxnwP4jD3BuN2P3RGmQo-btlbDJT4k,2213
+qontract_reconcile-0.10.1rc1158.dist-info/WHEEL,sha256=eOLhNAGa2EW3wWl_TU484h7q1UNgy0JXjjoqKoxAAQc,92
+qontract_reconcile-0.10.1rc1158.dist-info/entry_points.txt,sha256=GKQqCl2j2X1BJQ69een6rHcR26PmnxnONLNOQB-nRjY,491
+qontract_reconcile-0.10.1rc1158.dist-info/top_level.txt,sha256=l5ISPoXzt0SdR4jVdkfa7RPSKNc8zAHYWAnR-Dw8Ey8,24
+qontract_reconcile-0.10.1rc1158.dist-info/RECORD,,

reconcile/queries.py

@@ -102,6 +102,12 @@ APP_INTERFACE_SETTINGS_QUERY = """
       readTimeout
       connectTimeout
     }
+    terraformResourcesProviderExclusionsByProvisioner {
+      provisioner {
+          name
+      }
+      excludedProviders
+    }
   }
 }
 """
@@ -2754,3 +2760,24 @@ JENKINS_CONFIGS = """
 def get_jenkins_configs():
     gqlapi = gql.get_api()
     return gqlapi.query(JENKINS_CONFIGS)["jenkins_configs"]
+
+
+TF_RESOURCES_PROVIDER_EXCLUSIONS_BY_PROVISIONER = """
+{
+  tf_provider_exclusions_by_provisioner: app_interface_settings_v1 {
+    terraformResourcesProviderExclusionsByProvisioner {
+      provisioner {
+          name
+      }
+      excludedProviders
+    }
+  }
+}
+"""
+
+
+def get_tf_resources_provider_exclusions_by_provisioner():
+    gqlapi = gql.get_api()
+    return gqlapi.query(TF_RESOURCES_PROVIDER_EXCLUSIONS_BY_PROVISIONER)[
+        "tf_provider_exclusions_by_provisioner"
+    ]

reconcile/terraform_resources.py

@@ -241,7 +241,7 @@ def setup(
     vault_settings = get_app_interface_vault_settings()
     secret_reader = create_secret_reader(use_vault=vault_settings.vault)
 
-    settings = queries.get_app_interface_settings()
+    settings = queries.get_app_interface_settings() or {}
     # initialize terrascript (scripting engine to generate terraform manifests)
     ts, working_dirs = init_working_dirs(accounts, thread_pool_size, settings=settings)
 
@@ -265,7 +265,15 @@ def setup(
     else:
         ocm_map = None
     tf_namespaces_dicts = [ns.dict(by_alias=True) for ns in tf_namespaces]
-    ts.init_populate_specs(tf_namespaces_dicts, account_names)
+
+    provider_exclusions_by_provisioner = (
+        settings.get("terraformResourcesProviderExclusionsByProvisioner") or []
+    )
+    ts.init_populate_specs(
+        tf_namespaces_dicts,
+        account_names,
+        provider_exclusions_by_provisioner=provider_exclusions_by_provisioner,
+    )
     tf.populate_terraform_output_secrets(
         resource_specs=ts.resource_spec_inventory, init_rds_replica_source=True
     )

reconcile/utils/external_resources.py

@@ -1,9 +1,6 @@
 import json
 from collections import Counter
-from collections.abc import (
-    Mapping,
-    MutableMapping,
-)
+from collections.abc import Mapping, MutableMapping
 from typing import Any
 
 import anymarkup
@@ -25,7 +22,8 @@ PROVIDER_CLOUDFLARE = "cloudflare"
 
 
 def get_external_resource_specs(
-    namespace_info: Mapping[str, Any], provision_provider: str | None = None
+    namespace_info: Mapping[str, Any],
+    provision_provider: str | None = None,
 ) -> list[ExternalResourceSpec]:
     specs: list[ExternalResourceSpec] = []
     if not managed_external_resources(namespace_info):
@@ -34,8 +32,6 @@ def get_external_resource_specs(
     external_resources = namespace_info.get("externalResources") or []
     for e in external_resources:
         for r in e.get("resources", []):
-            if r.get("managed_by_erv2"):
-                continue
             spec = ExternalResourceSpec(
                 provision_provider=e["provider"],
                 provisioner=e["provisioner"],

reconcile/utils/terrascript_aws_client.py

@@ -8,11 +8,7 @@ import re
 import string
 import tempfile
 from collections import Counter
-from collections.abc import (
-    Iterable,
-    Mapping,
-    MutableMapping,
-)
+from collections.abc import Iterable, Mapping, MutableMapping
 from dataclasses import dataclass
 from ipaddress import (
     ip_address,
@@ -383,6 +379,16 @@ class ElasticSearchLogGroupInfo:
     log_group_identifier: str
 
 
+class ProviderExcludedError(Exception):
+    def __init__(self, spec: ExternalResourceSpec) -> None:
+        super().__init__(
+            self,
+            "The provider is not managed by terraform_resources in this provisioner. "
+            "Set the `managed_by_erv2: true` attribute in the external resource spec to fix it."
+            f"Provisioner: {spec.provisioner['name']}, Provider: {spec.provider}, Identifier: {spec.resource['identifier']}",
+        )
+
+
 class TerrascriptClient:  # pylint: disable=too-many-public-methods
     """
     At a high-level, this class is responsible for generating Terraform configuration in
@@ -1537,10 +1543,38 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
             for spec in specs:
                 self.populate_tf_resources(spec, ocm_map=ocm_map)
 
+    def _get_provisioner_provider_exclusions(
+        self,
+        spec: ExternalResourceSpec,
+        provider_exclusions_by_provisioner_name: Mapping[str, Iterable[str]],
+    ) -> list[str]:
+        return list(
+            provider_exclusions_by_provisioner_name.get(spec.provisioner["name"], [])
+        )
+
+    def _filter_specs_managed_by_erv2(
+        self,
+        specs: Iterable[ExternalResourceSpec],
+        provider_exclusions_by_provisioner_name: Mapping[str, Iterable[str]],
+    ) -> list[ExternalResourceSpec]:
+        filtered_specs: list[ExternalResourceSpec] = []
+        for spec in specs:
+            if spec.resource.get("managed_by_erv2"):
+                continue
+
+            if spec.provider in self._get_provisioner_provider_exclusions(
+                spec, provider_exclusions_by_provisioner_name
+            ):
+                raise ProviderExcludedError(spec)
+
+            filtered_specs.append(spec)
+        return filtered_specs
+
     def init_populate_specs(
         self,
         namespaces: Iterable[Mapping[str, Any]],
         account_names: Iterable[str] | None,
+        provider_exclusions_by_provisioner: Iterable[Mapping[str, Any]] | None = None,
     ) -> None:
         """
         Initiates resource specs from the definitions in app-interface
@@ -1551,9 +1585,24 @@ class TerrascriptClient:  # pylint: disable=too-many-public-methods
         self.account_resource_specs: dict[str, list[ExternalResourceSpec]] = {}
         self.resource_spec_inventory: ExternalResourceSpecInventory = {}
 
+        # Ensure provider exclusions are fetched
+        if not provider_exclusions_by_provisioner:
+            provider_exclusions_by_provisioner = (
+                queries.get_tf_resources_provider_exclusions_by_provisioner() or []
+            )
+
+        provider_exclusions_by_provisioner_name = {
+            p["provisioner"]["name"]: p["excludedProviders"]
+            for p in provider_exclusions_by_provisioner
+        }
+
         for namespace_info in namespaces:
-            specs = get_external_resource_specs(
-                namespace_info, provision_provider=PROVIDER_AWS
+            all_specs = get_external_resource_specs(
+                namespace_info,
+                provision_provider=PROVIDER_AWS,
+            )
+            specs = self._filter_specs_managed_by_erv2(
+                all_specs, provider_exclusions_by_provisioner_name
             )
             name_counter = Counter(spec.output_resource_name for spec in specs)
             duplicates = [name for name, count in name_counter.items() if count > 1]