qontract-reconcile 0.10.0__py3-none-any.whl → 0.10.1.dev1203__py3-none-any.whl

reconcile/terraform_cloudflare_dns.py

@@ -1,14 +1,14 @@
+import contextlib
 import logging
 import sys
 from collections.abc import (
     Callable,
     Mapping,
-)
-from typing import (
-    Any,
-    Optional,
     Sequence,
 )
+from typing import Any
+
+from deepdiff import DeepHash
 
 from reconcile.gql_definitions.terraform_cloudflare_dns import (
     app_interface_cloudflare_dns_settings,
@@ -18,8 +18,6 @@ from reconcile.gql_definitions.terraform_cloudflare_dns.app_interface_cloudflare
     AppInterfaceSettingCloudflareDNSQueryData,
 )
 from reconcile.gql_definitions.terraform_cloudflare_dns.terraform_cloudflare_zones import (
-    AWSAccountV1,
-    CloudflareAccountV1,
     CloudflareDnsRecordV1,
     CloudflareDnsZoneQueryData,
     CloudflareDnsZoneV1,
@@ -27,7 +25,6 @@ from reconcile.gql_definitions.terraform_cloudflare_dns.terraform_cloudflare_zon
 from reconcile.status import ExitCodes
 from reconcile.utils import gql
 from reconcile.utils.defer import defer
-from reconcile.utils.exceptions import SecretIncompleteError
 from reconcile.utils.external_resources import ExternalResourceSpec
 from reconcile.utils.runtime.integration import (
     DesiredStateShardConfig,
@@ -39,16 +36,22 @@ from reconcile.utils.secret_reader import (
     create_secret_reader,
 )
 from reconcile.utils.semver_helper import make_semver
-from reconcile.utils.terraform.config_client import TerraformConfigClientCollection
+from reconcile.utils.terraform.config_client import (
+    ClientAlreadyRegisteredError,
+    TerraformConfigClientCollection,
+)
 from reconcile.utils.terraform_client import TerraformClient
 from reconcile.utils.terrascript.cloudflare_client import (
-    DEFAULT_CLOUDFLARE_ACCOUNT_2FA,
-    DEFAULT_CLOUDFLARE_ACCOUNT_TYPE,
     DEFAULT_PROVIDER_RPS,
-    CloudflareAccountConfig,
-    TerraformS3BackendConfig,
-    TerrascriptCloudflareClient,
-    create_cloudflare_terrascript,
+    DNSZoneShardingStrategy,
+    IntegrationUndefined,
+    InvalidTerraformState,
+    TerrascriptCloudflareClientFactory,
+)
+from reconcile.utils.terrascript.models import (
+    CloudflareAccount,
+    Integration,
+    TerraformStateS3,
 )
 
 DEFAULT_NAMESPACE: Mapping[str, Any] = {
@@ -69,9 +72,9 @@ DEFAULT_CLOUDFLARE_ZONE_RECORDS_MAX = 500
 class TerraformCloudflareDNSIntegrationParams(PydanticRunParams):
     enable_deletion: bool
     thread_pool_size: int
-    selected_account: Optional[str] = None
-    selected_zone: Optional[str] = None
-    print_to_file: Optional[str]
+    selected_account: str | None = None
+    selected_zone: str | None = None
+    print_to_file: str | None
 
 
 class TerraformCloudflareDNSIntegration(
@@ -88,13 +91,13 @@ class TerraformCloudflareDNSIntegration(
         return self.qontract_integration.replace("_", "-")
 
     @defer
-    def run(self, dry_run: bool, defer: Optional[Callable] = None) -> None:
+    def run(self, dry_run: bool, defer: Callable | None = None) -> None:
         settings = self._get_app_interface_settings()
 
         if not settings.settings:
             raise RuntimeError("App interface setting undefined.")
 
-        if not settings.settings[0].vault:
+        if settings.settings[0].vault is None:
             raise RuntimeError("App interface vault setting undefined.")
 
         default_max_records = (
@@ -121,8 +124,7 @@ class TerraformCloudflareDNSIntegration(
             sys.exit(ExitCodes.ERROR)
 
         # Build Cloudflare clients
-        cf_clients = TerraformConfigClientCollection()
-        zone_clients = build_clients(
+        cf_clients = build_cloudflare_terraform_config_collection(
             secret_reader,
             query_zones,
             self.qontract_integration,
@@ -130,9 +132,6 @@ class TerraformCloudflareDNSIntegration(
             self.params.selected_zone,
         )
 
-        for client in zone_clients:
-            cf_clients.register_client(*client)
-
         zone_external_resource_specs = cloudflare_dns_zone_to_external_resource(
             query_zones.zones
         )
@@ -205,13 +204,17 @@ class TerraformCloudflareDNSIntegration(
     ) -> dict[str, Any]:
         desired_state = self._get_cloudflare_desired_state()
 
-        # return {"zones":desired_state.zones}
-        return desired_state.dict()
+        return {
+            "state": {
+                z.identifier: {"shard": z.identifier, "hash": DeepHash(z).get(z)}
+                for z in desired_state.zones or []
+            }
+        }
 
     def get_desired_state_shard_config(self) -> DesiredStateShardConfig:
         return DesiredStateShardConfig(
             shard_arg_name="selected_zone",
-            shard_path_selectors={"zones[*].identifier"},
+            shard_path_selectors={"state.*.shard"},
             sharded_run_review=lambda proposal: len(proposal.proposed_shards) <= 2,
         )
 
@@ -253,7 +256,7 @@ def ensure_record_number_not_exceed_max(
 
 
 def get_cloudflare_provider_rps(
-    records: Optional[Sequence[CloudflareDnsRecordV1]],
+    records: Sequence[CloudflareDnsRecordV1] | None,
 ) -> int:
     """
     Setting Cloudlare Terraform provider's RPS based on the size of the zone to improve performance of MR checks.
@@ -268,81 +271,15 @@ def get_cloudflare_provider_rps(
     return min(-(-size // 50), DEFAULT_PROVIDER_RPS)
 
 
-def create_backend_config(
-    secret_reader: SecretReaderBase,
-    aws_acct: AWSAccountV1,
-    cf_acct: CloudflareAccountV1,
-    zone: str,
-    integration_name: str,
-) -> TerraformS3BackendConfig:
-    aws_acct_creds = secret_reader.read_all_secret(aws_acct.automation_token)
-
-    # default from AWS account file
-    tf_state = aws_acct.terraform_state
-    if tf_state is None:
-        raise ValueError(
-            f"AWS account {aws_acct.name} cannot be used for Cloudflare "
-            f"account {cf_acct.name} because it doesn't define a terraform state "
-        )
-
-    integrations = tf_state.integrations or []
-    for i in integrations or []:
-        name = i.integration
-
-        bucket_key = bucket_name = bucket_region = None
-        if name.replace("-", "_") == integration_name:
-            # Currently terraform-state-1.yml can only have one bucket
-            # but multiple integrations, which means without schema changes
-            # we have to ensure the bucket key(file) is unique across
-            # all Cloudflare zones to support sharding per zone.
-
-            bucket_key = f"{integration_name}-{cf_acct.name}-{zone}.tfstate"
-            bucket_name = tf_state.bucket
-            bucket_region = tf_state.region
-            break
-
-    if bucket_name and bucket_key and bucket_region:
-        backend_config = TerraformS3BackendConfig(
-            aws_acct_creds["aws_access_key_id"],
-            aws_acct_creds["aws_secret_access_key"],
-            bucket_name,
-            bucket_key,
-            bucket_region,
-        )
-    else:
-        raise ValueError(f"No state bucket config found for account {aws_acct.name}")
-
-    return backend_config
-
-
-def get_cf_acct_config(
-    cf_acct: CloudflareAccountV1,
-    secret_reader: SecretReaderBase,
-) -> CloudflareAccountConfig:
-    cf_acct_creds = secret_reader.read_all_secret(cf_acct.api_credentials)
-    if not cf_acct_creds.get("api_token") or not cf_acct_creds.get("account_id"):
-        raise SecretIncompleteError(
-            f"secret {cf_acct.api_credentials.path} incomplete: api_token and/or account_id missing"
-        )
-    cf_acct_config = CloudflareAccountConfig(
-        cf_acct.name,
-        cf_acct_creds["api_token"],
-        cf_acct_creds["account_id"],
-        cf_acct.enforce_twofactor or DEFAULT_CLOUDFLARE_ACCOUNT_2FA,
-        cf_acct.q_type or DEFAULT_CLOUDFLARE_ACCOUNT_TYPE,
-    )
-    return cf_acct_config
-
-
-def build_clients(
+def build_cloudflare_terraform_config_collection(
     secret_reader: SecretReaderBase,
     query_zones: CloudflareDnsZoneQueryData,
-    integration_name: str,
-    selected_account: Optional[str] = None,
-    selected_zone: Optional[str] = None,
-) -> list[tuple[str, TerrascriptCloudflareClient]]:
-    clients = []
-    cf_acct_configs: dict[str, CloudflareAccountConfig] = {}
+    qontract_integration: str,
+    selected_account: str | None,
+    selected_zone: str | None,
+) -> TerraformConfigClientCollection:
+    cf_clients = TerraformConfigClientCollection()
+    cf_accounts: dict[str, CloudflareAccount] = {}
     for zone in query_zones.zones or []:
         cf_acct = zone.account
         cf_acct_name = cf_acct.name
@@ -351,38 +288,71 @@ def build_clients(
             continue
         if selected_zone and zone.identifier != selected_zone:
             continue
-        if cf_acct_name in cf_acct_configs:
-            cf_acct_config = cf_acct_configs[cf_acct_name]
+
+        if cf_acct_name in cf_accounts:
+            cf_account = cf_accounts[cf_acct_name]
         else:
-            cf_acct_config = get_cf_acct_config(cf_acct, secret_reader)
-            cf_acct_configs[cf_acct_name] = cf_acct_config
-        aws_acct = cf_acct.terraform_state_account
-        aws_backend_config = create_backend_config(
-            secret_reader,
-            aws_acct,
-            cf_acct,
-            zone.identifier,
-            integration_name=integration_name,
+            cf_account = CloudflareAccount(
+                cf_acct_name,
+                zone.account.api_credentials,
+                zone.account.enforce_twofactor,
+                zone.account.q_type,
+                zone.account.provider_version,
+            )
+            cf_accounts[cf_acct_name] = cf_account
+
+        tf_state = zone.account.terraform_state_account.terraform_state
+        if not tf_state:
+            raise ValueError(
+                f"AWS account {zone.account.terraform_state_account.name} cannot be used for Cloudflare "
+                f"account {cf_account.name} because it does not define a Terraform state "
+            )
+        bucket = tf_state.bucket
+        region = tf_state.region
+        integrations = tf_state.integrations
+
+        if not bucket:
+            raise InvalidTerraformState("Terraform state must have bucket defined")
+        if not region:
+            raise InvalidTerraformState("Terraform state must have region defined")
+
+        integration = None
+        for i in integrations:
+            if i.integration.replace("-", "_") == qontract_integration:
+                integration = i
+                break
+
+        if not integration:
+            raise IntegrationUndefined(
+                f"Must declare integration name under Terraform state in {zone.account.terraform_state_account.name} AWS account for {cf_account.name} Cloudflare account in app-interface"
+            )
+
+        tf_state_s3 = TerraformStateS3(
+            zone.account.terraform_state_account.automation_token,
+            bucket,
+            region,
+            Integration(integration.integration.replace("-", "_"), integration.key),
         )
 
         rps = get_cloudflare_provider_rps(zone.records)
 
-        ts_config = create_cloudflare_terrascript(
-            account_config=cf_acct_config,
-            backend_config=aws_backend_config,
-            provider_version=cf_acct.provider_version,
-            provider_rps=rps,
-            is_managed_account=False,
+        client = TerrascriptCloudflareClientFactory.get_client(
+            tf_state_s3,
+            cf_account,
+            DNSZoneShardingStrategy(cf_account, zone.identifier),
+            secret_reader,
+            False,
+            rps,
         )
 
-        ts_client = TerrascriptCloudflareClient(ts_config)
-        clients.append((f"{cf_acct.name}-{zone.identifier}", ts_client))
+        with contextlib.suppress(ClientAlreadyRegisteredError):
+            cf_clients.register_client(f"{cf_account.name}-{zone.identifier}", client)
 
-    return clients
+    return cf_clients
 
 
 def cloudflare_dns_zone_to_external_resource(
-    zones: Optional[list[CloudflareDnsZoneV1]],
+    zones: list[CloudflareDnsZoneV1] | None,
 ) -> list[ExternalResourceSpec]:
     """
     This is a method that massage a list of CloudflareDnsZoneV1 into ExternalResourceSpec

reconcile/terraform_cloudflare_resources.py

@@ -1,10 +1,8 @@
 import logging
 import sys
+from collections.abc import Iterable
 from typing import (
     Any,
-    Iterable,
-    Optional,
-    Tuple,
     cast,
 )
 
@@ -40,6 +38,7 @@ from reconcile.utils.external_resource_spec import ExternalResourceSpecInventory
 from reconcile.utils.external_resources import (
     PROVIDER_CLOUDFLARE,
     get_external_resource_specs,
+    publish_metrics,
 )
 from reconcile.utils.oc import StatusCodeError
 from reconcile.utils.oc_map import (
@@ -119,7 +118,7 @@ def create_backend_config(
 def build_clients(
     secret_reader: SecretReaderBase,
     query_accounts: TerraformCloudflareAccountsQueryData,
-    selected_account: Optional[str] = None,
+    selected_account: str | None = None,
 ) -> list[tuple[str, TerrascriptCloudflareClient]]:
     clients = []
     for cf_acct in query_accounts.accounts or []:
@@ -157,8 +156,8 @@ def _build_oc_resources(
     secret_reader: SecretReaderBase,
     use_jump_host: bool,
     thread_pool_size: int,
-    internal: Optional[bool] = None,
-    account_names: Optional[Iterable[str]] = None,
+    internal: bool | None = None,
+    account_names: Iterable[str] | None = None,
 ) -> tuple[ResourceInventory, OCMap]:
     ri = ResourceInventory()
 
@@ -193,7 +192,7 @@ def _build_oc_resources(
 def _populate_oc_resources(
     spec: CurrentStateSpec,
     ri: ResourceInventory,
-    account_names: Optional[Iterable[str]],
+    account_names: Iterable[str] | None,
 ):
     """
     This was taken from terraform_resources and might be a later candidate for DRY.
@@ -309,12 +308,12 @@ def _filter_cloudflare_namespaces(
 @defer
 def run(
     dry_run: bool,
-    print_to_file: Optional[str],
+    print_to_file: str | None,
     enable_deletion: bool,
     thread_pool_size: int,
-    selected_account: Optional[str] = None,
+    selected_account: str | None = None,
     vault_output_path: str = "",
-    internal: Optional[bool] = None,
+    internal: bool | None = None,
     use_jump_host: bool = True,
     defer=None,
 ) -> None:
@@ -341,7 +340,7 @@ def run(
     )
 
     if not cloudflare_namespaces:
-        logging.info("No namespaces were detected, nothing to do.")
+        logging.info("No cloudflare namespaces were detected, nothing to do.")
         sys.exit(ExitCodes.SUCCESS)
 
     # Build Cloudflare clients
@@ -362,6 +361,8 @@ def run(
 
     cf_clients.populate_resources()
 
+    publish_metrics(cf_clients.resource_spec_inventory, QONTRACT_INTEGRATION)
+
     ri, oc_map = _build_oc_resources(
         cloudflare_namespaces,
         secret_reader,
@@ -427,8 +428,9 @@ def run(
         )
 
 
-def _get_cloudflare_desired_state() -> Tuple[
-    TerraformCloudflareAccountsQueryData, TerraformCloudflareResourcesQueryData
+def _get_cloudflare_desired_state() -> tuple[
+    TerraformCloudflareAccountsQueryData,
+    TerraformCloudflareResourcesQueryData,
 ]:
     query_accounts = terraform_cloudflare_accounts.query(query_func=gql.get_api().query)
     query_resources = terraform_cloudflare_resources.query(

reconcile/terraform_cloudflare_users.py

@@ -1,13 +1,11 @@
+import contextlib
 from collections.abc import (
     Iterable,
     Mapping,
     MutableMapping,
 )
 from dataclasses import dataclass
-from typing import (
-    Any,
-    Optional,
-)
+from typing import Any
 
 from reconcile.gql_definitions.terraform_cloudflare_users import (
     app_interface_setting_cloudflare_and_vault,
@@ -69,8 +67,8 @@ class CloudflareUser:
 
 
 class TerraformCloudflareUsersParams(PydanticRunParams):
-    print_to_file: Optional[str]
-    account_name: Optional[str]
+    print_to_file: str | None
+    account_name: str | None
     thread_pool_size: int
     enable_deletion: bool
 
@@ -84,17 +82,16 @@ class TerraformCloudflareUsers(
 
     def get_early_exit_desired_state(
         self, *args: Any, **kwargs: Any
-    ) -> Optional[dict[str, Any]]:
-
+    ) -> dict[str, Any] | None:
         cloudflare_roles, settings = self._get_desired_state()
 
         if not settings.settings:
             raise RuntimeError("App interface setting not defined")
 
         early_exit_desired_state = cloudflare_roles.dict()
-        early_exit_desired_state.update(
-            {CLOUDFLARE_EMAIL_DOMAIN_ALLOW_LIST_KEY: settings.settings}
-        )
+        early_exit_desired_state.update({
+            CLOUDFLARE_EMAIL_DOMAIN_ALLOW_LIST_KEY: settings.settings
+        })
         return early_exit_desired_state
 
     def _get_desired_state(
@@ -212,7 +209,7 @@ class TerraformCloudflareUsers(
         self,
         query_data: CloudflareAccountRoleQueryData,
         secret_reader: SecretReaderBase,
-        account_name: Optional[str],
+        account_name: str | None,
     ) -> TerraformConfigClientCollection:
         cf_clients = TerraformConfigClientCollection()
         for role in query_data.cloudflare_account_roles or []:
@@ -230,7 +227,7 @@ class TerraformCloudflareUsers(
             if not tf_state:
                 raise ValueError(
                     f"AWS account {role.account.terraform_state_account.name} cannot be used for Cloudflare "
-                    f"account {cf_account.name} because it does define a Terraform state "
+                    f"account {cf_account.name} because it does not define a Terraform state "
                 )
 
             bucket = tf_state.bucket
@@ -268,18 +265,16 @@ class TerraformCloudflareUsers(
                 False,
             )
 
-            try:
+            with contextlib.suppress(ClientAlreadyRegisteredError):
                 cf_clients.register_client(cf_account.name, client)
-            except ClientAlreadyRegisteredError:
-                pass
 
         return cf_clients
 
 
 def get_cloudflare_users(
-    cloudflare_roles: Optional[Iterable[CloudflareAccountRoleV1]],
-    account_name: Optional[str],
-    email_domain_allow_list: Optional[Iterable[str]],
+    cloudflare_roles: Iterable[CloudflareAccountRoleV1] | None,
+    account_name: str | None,
+    email_domain_allow_list: Iterable[str] | None,
 ) -> dict[str, dict[str, CloudflareUser]]:
     """
     Returns a two-level dictionary of users with 1st level keys mapping to Cloudflare account names
@@ -305,13 +300,13 @@ def get_cloudflare_users(
                                 user.cloudflare_user
                             ].roles.update(set(cf_role.roles))
                         else:
-                            users[cf_role.account.name][
-                                user.cloudflare_user
-                            ] = CloudflareUser(
-                                user.cloudflare_user,
-                                cf_role.account.name,
-                                user.org_username,
-                                set(cf_role.roles),
+                            users[cf_role.account.name][user.cloudflare_user] = (
+                                CloudflareUser(
+                                    user.cloudflare_user,
+                                    cf_role.account.name,
+                                    user.org_username,
+                                    set(cf_role.roles),
+                                )
                             )
 
                     else:
@@ -328,7 +323,7 @@ def get_cloudflare_users(
 
 
 def build_external_resource_spec_from_cloudflare_users(
-    cloudflare_users: Mapping[str, Mapping[str, CloudflareUser]]
+    cloudflare_users: Mapping[str, Mapping[str, CloudflareUser]],
 ) -> Iterable[ExternalResourceSpec]:
     """
     This method transforms :param cloudflare_users: into a list of ExternalResourceSpec
@@ -347,6 +342,11 @@ def build_external_resource_spec_from_cloudflare_users(
                 "provider": "account_member",
                 "identifier": safe_resource_id(cf_user.org_username),
                 "email_address": cf_user.email_address,
+                # Setting status to 'accepted' skips the need for the user to accept
+                # all the invites for the accounts that they're added to. This only
+                # works if the Cloudflare user account already exists, which seems to be
+                # acceptable for now.
+                "status": "accepted",
                 "account_id": "${var.account_id}",
                 "role_ids": [
                     # I know this is ugly :(