qontract-reconcile 0.10.2.dev504__py3-none-any.whl → 0.10.2.dev506__py3-none-any.whl

reconcile/terraform_cloudflare_users.py

@@ -1,374 +0,0 @@
-import contextlib
-from collections.abc import (
-    Iterable,
-    Mapping,
-    MutableMapping,
-)
-from dataclasses import dataclass
-from typing import Any
-
-from reconcile.gql_definitions.terraform_cloudflare_users import (
-    app_interface_setting_cloudflare_and_vault,
-    terraform_cloudflare_roles,
-)
-from reconcile.gql_definitions.terraform_cloudflare_users.app_interface_setting_cloudflare_and_vault import (
-    AppInterfaceSettingCloudflareAndVaultQueryData,
-)
-from reconcile.gql_definitions.terraform_cloudflare_users.terraform_cloudflare_roles import (
-    CloudflareAccountRoleQueryData,
-    CloudflareAccountRoleV1,
-)
-from reconcile.utils import gql
-from reconcile.utils.external_resource_spec import ExternalResourceSpec
-from reconcile.utils.runtime.integration import (
-    PydanticRunParams,
-    QontractReconcileIntegration,
-)
-from reconcile.utils.secret_reader import (
-    SecretReaderBase,
-    create_secret_reader,
-)
-from reconcile.utils.semver_helper import make_semver
-from reconcile.utils.terraform import safe_resource_id
-from reconcile.utils.terraform.config_client import (
-    ClientAlreadyRegisteredError,
-    TerraformConfigClientCollection,
-)
-from reconcile.utils.terraform_client import (
-    TerraformApplyFailedError,
-    TerraformClient,
-    TerraformDeletionDetectedError,
-    TerraformPlanFailedError,
-)
-from reconcile.utils.terrascript.cloudflare_client import (
-    AccountShardingStrategy,
-    IntegrationUndefinedError,
-    InvalidTerraformStateError,
-    TerrascriptCloudflareClientFactory,
-)
-from reconcile.utils.terrascript.models import (
-    CloudflareAccount,
-    Integration,
-    TerraformStateS3,
-)
-
-QONTRACT_INTEGRATION = "terraform_cloudflare_users"
-QONTRACT_INTEGRATION_VERSION = make_semver(0, 1, 0)
-QONTRACT_TF_PREFIX = "qrtfcfusers"
-CLOUDFLARE_EMAIL_DOMAIN_ALLOW_LIST_KEY = "cloudflareEmailDomainAllowList"
-
-
-@dataclass
-class CloudflareUser:
-    email_address: str
-    account_name: str
-    org_username: str
-    roles: set[str]
-
-
-class TerraformCloudflareUsersParams(PydanticRunParams):
-    print_to_file: str | None
-    account_name: str | None
-    thread_pool_size: int
-    enable_deletion: bool
-
-
-class TerraformCloudflareUsers(
-    QontractReconcileIntegration[TerraformCloudflareUsersParams]
-):
-    @property
-    def name(self) -> str:
-        return QONTRACT_INTEGRATION.replace("_", "-")
-
-    def get_early_exit_desired_state(
-        self, *args: Any, **kwargs: Any
-    ) -> dict[str, Any] | None:
-        cloudflare_roles, settings = self._get_desired_state()
-
-        if not settings.settings:
-            raise RuntimeError("App interface setting not defined")
-
-        early_exit_desired_state = cloudflare_roles.model_dump()
-        early_exit_desired_state.update({
-            CLOUDFLARE_EMAIL_DOMAIN_ALLOW_LIST_KEY: settings.settings
-        })
-        return early_exit_desired_state
-
-    def _get_desired_state(
-        self,
-    ) -> tuple[
-        CloudflareAccountRoleQueryData, AppInterfaceSettingCloudflareAndVaultQueryData
-    ]:
-        cloudflare_roles = terraform_cloudflare_roles.query(
-            query_func=gql.get_api().query
-        )
-
-        settings = app_interface_setting_cloudflare_and_vault.query(
-            query_func=gql.get_api().query
-        )
-
-        return cloudflare_roles, settings
-
-    def run(self, dry_run: bool) -> None:
-        print_to_file = self.params.print_to_file
-        account_name = self.params.account_name
-        thread_pool_size = self.params.thread_pool_size
-        enable_deletion = self.params.enable_deletion
-
-        cloudflare_roles, settings = self._get_desired_state()
-
-        if not settings.settings:
-            raise RuntimeError("App interface setting not defined")
-
-        secret_reader = create_secret_reader(use_vault=settings.settings[0].vault)
-
-        cf_clients = self._build_cloudflare_terraform_config_client_collection(
-            cloudflare_roles, secret_reader, account_name
-        )
-
-        users = get_cloudflare_users(
-            cloudflare_roles.cloudflare_account_roles,
-            account_name,
-            settings.settings[0].cloudflare_email_domain_allow_list,
-        )
-        specs = build_external_resource_spec_from_cloudflare_users(users)
-
-        cf_clients.add_specs(specs)
-        cf_clients.populate_resources()
-
-        working_dirs = cf_clients.dump(print_to_file=print_to_file)
-
-        if print_to_file:
-            return
-
-        # for storing unique CloudflareAccountV1 since cloudflare_account_role_v1 can contain duplicates due to schema
-        account_names_to_account = {
-            role.account.name: role.account
-            for role in cloudflare_roles.cloudflare_account_roles or []
-            if role.account.name in cf_clients.dump()
-        }
-
-        accounts = [
-            acct.model_dump(by_alias=True)
-            for _, acct in account_names_to_account.items()
-        ]
-
-        self._run_terraform(
-            dry_run,
-            enable_deletion,
-            thread_pool_size,
-            working_dirs,
-            accounts,
-        )
-
-    def _run_terraform(
-        self,
-        dry_run: bool,
-        enable_deletion: bool,
-        thread_pool_size: int,
-        working_dirs: Mapping[str, str],
-        accounts: Iterable[Mapping[str, Any]],
-    ) -> None:
-        tf = TerraformClient(
-            QONTRACT_INTEGRATION,
-            QONTRACT_INTEGRATION_VERSION,
-            QONTRACT_TF_PREFIX,
-            accounts,
-            working_dirs,
-            thread_pool_size,
-        )
-
-        try:
-            disabled_deletions_detected, err = tf.plan(enable_deletion)
-            if err:
-                raise TerraformPlanFailedError(
-                    f"Failed to run terraform plan for integration {QONTRACT_INTEGRATION}"
-                )
-            if disabled_deletions_detected:
-                raise TerraformDeletionDetectedError(
-                    "Deletions detected but they are disabled"
-                )
-
-            if dry_run:
-                return
-
-            err = tf.apply()
-            if err:
-                raise TerraformApplyFailedError(
-                    f"Failed to run terraform apply for integration {QONTRACT_INTEGRATION}"
-                )
-        finally:
-            tf.cleanup()
-
-    def _build_cloudflare_terraform_config_client_collection(
-        self,
-        query_data: CloudflareAccountRoleQueryData,
-        secret_reader: SecretReaderBase,
-        account_name: str | None,
-    ) -> TerraformConfigClientCollection:
-        cf_clients = TerraformConfigClientCollection()
-        for role in query_data.cloudflare_account_roles or []:
-            if account_name and role.account.name != account_name:
-                continue
-            cf_account = CloudflareAccount(
-                role.account.name,
-                role.account.api_credentials,
-                role.account.enforce_twofactor,
-                role.account.q_type,
-                role.account.provider_version,
-            )
-
-            tf_state = role.account.terraform_state_account.terraform_state
-            if not tf_state:
-                raise ValueError(
-                    f"AWS account {role.account.terraform_state_account.name} cannot be used for Cloudflare "
-                    f"account {cf_account.name} because it does not define a Terraform state "
-                )
-
-            bucket = tf_state.bucket
-            region = tf_state.region
-            integrations = tf_state.integrations
-
-            if not bucket:
-                raise InvalidTerraformStateError(
-                    "Terraform state must have bucket defined"
-                )
-            if not region:
-                raise InvalidTerraformStateError(
-                    "Terraform state must have region defined"
-                )
-
-            integration = None
-            for i in integrations:
-                if i.integration.replace("-", "_") == QONTRACT_INTEGRATION:
-                    integration = i
-                    break
-
-            if not integration:
-                raise IntegrationUndefinedError(
-                    "Must declare integration name under Terraform state in app-interface"
-                )
-
-            tf_state_s3 = TerraformStateS3(
-                role.account.terraform_state_account.automation_token,
-                bucket,
-                region,
-                Integration(integration.integration, integration.key),
-            )
-
-            client = TerrascriptCloudflareClientFactory.get_client(
-                tf_state_s3,
-                cf_account,
-                AccountShardingStrategy(cf_account),
-                secret_reader,
-                False,
-            )
-
-            with contextlib.suppress(ClientAlreadyRegisteredError):
-                cf_clients.register_client(cf_account.name, client)
-
-        return cf_clients
-
-
-def get_cloudflare_users(
-    cloudflare_roles: Iterable[CloudflareAccountRoleV1] | None,
-    account_name: str | None,
-    email_domain_allow_list: Iterable[str] | None,
-) -> dict[str, dict[str, CloudflareUser]]:
-    """
-    Returns a two-level dictionary of users with 1st level keys mapping to Cloudflare account names
-    and 2nd level keys mapping to the user's email address.
-    The method also takes into consideration :param account_name: and :param email_domain_allow_list: which can be
-    used to filter users not matching these parameters
-    """
-    users: dict[str, dict[str, CloudflareUser]] = {}
-
-    for cf_role in cloudflare_roles or []:
-        if account_name and cf_role.account.name != account_name:
-            continue
-        for access_role in cf_role.access_roles or []:
-            for user in access_role.users:
-                if user.cloudflare_user is not None and (
-                    user.cloudflare_user.split("@")[1]
-                    in (email_domain_allow_list or [])
-                ):
-                    temp = users.get(cf_role.account.name)
-                    if temp is not None:
-                        if temp.get(user.cloudflare_user) is not None:
-                            users[cf_role.account.name][
-                                user.cloudflare_user
-                            ].roles.update(set(cf_role.roles))
-                        else:
-                            users[cf_role.account.name][user.cloudflare_user] = (
-                                CloudflareUser(
-                                    user.cloudflare_user,
-                                    cf_role.account.name,
-                                    user.org_username,
-                                    set(cf_role.roles),
-                                )
-                            )
-
-                    else:
-                        users[cf_role.account.name] = {
-                            user.cloudflare_user: CloudflareUser(
-                                user.cloudflare_user,
-                                cf_role.account.name,
-                                user.org_username,
-                                set(cf_role.roles),
-                            )
-                        }
-
-    return users
-
-
-def build_external_resource_spec_from_cloudflare_users(
-    cloudflare_users: Mapping[str, Mapping[str, CloudflareUser]],
-) -> Iterable[ExternalResourceSpec]:
-    """
-    This method transforms :param cloudflare_users: into a list of ExternalResourceSpec
-    as TerrascriptCloudflareClient works only with the ExternalResourceSpec.
-    """
-    specs: list[ExternalResourceSpec] = []
-
-    for v in cloudflare_users.values():
-        for cf_user in v.values():
-            data_source_cloudflare_account_roles = {
-                "identifier": safe_resource_id(cf_user.account_name),
-                "account_id": "${var.account_id}",
-            }
-
-            cloudflare_account_member = {
-                "provider": "account_member",
-                "identifier": safe_resource_id(cf_user.org_username),
-                "email_address": cf_user.email_address,
-                "account_id": "${var.account_id}",
-                "role_ids": [
-                    # I know this is ugly :(
-                    # Terrascript doesn't support local values. Hence, we have to rely on string templating
-                    # (https://developer.hashicorp.com/terraform/language/expressions/strings#string-templates) to get
-                    # cloudflare role ids from role name.
-                    # This string template essentially uses cloudflare_account_roles (https://registry.terraform.io/providers/cloudflare/cloudflare/latest/docs/data-sources/account_roles)
-                    # data source to get role id corresponding to a role name. We populate this string template for every role name listed.
-                    f'%{{ for role in data.cloudflare_account_roles.{safe_resource_id(cf_user.account_name)}.roles ~}}  %{{if role.name == "{each}" ~}}${{role.id}}%{{ endif ~}}  %{{ endfor ~}}'
-                    for each in cf_user.roles
-                ],
-                "cloudflare_account_roles": data_source_cloudflare_account_roles,
-            }
-            specs.append(
-                _get_external_spec_from_resource(
-                    cloudflare_account_member, cf_user.account_name
-                )
-            )
-
-    return specs
-
-
-def _get_external_spec_from_resource(
-    resource: MutableMapping[Any, Any], account_name: str
-) -> ExternalResourceSpec:
-    return ExternalResourceSpec(
-        provision_provider="cloudflare",
-        provisioner={"name": f"{account_name}"},
-        namespace={},
-        resource=resource,
-    )

reconcile/typed_queries/cloudflare.py

@@ -1,10 +0,0 @@
-from reconcile.gql_definitions.terraform_cloudflare_resources.terraform_cloudflare_accounts import (
-    CloudflareAccountV1,
-    query,
-)
-from reconcile.utils import gql
-
-
-def get_cloudflare_accounts() -> list[CloudflareAccountV1]:
-    data = query(gql.get_api().query)
-    return list(data.accounts or [])

reconcile/utils/terrascript/cloudflare_client.py

@@ -1,310 +0,0 @@
-import tempfile
-from abc import (
-    ABC,
-    abstractmethod,
-)
-from collections.abc import Iterable
-from dataclasses import dataclass
-
-from terrascript import (
-    Backend,
-    Data,
-    Output,
-    Resource,
-    Terraform,
-    Terrascript,
-    Variable,
-    provider,
-)
-
-from reconcile.cli import TERRAFORM_VERSION
-from reconcile.utils.exceptions import SecretIncompleteError
-from reconcile.utils.external_resource_spec import (
-    ExternalResourceSpec,
-    ExternalResourceSpecInventory,
-)
-from reconcile.utils.secret_reader import SecretReaderBase
-from reconcile.utils.terraform.config import TerraformS3BackendConfig
-from reconcile.utils.terraform.config_client import TerraformConfigClient
-from reconcile.utils.terrascript.cloudflare_resources import (
-    cloudflare_account,
-    create_cloudflare_terrascript_resource,
-)
-from reconcile.utils.terrascript.models import (
-    CloudflareAccount,
-    Integration,
-    TerraformStateS3,
-)
-
-TMP_DIR_PREFIX = "terrascript-cloudflare-"
-
-DEFAULT_CLOUDFLARE_ACCOUNT_TYPE = "standard"
-DEFAULT_CLOUDFLARE_ACCOUNT_2FA = False
-DEFAULT_IS_MANAGED_CLOUDFLARE_ACCOUNT = True
-DEFAULT_PROVIDER_RPS = 4
-
-
-@dataclass
-class CloudflareAccountConfig:
-    """Configuration related to authenticating API calls to Cloudflare."""
-
-    name: str
-    api_token: str
-    account_id: str
-    enforce_twofactor: bool = DEFAULT_CLOUDFLARE_ACCOUNT_2FA
-    type: str = DEFAULT_CLOUDFLARE_ACCOUNT_TYPE
-    is_managed_account: bool = DEFAULT_IS_MANAGED_CLOUDFLARE_ACCOUNT
-
-
-def create_cloudflare_terrascript(
-    account_config: CloudflareAccountConfig,
-    backend_config: TerraformS3BackendConfig,
-    provider_version: str,
-    provider_rps: int = DEFAULT_PROVIDER_RPS,
-    is_managed_account: bool = True,
-) -> Terrascript:
-    """
-    Configures a Terrascript class with the required provider(s) and backend
-    configuration.
-
-    This is offloaded to a separate function to avoid mixing additional
-    logic into TerrascriptCloudflareClient.
-
-    :param account_config: CloudflareAccount configuration.
-    :param backend_config: S3 as backend to store Terraform state.
-    :param provider_version: Terraform Cloudflare provider version.
-    :is_managed_account:
-            If the target cloudflare account is being managed by the caller or not.
-            Currently this is deferred to terraform-cloudflare-resources.
-            Until further improvement(Tracked by APPSRE-7035),
-            this argument can be set to False in other integrations.
-            Defaults to True.
-
-    :return: a Terrascript object that contains corresponding resources
-    """
-    terrascript = Terrascript()
-
-    backend = Backend(
-        "s3",
-        access_key=backend_config.access_key,
-        secret_key=backend_config.secret_key,
-        bucket=backend_config.bucket,
-        key=backend_config.key,
-        region=backend_config.region,
-    )
-
-    required_providers = {
-        "cloudflare": {
-            "source": "cloudflare/cloudflare",
-            "version": provider_version,
-        }
-    }
-
-    terrascript += Terraform(
-        backend=backend,
-        required_providers=required_providers,
-        required_version=TERRAFORM_VERSION[0],
-    )
-
-    cloudflare_provider_values = {
-        "api_token": account_config.api_token,
-        "rps": provider_rps,
-    }
-    if provider_version.startswith("3"):
-        cloudflare_provider_values["account_id"] = (
-            account_config.account_id
-        )  # needed for some resources, see note below
-
-    terrascript += provider.cloudflare(**cloudflare_provider_values)
-
-    cloudflare_account_values = {
-        "name": account_config.name,
-        "enforce_twofactor": account_config.enforce_twofactor,
-        "type": account_config.type,
-    }
-
-    if is_managed_account:
-        terrascript += cloudflare_account(
-            account_config.name,
-            **cloudflare_account_values,
-        )
-
-    # Some resources need "account_id" to be set at the resource level
-    # The cloudflare provider is being migrated from settings account_id at the provider
-    # level to requiring it at the resource level, for resources that needs it.
-    # This is also listed in version 4.x breaking changes:
-    #   https://github.com/cloudflare/terraform-provider-cloudflare/issues/1646
-    terrascript += Variable(
-        "account_id", type="string", default=account_config.account_id
-    )
-
-    return terrascript
-
-
-class TerrascriptCloudflareClient(TerraformConfigClient):
-    """
-    Build the Terrascript configuration, collect resources, and return Terraform JSON
-    configuration.
-
-    There's actually very little that's specific to Cloudflare in this class. This could
-    become a more general TerrascriptClient that could in theory support any resource
-    types with some minor modifications to how resource classes (self._resource_classes)
-    are tracked.
-    """
-
-    def __init__(
-        self,
-        ts_client: Terrascript,
-    ) -> None:
-        self._terrascript = ts_client
-        self._resource_specs: ExternalResourceSpecInventory = {}
-
-    def add_spec(self, spec: ExternalResourceSpec) -> None:
-        self._resource_specs[spec.id_object()] = spec
-
-    def populate_resources(self) -> None:
-        """
-        Add the resource spec to Terrascript using the resource-specific classes
-        to determine which resources to create.
-        """
-        for spec in self._resource_specs.values():
-            resources_to_add = create_cloudflare_terrascript_resource(spec)
-            self._add_resources(resources_to_add)
-
-    def dump(self, existing_dir: str | None = None) -> str:
-        """Write the Terraform JSON representation of the resources to disk"""
-        if existing_dir is None:
-            working_dir = tempfile.mkdtemp(prefix=TMP_DIR_PREFIX)
-        else:
-            working_dir = existing_dir
-        with open(
-            working_dir + "/config.tf.json", "w", encoding="locale"
-        ) as terraform_config_file:
-            terraform_config_file.write(self.dumps())
-
-        return working_dir
-
-    def dumps(self) -> str:
-        """Return the Terraform JSON representation of the resources"""
-        return str(self._terrascript)
-
-    def _add_resources(self, tf_resources: Iterable[Resource | Output | Data]) -> None:
-        for resource in tf_resources:
-            self._terrascript.add(resource)
-
-
-class TerraformS3StateNamingStrategy(ABC):
-    @abstractmethod
-    def get_object_key(self, qr_integration: Integration) -> str:
-        pass
-
-
-class Default(TerraformS3StateNamingStrategy):
-    def get_object_key(self, qr_integration: Integration) -> str:
-        return qr_integration.key
-
-
-class AccountShardingStrategy(TerraformS3StateNamingStrategy):
-    """
-    This strategy is in place until we solve for keyStrategy as specified in
-    https://issues.redhat.com/browse/APPSRE-6933
-    """
-
-    def __init__(self, account: CloudflareAccount):
-        super().__init__()
-        self.account: CloudflareAccount = account
-
-    def get_object_key(self, qr_integration: Integration) -> str:
-        return f"{qr_integration.name}-{self.account.name}.tfstate"
-
-
-class DNSZoneShardingStrategy(TerraformS3StateNamingStrategy):
-    def __init__(self, account: CloudflareAccount, zone_identifier: str):
-        super().__init__()
-        self.account: CloudflareAccount = account
-        self.zone: str = zone_identifier
-
-    def get_object_key(self, qr_integration: Integration) -> str:
-        old_integration_key = qr_integration.name.replace(
-            "-", "_"
-        )  # This is because the state file was already created using this name before the refactoring
-        return f"{old_integration_key}-{self.account.name}-{self.zone}.tfstate"
-
-
-class TerrascriptCloudflareClientFactory:
-    @staticmethod
-    def _create_backend_config(
-        tf_state_s3: TerraformStateS3, key: str, secret_reader: SecretReaderBase
-    ) -> TerraformS3BackendConfig:
-        aws_acct_creds = secret_reader.read_all_secret(tf_state_s3.automation_token)
-        aws_access_key_id = aws_acct_creds.get("aws_access_key_id")
-        aws_secret_access_key = aws_acct_creds.get("aws_secret_access_key")
-        if not aws_access_key_id or not aws_secret_access_key:
-            raise SecretIncompleteError(
-                f"secret {tf_state_s3.automation_token} incomplete: aws_access_key_id and/or aws_secret_access_key missing"
-            )
-
-        return TerraformS3BackendConfig(
-            aws_access_key_id,
-            aws_secret_access_key,
-            tf_state_s3.bucket,
-            key,
-            tf_state_s3.region,
-        )
-
-    @staticmethod
-    def _create_cloudflare_account_config(
-        cf_acct: CloudflareAccount, secret_reader: SecretReaderBase
-    ) -> CloudflareAccountConfig:
-        cf_acct_creds = secret_reader.read_all_secret(cf_acct.api_credentials)
-        cf_acct_config = CloudflareAccountConfig(
-            cf_acct.name,
-            cf_acct_creds["api_token"],
-            cf_acct_creds["account_id"],
-            cf_acct.enforce_twofactor or DEFAULT_CLOUDFLARE_ACCOUNT_2FA,
-            cf_acct.type or DEFAULT_CLOUDFLARE_ACCOUNT_TYPE,
-        )
-        return cf_acct_config
-
-    @classmethod
-    def get_client(
-        cls,
-        tf_state_s3: TerraformStateS3,
-        cf_acct: CloudflareAccount,
-        sharding_strategy: TerraformS3StateNamingStrategy | None,
-        secret_reader: SecretReaderBase,
-        is_managed_account: bool,
-        provider_rps: int = DEFAULT_PROVIDER_RPS,
-    ) -> TerrascriptCloudflareClient:
-        key = _get_terraform_s3_state_key_name(
-            tf_state_s3.integration, sharding_strategy
-        )
-        backend_config = cls._create_backend_config(tf_state_s3, key, secret_reader)
-        cf_acct_config = cls._create_cloudflare_account_config(cf_acct, secret_reader)
-        ts_config = create_cloudflare_terrascript(
-            cf_acct_config,
-            backend_config,
-            cf_acct.provider_version,
-            provider_rps=provider_rps,
-            is_managed_account=is_managed_account,
-        )
-        client = TerrascriptCloudflareClient(ts_config)
-        return client
-
-
-def _get_terraform_s3_state_key_name(
-    integration: Integration,
-    sharding_strategy: TerraformS3StateNamingStrategy | None,
-) -> str:
-    if sharding_strategy is None:
-        sharding_strategy = Default()
-
-    return sharding_strategy.get_object_key(integration)
-
-
-class IntegrationUndefinedError(Exception):
-    pass
-
-
-class InvalidTerraformStateError(Exception):
-    pass